Add code files

CompareRegressors.py

@@ -0,0 +1,227 @@
+# 786/110
+
+import os
+import sys
+import time
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+import DataUtils
+import Settings as settings
+from DataUtils import generate_random_eqn, multidim_dataset_from_eqn_list
+from DataUtils import simple_eqn_to_str
+from MLP_Model import MLP_Model
+# from models.smtree_model import  SMTree_Model
+# from models.smftree_model import SMFTree_Model
+# from models.smfftree_model import SMFFTree_Model
+# from models.addoptree_model import AddOpTree_Model
+# from models.crtree_model import CRTree_Model
+# from models.sptree_model import SpTree_Model
+# from models.lbl_model import LBLTree_Model
+from SymbolicFunctionLearner import SFL
+# from models.mcts_model import MCTS_Model
+from gp_model import Genetic_Model
+
+""" Test Hyperparameters """
+num_eqns_to_create = 150
+num_training = 1000
+num_valid = 1000
+num_test = 1000
+allowable_ops = settings.function_set.copy()
+num_vars = 1  # 10
+num_max_levels = 3  # 4
+
+settings.n_tree_layers = num_max_levels
+settings.num_features = num_vars
+settings.show_output = False
+settings.keep_logs = False
+settings.mode = "sr"
+
+show_found_eqns = True
+
+use_constants_in_eqns = True
+
+if not os.path.exists('images'):
+    os.makedirs('images')
+
+
+def plot_all_models(eqn_str, eqn_number, all_models, this_train_x, this_train_y,
+                    this_test_x, this_test_y):
+    plt.figure()
+    plt.title('Compare models: {}'.format(eqn_str))
+
+    min_y = np.min(this_test_y)
+    max_y = np.max(this_test_y)
+    y_range = max_y - min_y
+
+    plt.scatter(this_train_x, this_train_y, color='xkcd:dark pink',
+                marker='o', s=100, label='Training set')
+
+    plt.scatter(this_test_x, this_test_y, color='gray', alpha=0.5, marker='.', label='Ground truth')
+
+    for i in range(len(all_models)):
+        this_model = all_models[i]
+        test_hat_y = this_model.predict(this_test_x)
+        plt.scatter(this_test_x,
+                    test_hat_y, alpha=0.7, marker='.', label='{}'.format(this_model.name))
+
+    for xc in settings.train_scope:
+        plt.axvline(x=xc, color='k', linestyle='dashed', linewidth=2)
+
+    plt.ylim([min_y - 0.5 * y_range, max_y + 0.5 * y_range])
+
+    plt.legend()
+    plt.savefig("images/eqn_{}.png".format(eqn_number))
+    plt.close()
+
+
+models_to_test = []
+models_to_test.append(Genetic_Model)
+models_to_test.append(MLP_Model)
+# models_to_test.append(Tree_Model)
+
+models_to_test.append(SFL)
+# models_to_test.append(SMTree_Model)
+# models_to_test.append(SMFTree_Model)
+# models_to_test.append(SMFFTree_Model)
+# models_to_test.append(AddOpTree_Model)
+
+# models_to_test.append(SpTree_Model)
+
+
+lists_of_error_scores = []
+lists_of_iter_times = []
+all_models = []
+for model_type in models_to_test:
+    all_models.append(model_type())
+    lists_of_error_scores.append([])
+    lists_of_iter_times.append([])
+
+# all_models.append(SpTree_Model(use_scopers=True))
+# lists_of_error_scores.append([])
+# lists_of_iter_times.append([])
+
+seen_eqns = []
+winning_entries = []
+valid_err = 0
+
+print("Starting program.")
+print(settings.num_features)
+
+start_time = time.time()
+
+for eqn_n in range(1, num_eqns_to_create + 1):
+    print("----------------")
+    # Create some random equation
+    current_eqn_as_list = generate_random_eqn(allowable_ops, num_vars, num_max_levels,
+                                              allow_constants=use_constants_in_eqns)
+    current_eqn_as_str = simple_eqn_to_str(current_eqn_as_list)
+
+    # should be a new equation
+    while current_eqn_as_str in seen_eqns:
+        current_eqn_as_list = generate_random_eqn(allowable_ops, num_vars, num_max_levels,
+                                                  allow_constants=use_constants_in_eqns)
+        current_eqn_as_str = simple_eqn_to_str(current_eqn_as_list)
+    seen_eqns.append(current_eqn_as_str)
+
+    print(current_eqn_as_list)
+    print("Random equation {} of {}".format(eqn_n, num_eqns_to_create))
+    print("True function:     {}\n".format(current_eqn_as_str))
+    with open("images/compare_test_output.txt", "a") as output_file:
+        output_file.write("\n{}\nTrue equation:\n{}\n".format(eqn_n, current_eqn_as_str))
+
+    # Create a dataset with that equation
+    train_data_x, train_data_y = multidim_dataset_from_eqn_list(current_eqn_as_list, num_training, n_vars=num_vars)
+    test_data_x, test_data_y = multidim_dataset_from_eqn_list(current_eqn_as_list, num_test, n_vars=num_vars,
+                                                              min_x=settings.test_scope[0],
+                                                              max_x=settings.test_scope[1])
+
+    # train_data_x, train_data_y = create_dataset_from_eqn_list(current_eqn_as_list, num_vars, num_training,
+    #                                                           settings.train_scope[0], settings.train_scope[1])
+    # test_data_x, test_data_y = create_dataset_from_eqn_list(current_eqn_as_list, num_vars, num_test,
+    #                                                         settings.test_scope[0], settings.test_scope[1])
+
+    for model_i in range(len(all_models)):
+        model = all_models[model_i]
+        model.reset()
+        itertime_start = time.time()
+        # if False:  # model_i >= len(all_models) - 2:
+        #     model.repeat_train(train_data_x, train_data_y)
+        # else:
+        model_eqn, _, best_err = model.repeat_train(train_data_x, train_data_y,
+                                                    test_x=test_data_x, test_y=test_data_y,
+                                                    verbose=False)
+        if show_found_eqns:
+            print("{} function:  {}".format(model.name, model_eqn)[:550])
+
+        # Test model on that equation
+        # test_err = model.test(test_data_x, test_data_y)
+        test_err = max(np.exp(-10), best_err)  # data_utils.test_from_formula(model_eqn, test_data_x, test_data_y)
+        print(" ---> {} Test Error: {:.5f}".format(model.short_name, test_err))
+        lists_of_error_scores[model_i].extend([test_err])
+        lists_of_iter_times[model_i].append(time.time() - itertime_start)
+        sys.stdout.flush()
+
+        # y_gold_list = list(model.sess.run(model.y_gold,
+        #                                  feed_dict={model.data_x: np.reshape(test_data_x[:4][:], [-1, num_vars]),
+        #                                             model.data_y: np.reshape(test_data_y[:4][:], [-1, 1]),
+        #                                             model.var_random_u: model.var_random_selector,
+        #                                             model.op_random_u: model.op_random_selector}).reshape(1, -1)[0])
+        # y_hat_list2 = data_utils.predict_from_formula(model.get_simple_formula(digits=4), test_data_x[:4][:])
+        #
+        # print('Performance on sample validation data:')
+        # for feature_i in range(model.n_input_variables):
+        #     print('x{}:      '.format(feature_i + 1),
+        #           ['{:7.4f}'.format(yyy[feature_i]) for yyy in test_data_x[:4][:]])
+        # print("-----------------------------------------------------")
+        # print('y_gold: ', ['{:7.4f}'.format(yyy) for yyy in y_gold_list])
+        # print('y_hat2: ', ['{:7.4f}'.format(yyy) for yyy in y_hat_list2])
+
+        if test_err < 0.035:
+            winning_entries.append("{} - {}".format(current_eqn_as_str, model.short_name))
+
+        with open("images/compare_test_output.txt", "a") as output_file:
+            output_file.write("{}: {}\n{}\n".format(model.short_name, test_err, model_eqn))
+
+    print()
+
+    DataUtils.plot_hist_of_errors(lists_of_error_scores, all_models, eqn_n)
+
+    # todo: something is wrong with this function!!
+    # todo: it plots the model as is (after last iter), not at best iter, of repeat_train
+    # DataUtils.plot_all_models_predicted_actual(all_models, test_data_x, test_data_y,
+    #                                            set_name="Eqn {}: {}".format(eqn_n, current_eqn_as_str),
+    #                                            fig_name="eqn_{}".format(eqn_n))
+
+    plt.figure()
+    x_axis = [i + 1 for i in range(eqn_n)]
+    for iter_times_i in range(len(lists_of_iter_times)):
+        plt.plot(x_axis, lists_of_iter_times[iter_times_i], label=all_models[iter_times_i].short_name)
+    plt.xlabel("Iteration")
+    plt.ylabel("Running time")
+    plt.legend()
+    plt.savefig("images/time_curve.png")
+    plt.close()
+
+    # if num_vars == 1:
+    #     plot_all_models(current_eqn_as_str, eqn_n, all_models,
+    #                     train_data_x, train_data_y,
+    #                     test_data_x, test_data_y)
+
+running_time = time.time() - start_time
+print("Done. Took {} seconds.\n".format(running_time))
+#
+# plt.figure()
+# plt.hist([list_of_error_scores_GP, list_of_error_scores_MLP, list_of_error_scores_Tree], label=['GP', 'MLP', 'Tree'])
+# plt.legend(loc='upper right')
+# plt.show()
+#
+# plt.figure()
+# plt.hist(list_of_error_scores_GP, edgecolor='k', linewidth=1.2)
+# plt.show()
+#
+# plt.figure()
+# plt.hist(list_of_error_scores_GP, cumulative=True)
+# plt.show()
+#

DESolver.py

@@ -0,0 +1,64 @@
+""""""""""""""""""""""""""""""""" 
+This file is for running.
+Do not modify this file.
+
+For running: DiffEqnSolver.py
+For modifying: settings.py
+"""""""""""""""""""""""""""""""""
+
+import os
+import time
+
+import DataUtils
+import Settings as settings
+from SymbolicFunctionLearner import SFL
+
+settings.mode = "de"
+
+current_model = SFL()
+if not os.path.exists('images'):
+    os.makedirs('images')
+
+print('\nBeginning experiment: {}'.format(current_model.name))
+
+print("{} tree layers.".format(settings.n_tree_layers))
+print("{} features of {} component(s) each.".format(settings.num_features, settings.num_dims_per_feature))
+print("{} components in output.".format(settings.n_dims_in_output))
+print("{} operators: {}.".format(len(current_model.function_set),
+                                 current_model.function_set))
+
+train_errors = []
+valid_errors = []
+test_errors = []
+true_eqns = []
+
+train_X = DataUtils.generate_data(settings.train_N, n_vars=current_model.n_input_variables,
+                                  avoid_zero=settings.avoid_zero)
+valid_X = DataUtils.generate_data(settings.train_N, n_vars=current_model.n_input_variables,
+                                  avoid_zero=settings.avoid_zero)
+test_X = DataUtils.generate_data(settings.test_N, n_vars=current_model.n_input_variables,
+                                 min_x=settings.test_scope[0],
+                                 max_x=settings.test_scope[1])
+
+print("\n========================")
+print("Starting Solver.")
+print("==========================\n")
+
+# Train the model from scratch several times, keeping the best one.
+start_time = time.time()
+best_model, best_iter, best_err = current_model.repeat_train(train_X,
+                                                             num_repeats=settings.num_train_repeat_processes,
+                                                             test_x=test_X)
+running_time = time.time() - start_time
+
+print("best_model: {}".format(best_model))
+print("----------------------")
+print("Finished DE. Took {:.2f} minutes.\n".format(running_time / 60))
+
+
+print("Final solution found at attempt {}:".format(best_iter))
+print("y = {}".format(best_model))
+print("Test error:  {}".format(best_err))
+if best_err < 0.02:
+    print("Attained error less than 0.02 - great!")
+print()

DataUtils.py

@@ -0,0 +1,801 @@
+""""""""""""""""""""""""""""""""" 
+Do not run or modify this file.
+
+For running: DiffEqnSolver.py
+For modifying: settings.py
+"""""""""""""""""""""""""""""""""
+
+import matplotlib.pyplot as plt
+import numpy as np
+import sympy
+import tensorflow as tf
+from matplotlib import cm
+from sympy import expand, sympify, lambdify
+
+import Settings
+
+
+def safe_abs(x):
+    return np.sqrt(x * x + Settings.eps)
+
+
+def safe_div(x, y):
+    return np.sign(y) * x / safe_abs(y)
+
+
+def tf_diff_abs(x):
+    return tf.sqrt(tf.square(x) + Settings.eps)
+
+
+def tf_diff_sqrt(x):
+    return tf.sqrt(tf_diff_abs(x))
+
+
+def tf_diff_log(x):
+    return tf.math.log(tf_diff_abs(x))
+
+
+def our_tanh(x, factor=1000):
+    return factor * tf.tanh(x / factor)
+
+
+def spike(x):
+    return 1.0 / (1 + 200 * tf.square(x))
+    # return tf.math.exp(-10 * tf.square(x))
+
+
+def true_function(input_x):
+    return predict_from_formula(Settings.true_eqn, input_x)
+
+
+def is_float(value):
+    try:
+        float(value)
+        return True
+    except ValueError:
+        return False
+
+
+# Function to generate random equation as operator/input list
+# Variables are numbered 1 ... n, and 0 does not appear
+# Constants appear as [float] e.g [3.14]
+def generate_random_eqn_two_list(op_list, n_vars, n_levels, allow_constants=True):
+    eqn_ops = list(np.random.choice(op_list, size=int(2 ** n_levels) - 1, replace=True))
+
+    if allow_constants:
+        eqn_vars = list(np.random.choice(range(1, max(int(n_vars * 1.6), n_vars + 2)),
+                                         size=int(2 ** n_levels), replace=True))
+        for i in range(len(eqn_vars)):
+            if eqn_vars[i] >= n_vars + 1:
+                eqn_vars[i] = [np.random.uniform(Settings.test_scope[0], Settings.test_scope[1])]
+    else:
+        eqn_vars = list(np.random.choice(range(1, 1 + n_vars), size=int(2 ** n_levels), replace=True))
+    return [eqn_ops, eqn_vars]
+
+
+# Function to generate random equation as operator/input list and weight/bias list
+# Variables are numbered 1 ... n, and 0 does not appear
+# Constants appear in weight and bias lists.
+# const_ratio determines how many weights are not 1, and how many biases are not 0
+def generate_random_eqn(op_list, n_vars, n_levels, allow_constants=True, const_ratio=0.8):
+    eqn_ops = list(np.random.choice(op_list, size=int(2 ** n_levels) - 1, replace=True))
+    eqn_vars = list(np.random.choice(range(1, (n_vars + 1)), size=int(2 ** n_levels), replace=True))
+    max_bound = max(np.abs(Settings.test_scope[0]), np.abs(Settings.test_scope[1]))
+    eqn_weights = list(np.random.uniform(-1 * max_bound, max_bound, size=len(eqn_vars)))
+    eqn_biases = list(np.random.uniform(-1 * max_bound, max_bound, size=len(eqn_vars)))
+
+    if not allow_constants:
+        const_ratio = 0.0
+    random_const_chooser_w = np.random.uniform(0, 1, len(eqn_weights))
+    random_const_chooser_b = np.random.uniform(0, 1, len(eqn_biases))
+
+    for i in range(len(eqn_weights)):
+        if random_const_chooser_w[i] >= const_ratio:
+            eqn_weights[i] = 1
+        if random_const_chooser_b[i] >= const_ratio:
+            eqn_biases[i] = 0
+
+    return [eqn_ops, eqn_vars, eqn_weights, eqn_biases]
+
+
+# Function to create a multidim input data set given an operator/input list
+def generate_data(n_points, n_vars=Settings.num_features,
+                  n_input_dims=Settings.num_dims_per_feature,
+                  min_x=Settings.train_scope[0], max_x=Settings.train_scope[1],
+                  avoid_zero=False):
+    if not avoid_zero:
+        x_data = [np.random.uniform(min_x, max_x, size=[n_input_dims, n_vars]) for _ in range(n_points)]
+    else:
+        x_data = []
+        for _ in range(n_points):
+            candidate = np.random.uniform(min_x, max_x, size=[n_input_dims, n_vars])
+            while np.linalg.norm(candidate) < 0.1:
+                candidate = np.random.uniform(min_x, max_x, size=[n_input_dims, n_vars])
+            x_data.append(candidate)
+    return np.array(x_data)
+
+
+# Function to create a data set given an operator/input list
+def create_dataset_from_eqn_list(eqn_as_list, n_vars, n_points, min_x, max_x):
+    x_data = [list(np.random.uniform(min_x, max_x, n_vars)) for _ in range(n_points)]
+    y_data = [evaluate_eqn_list_on_datum(eqn_as_list, x_data_i) + np.random.normal(0, 0.05) for x_data_i in x_data]
+    return [np.array(x_data), np.array(y_data)]
+
+
+# Function to create a multidim data set given an operator/input list
+def multidim_dataset_from_eqn_list(eqn_as_list, n_points,
+                                   n_vars=Settings.num_features,
+                                   n_input_dims=Settings.num_dims_per_feature,
+                                   n_output_dims=Settings.n_dims_in_output,
+                                   min_x=Settings.train_scope[0], max_x=Settings.train_scope[1],
+                                   avoid_zero=False):
+    if not avoid_zero:
+        x_data = [np.random.uniform(min_x, max_x, size=[n_input_dims, n_vars]) for _ in range(n_points)]
+    else:
+        x_data = []
+        for _ in range(n_points):
+            candidate = np.random.uniform(min_x, max_x, size=[n_input_dims, n_vars])
+            while np.linalg.norm(candidate) < 0.1:
+                candidate = np.random.uniform(min_x, max_x, size=[n_input_dims, n_vars])
+            x_data.append(candidate)
+    y_data = [evaluate_eqn_list_on_multidim_datum(eqn_as_list, x_data_i)  # + np.random.normal(0, 0.05)
+              for x_data_i in x_data]
+    if n_output_dims == 1:
+        y_data = [np.mean(old_y) for old_y in y_data]
+    return [np.array(x_data), np.reshape(np.array(y_data), [n_points, n_output_dims, 1])]
+
+
+def make_y_multi_safe(old_y, n_dims_per_input_var=1, n_dims_in_output=1):
+    if isinstance(old_y, list):
+        new_y = np.array(old_y)
+        new_y.reshape([-1, n_dims_in_output, 1])
+    else:
+        new_y = old_y.copy()
+    if len(new_y.shape) == 1:
+        assert (n_dims_in_output == 1)
+        new_y = [[[y_value] for _ in range(n_dims_per_input_var)] for y_value in new_y]
+        new_y = np.array(new_y)
+    elif len(new_y.shape) == 2:
+        assert (n_dims_in_output == 1)
+        new_y = [[y_value for _ in range(n_dims_per_input_var)] for y_value in new_y]
+        new_y = np.array(new_y)
+    elif new_y.shape[1] < n_dims_per_input_var:
+        assert (n_dims_in_output == 1)
+        new_y = [[y_value[0] for _ in range(n_dims_per_input_var)] for y_value in new_y]
+        new_y = np.array(new_y)
+    return new_y
+
+
+# Function to evaluate equation (in two-list format) on a data point
+def evaluate_eqn_list_on_datum_two_list(eqn_as_list, input_x):
+    eqn_ops = eqn_as_list[0]
+    eqn_vars = eqn_as_list[1]
+    current_op = eqn_ops[0]
+
+    if len(eqn_ops) == 1:
+        if type(eqn_vars[0]) is list:
+            left_side = eqn_vars[0][0]
+        else:
+            left_side = input_x[eqn_vars[0] - 1]
+        if type(eqn_vars[1]) is list:
+            right_side = eqn_vars[1][0]
+        else:
+            right_side = input_x[eqn_vars[1] - 1]
+
+    else:
+        split_point = int((len(eqn_ops) + 1) / 2)
+        left_ops = eqn_ops[1:split_point]
+        right_ops = eqn_ops[split_point:]
+
+        left_vars = eqn_vars[:split_point]
+        right_vars = eqn_vars[split_point:]
+
+        left_side = evaluate_eqn_list_on_datum_two_list([left_ops, left_vars], input_x)
+        right_side = evaluate_eqn_list_on_datum_two_list([right_ops, right_vars], input_x)
+
+    if current_op == 'id':
+        return left_side
+
+    if current_op == 'sqrt':
+        return np.sqrt(np.abs(left_side))
+
+    if current_op == 'log':
+        return np.log(np.sqrt(left_side * left_side + 1e-10))
+
+    if current_op == 'sin':
+        return np.sin(left_side)
+
+    if current_op == 'exp':
+        return np.exp(left_side)
+
+    if current_op == 'add':
+        return left_side + right_side
+
+    if current_op == 'mul':
+        return left_side * right_side
+
+    if current_op == 'sub':
+        return left_side - right_side
+
+    if current_op == 'div':
+        return safe_div(left_side, right_side)
+
+    return None
+
+
+# Function to evaluate equation (in list format) on a data point
+def evaluate_eqn_list_on_datum(eqn_as_list, input_x):
+    eqn_ops = eqn_as_list[0]
+    eqn_vars = eqn_as_list[1]
+    eqn_weights = eqn_as_list[2]
+    eqn_biases = eqn_as_list[3]
+    current_op = eqn_ops[0]
+
+    if len(eqn_ops) == 1:
+        left_side = eqn_weights[0] * input_x[eqn_vars[0] - 1] + eqn_biases[0]
+        right_side = eqn_weights[1] * input_x[eqn_vars[1] - 1] + eqn_biases[1]
+
+    else:
+        split_point = int((len(eqn_ops) + 1) / 2)
+        left_ops = eqn_ops[1:split_point]
+        right_ops = eqn_ops[split_point:]
+
+        left_vars = eqn_vars[:split_point]
+        right_vars = eqn_vars[split_point:]
+
+        left_weights = eqn_weights[:split_point]
+        right_weights = eqn_weights[split_point:]
+
+        left_biases = eqn_biases[:split_point]
+        right_biases = eqn_biases[split_point:]
+
+        left_side = evaluate_eqn_list_on_datum([left_ops, left_vars, left_weights, left_biases], input_x)
+        right_side = evaluate_eqn_list_on_datum([right_ops, right_vars, right_weights, right_biases], input_x)
+
+    if current_op == 'id':
+        return left_side
+
+    if current_op == 'sqrt':
+        return np.sqrt(np.abs(left_side))
+
+    if current_op == 'log':
+        return np.log(np.sqrt(left_side * left_side + 1e-10))
+
+    if current_op == 'sin':
+        return np.sin(left_side)
+
+    if current_op == 'exp':
+        return np.exp(left_side)
+
+    if current_op == 'add':
+        return left_side + right_side
+
+    if current_op == 'mul':
+        return left_side * right_side
+
+    if current_op == 'sub':
+        return left_side - right_side
+
+    if current_op == 'div':
+        return safe_div(left_side, right_side)
+
+    return None
+
+
+# Function to evaluate equation (in two-list format) on a data point
+def evaluate_eqn_list_on_multidim_datum_two_list(eqn_as_list, input_x):
+    eqn_ops = eqn_as_list[0]
+    eqn_vars = eqn_as_list[1]
+    current_op = eqn_ops[0]
+
+    if len(eqn_ops) == 1:
+        if type(eqn_vars[0]) is list:
+            left_side = eqn_vars[0][0]
+        else:
+            left_side = input_x[:, eqn_vars[0] - 1]
+        if type(eqn_vars[1]) is list:
+            right_side = eqn_vars[1][0]
+        else:
+            right_side = input_x[:, eqn_vars[1] - 1]
+
+    else:
+        split_point = int((len(eqn_ops) + 1) / 2)
+        left_ops = eqn_ops[1:split_point]
+        right_ops = eqn_ops[split_point:]
+
+        left_vars = eqn_vars[:split_point]
+        right_vars = eqn_vars[split_point:]
+
+        left_side = evaluate_eqn_list_on_multidim_datum_two_list([left_ops, left_vars], input_x)
+        right_side = evaluate_eqn_list_on_multidim_datum_two_list([right_ops, right_vars], input_x)
+
+    if current_op == 'id':
+        return left_side
+
+    if current_op == 'sqrt':
+        return np.sqrt(np.abs(left_side))
+
+    if current_op == 'log':
+        return np.log(np.sqrt(left_side * left_side + 1e-10))
+
+    if current_op == 'sin':
+        return np.sin(left_side)
+
+    if current_op == 'exp':
+        return np.exp(left_side)
+
+    if current_op == 'add':
+        return left_side + right_side
+
+    if current_op == 'mul':
+        return left_side * right_side
+
+    if current_op == 'sub':
+        return left_side - right_side
+
+    if current_op == 'div':
+        return safe_div(left_side, right_side)
+    return None
+
+
+# Function to evaluate equation (in list format) on a data point
+def evaluate_eqn_list_on_multidim_datum(eqn_as_list, input_x):
+    eqn_ops = eqn_as_list[0]
+    eqn_vars = eqn_as_list[1]
+    eqn_weights = eqn_as_list[2]
+    eqn_biases = eqn_as_list[3]
+    current_op = eqn_ops[0]
+
+    if len(eqn_ops) == 1:
+        left_side = eqn_weights[0] * input_x[:, eqn_vars[0] - 1] + eqn_biases[0]
+        right_side = eqn_weights[1] * input_x[:, eqn_vars[1] - 1] + eqn_biases[1]
+
+    else:
+        split_point = int((len(eqn_ops) + 1) / 2)
+        left_ops = eqn_ops[1:split_point]
+        right_ops = eqn_ops[split_point:]
+
+        left_vars = eqn_vars[:split_point]
+        right_vars = eqn_vars[split_point:]
+
+        left_weights = eqn_weights[:split_point]
+        right_weights = eqn_weights[split_point:]
+
+        left_biases = eqn_biases[:split_point]
+        right_biases = eqn_biases[split_point:]
+
+        left_side = evaluate_eqn_list_on_multidim_datum([left_ops, left_vars, left_weights, left_biases], input_x)
+        right_side = evaluate_eqn_list_on_multidim_datum([right_ops, right_vars, right_weights, right_biases], input_x)
+
+    if current_op == 'id':
+        return left_side
+
+    if current_op == 'sqrt':
+        return np.sqrt(np.abs(left_side))
+
+    if current_op == 'log':
+        return np.log(np.sqrt(left_side * left_side + 1e-10))
+
+    if current_op == 'sin':
+        return np.sin(left_side)
+
+    if current_op == 'exp':
+        return np.exp(left_side)
+
+    if current_op == 'add':
+        return left_side + right_side
+
+    if current_op == 'mul':
+        return left_side * right_side
+
+    if current_op == 'sub':
+        return left_side - right_side
+
+    if current_op == 'div':
+        return safe_div(left_side, right_side)
+    return None
+
+
+
+def choices_to_init_weight_matrix(choice_list, all_choices):
+    init_weight_matrix = np.zeros(shape=[len(choice_list), len(all_choices)])
+    for row in range(len(choice_list)):
+        if choice_list[row] in all_choices:
+            init_weight_matrix[row][all_choices.index(choice_list[row])] = Settings.init_weight_value
+        elif isinstance(choice_list[row], str) and "not" in choice_list[row] and choice_list[row].index("not") == 0 and \
+                        choice_list[row][len("not"):] in all_choices:
+            init_weight_matrix[row][all_choices.index(choice_list[row][len("not"):])] = -100
+    return init_weight_matrix.T
+
+
+def predict_from_formula(formula_str, x_values):
+    if Settings.num_features == 1:
+        x_variables = [["x"]]
+    else:
+        x_variables = [["x{}".format(var_i + 1) for var_i in range(Settings.num_features)]]
+    f = lambdify(x_variables, formula_str, 'numpy')
+    if isinstance(x_values, list):
+        return [f(x_values[row_i]) for row_i in range(len(x_values))]
+    elif len(x_values.shape) == 2:
+        return [f(x_values[row_i, :]) for row_i in range(x_values.shape[0])]
+    else:
+        return [f(x_values[row_i, :, :].reshape([-1, 1])) for row_i in range(x_values.shape[0])]
+
+
+def leaves_up_from_dfs_order(orig_list, num_layers):
+    if num_layers <= 1:
+        return orig_list
+    index_list = [[i, i+1] for i in range(0, int(2**(num_layers-1)), 2)]
+
+    # print("start with: {}".format(index_list))
+    last_value=index_list[-1][-1]
+    while len(index_list) > 1:
+        new_index_list = []
+        for j in range(int(len(index_list) / 2)):
+            last_value += 1
+            new_list = [last_value]
+            new_list.extend(index_list[2*j])
+
+            last_value += 1
+            new_list.append(last_value)
+            new_list.extend(index_list[2*j+1])
+            new_index_list.append(new_list)
+        index_list = new_index_list
+
+        # print(" index list is now: {}".format(index_list))
+    final_index_list = [last_value + 1]
+    final_index_list.extend(index_list[0])
+
+    ret_val = [i for i in range(len(final_index_list))]
+    for i in range(len(final_index_list)):
+        ret_val[final_index_list[i]] = orig_list[i]
+    return ret_val # [orig_list[i] for i in final_index_list]
+
+
+def simplify_formula(formula_to_simplify, digits=None):
+    if len("{}".format(formula_to_simplify)) > 1500:
+        return "{}".format(expand(formula_to_simplify))
+    orig_form_str = sympify(formula_to_simplify)
+    if len("{}".format(orig_form_str)) > 1000:
+        return "{}".format(expand(orig_form_str))
+
+    if len("{}".format(orig_form_str)) < 700:
+        # orig_form_str = simplify(expand(orig_form_str))
+        orig_form_str = expand(orig_form_str)
+
+    rounded = orig_form_str
+    for a in sympy.preorder_traversal(orig_form_str):
+        if isinstance(a, sympy.Float):
+            if digits is not None:
+                if np.abs(a) < 10**(-1*digits):
+                    rounded = rounded.subs(a, 0)
+                else:
+                    rounded = rounded.subs(a, round(a, digits))
+            elif np.abs(a) < Settings.big_eps:
+                rounded = rounded.subs(a, 0)
+
+    return "{}".format(rounded)
+
+
+def eqn_to_str_two_list(eqn_as_list, var_y_index=9999, unary_use_both=False):
+    eqn_ops = eqn_as_list[0]
+    eqn_vars = eqn_as_list[1]
+    current_op = eqn_ops[0]
+    # print("eqn_to_str:")
+    # print(eqn_ops)
+    # print(eqn_vars)
+
+    if len(eqn_ops) == 1:
+
+        if type(eqn_vars[0]) is list:
+            left_side = "{:.3f}".format(eqn_vars[0][0])
+        elif eqn_vars[0] == var_y_index:
+            left_side = "y"
+        else:
+            left_side = "x{}".format(eqn_vars[0])
+        if type(eqn_vars[1]) is list:
+            right_side = "{:.3f}".format(eqn_vars[1][0])
+        elif eqn_vars[1] == var_y_index:
+            right_side = "y"
+        else:
+            right_side = "x{}".format(eqn_vars[1])
+
+
+    else:
+        split_point = int((len(eqn_ops) + 1) / 2)
+        left_ops = eqn_ops[1:split_point]
+        right_ops = eqn_ops[split_point:]
+
+        left_vars = eqn_vars[:split_point]
+        right_vars = eqn_vars[split_point:]
+
+        left_side = eqn_to_str_two_list([left_ops, left_vars])
+        right_side = eqn_to_str_two_list([right_ops, right_vars])
+
+    left_is_float = False
+    right_is_float = False
+    left_value = np.nan
+    right_value = np.nan
+
+    if is_float(left_side):
+        left_value = float(left_side)
+        left_is_float = True
+    if is_float(right_side):
+        right_value = float(right_side)
+        right_is_float = True
+
+    if current_op == 'id':
+        return left_side
+
+    if current_op == 'sqrt':
+        if left_is_float:
+            return "{:.3f}".format(np.sqrt(np.abs(left_value)))
+        return "sqrt({})".format(left_side)
+
+    if current_op == 'log':
+        if left_is_float:
+            return "{:.3f}".format(np.math.log(safe_abs(left_value)))
+        return "log({})".format(left_side)
+
+    if current_op == 'sin':
+        if left_is_float:
+            return "{:.3f}".format(np.sin(left_value))
+        return "sin({})".format(left_side)
+
+    if current_op == 'exp':
+        if left_is_float:
+            return "{:.3f}".format(np.exp(left_value))
+        return "exp({})".format(left_side)
+
+    if current_op == 'add':
+        if left_is_float and right_is_float:
+            return "{:.3f}".format(left_value + right_value)
+        return "({} + {})".format(left_side, right_side)
+
+    if current_op == 'mul':
+        if left_is_float and right_is_float:
+            return "{:.3f}".format(left_value * right_value)
+        return "({} * {})".format(left_side, right_side)
+
+    if current_op == 'sub':
+        if left_is_float and right_is_float:
+            return "{:.3f}".format(left_value - right_value)
+        return "({} - {})".format(left_side, right_side)
+
+    if current_op == 'div':
+        if left_is_float and right_is_float:
+            return "{:.3f}".format(safe_div(left_value, right_value))
+        return "({} / {})".format(left_side, right_side)
+
+    return None
+
+
+def eqn_to_str(eqn_as_list, var_y_index=9999, unary_use_both=False):
+    eqn_ops = eqn_as_list[0]
+    eqn_vars = eqn_as_list[1]
+    eqn_weights = eqn_as_list[2]
+    eqn_biases = eqn_as_list[3]
+    current_op = eqn_ops[0]
+    # print("eqn_to_str:")
+    # print(eqn_ops)
+    # print(eqn_vars)
+
+    if len(eqn_ops) == 1:
+
+        if eqn_vars[0] == var_y_index:
+            left_side = "y"
+        else:
+            left_side = "({} * x{} + {})".format(eqn_weights[0], eqn_vars[0], eqn_biases[0])
+        if eqn_vars[1] == var_y_index:
+            right_side = "y"
+        else:
+            right_side = "({} * x{} + {})".format(eqn_weights[1], eqn_vars[1], eqn_biases[1])
+
+
+    else:
+        split_point = int((len(eqn_ops) + 1) / 2)
+        left_ops = eqn_ops[1:split_point]
+        right_ops = eqn_ops[split_point:]
+
+        left_vars = eqn_vars[:split_point]
+        right_vars = eqn_vars[split_point:]
+
+        left_weights = eqn_weights[:split_point]
+        right_weights = eqn_weights[split_point:]
+
+        left_biases = eqn_biases[:split_point]
+        right_biases = eqn_biases[split_point:]
+
+        left_side = eqn_to_str([left_ops, left_vars, left_weights, left_biases])
+        right_side = eqn_to_str([right_ops, right_vars, right_weights, right_biases])
+
+    left_is_float = False
+    right_is_float = False
+    left_value = np.nan
+    right_value = np.nan
+
+    if is_float(left_side):
+        left_value = float(left_side)
+        left_is_float = True
+    if is_float(right_side):
+        right_value = float(right_side)
+        right_is_float = True
+
+    if current_op == 'id':
+        return left_side
+
+    if current_op == 'sqrt':
+        if left_is_float:
+            return "{:.3f}".format(np.sqrt(np.abs(left_value)))
+        return "sqrt({})".format(left_side)
+
+    if current_op == 'log':
+        if left_is_float:
+            return "{:.3f}".format(np.math.log(safe_abs(left_value)))
+        return "log({})".format(left_side)
+
+    if current_op == 'sin':
+        if left_is_float:
+            return "{:.3f}".format(np.sin(left_value))
+        return "sin({})".format(left_side)
+
+    if current_op == 'exp':
+        if left_is_float:
+            return "{:.3f}".format(np.exp(left_value))
+        return "exp({})".format(left_side)
+
+    if current_op == 'add':
+        if left_is_float and right_is_float:
+            return "{:.3f}".format(left_value + right_value)
+        return "({} + {})".format(left_side, right_side)
+
+    if current_op == 'mul':
+        if left_is_float and right_is_float:
+            return "{:.3f}".format(left_value * right_value)
+        return "({} * {})".format(left_side, right_side)
+
+    if current_op == 'sub':
+        if left_is_float and right_is_float:
+            return "{:.3f}".format(left_value - right_value)
+        return "({} - {})".format(left_side, right_side)
+
+    if current_op == 'div':
+        if left_is_float and right_is_float:
+            return "{:.3f}".format(safe_div(left_value, right_value))
+        return "({} / {})".format(left_side, right_side)
+
+    return None
+
+def simple_eqn_to_str(eqn_as_list, var_y_index=9999):
+    return simplify_formula(eqn_to_str(eqn_as_list, var_y_index=var_y_index))
+
+
+def get_samples(n_train, n_batch, train_x, train_y):
+    both_samples = np.random.choice(n_train, size=2*n_batch, replace=False)
+
+    sample = both_samples[:n_batch]
+    valid_sample = both_samples[n_batch:]
+
+    mini_batch_train_data_x = train_x[sample][:][:]
+    mini_batch_train_data_y = train_y[sample][:][:]
+
+    mini_valid_sample_x = train_x[valid_sample][:][:]
+    mini_valid_sample_y = train_y[valid_sample][:][:]
+    return mini_batch_train_data_x, mini_batch_train_data_y, mini_valid_sample_x, mini_valid_sample_y
+
+
+###############################################
+#
+#   Plotting functions
+#
+###############################################
+
+
+def plot_1d_curve(train_x, train_y_true, train_y_pred, test_x, test_y_true, test_y_pred,
+                  title="", file_suffix="", show_ground_truth=True):
+    plt.figure()
+    plt.title(title)
+    if show_ground_truth:
+        plt.scatter(train_x, train_y_true, color='gray', alpha=0.5, marker='.', label='Ground truth')
+        if test_x is not None:
+            plt.scatter(test_x, test_y_true, color='gray', alpha=0.5, marker='.')
+
+    if test_x is not None:
+        plt.scatter(test_x, test_y_pred, color='red', alpha=0.7, marker='.', label='Model (test set)')
+    plt.scatter(train_x, train_y_pred, color='blue', alpha=0.7, marker='.', label='Model (train set)')
+
+    for xc in Settings.train_scope:
+        plt.axvline(x=xc, color='k', linestyle='dashed', linewidth=2)
+
+    plt.xlabel("x")
+    plt.ylabel("y")
+
+    plt.legend()
+    plt.savefig("images/true_vs_pred_curve{}.png".format(file_suffix))
+    plt.close()
+
+
+def plot_2d_curve(x_1, x_2, y, g,
+                  title=""):
+
+    fig = plt.figure(figsize=(11, 5))
+    # ax = fig.gca(projection='3d')
+    ax = fig.add_subplot(1, 2, 1, projection='3d')
+    plt.title("Learned function")
+
+    surf = ax.plot_surface(x_1, x_2, y, cmap=cm.coolwarm,
+                           linewidth=0, antialiased=False, label="Ground truth")
+
+
+    plt.xlabel("x")
+    plt.ylabel("y")
+    fig.colorbar(surf, shrink=0.5, aspect=5)
+
+    ax = fig.add_subplot(1, 2, 2, projection='3d')
+    plt.title("Residual (g)")
+
+    surf = ax.plot_surface(x_1, x_2, g, cmap=cm.coolwarm,
+                           linewidth=0, antialiased=False, label="Ground truth")
+
+    plt.xlabel("x")
+    plt.ylabel("y")
+    fig.colorbar(surf, shrink=0.5, aspect=5)
+
+    # plt.show()
+
+    # plt.legend()
+    plt.savefig("images/pred_g_2d.png")
+    plt.close()
+
+
+def plot_predicted_vs_actual(pred_y_train, true_y_train,
+                             pred_y_test=None, true_y_test=None,
+                             model_name="Model", set_name="", show=False):
+    plt.figure()
+    if set_name != "":
+        set_name = "({})".format(set_name)
+    plt.title('{}: Predicted vs. Actual {}'.format(model_name, set_name))
+
+    plt.scatter(true_y_train, true_y_train, color='gray', alpha=0.5, marker='.', label='Ground truth')
+    if pred_y_test is not None:
+        plt.scatter(true_y_test, true_y_test, color='gray', alpha=0.5, marker='.')
+
+    if pred_y_test is not None:
+        plt.scatter(true_y_test, pred_y_test, color="red", alpha=0.6, marker='.', label="{}: Test".format(model_name))
+
+    plt.scatter(true_y_train, pred_y_train, color="blue", alpha=0.7, marker='.', label="{}: Train".format(model_name))
+
+    plt.ylabel("Observed")
+    plt.xlabel("Expected")
+    plt.legend()
+
+    plt.savefig("images/single_predicted_vs_actual.png")
+    if show:
+        plt.show()
+    plt.close()
+
+
+# acc_logs: list of accuracy logs to be plotted.
+def plot_accuracy_over_time(iters_log, acc_logs, error_types):
+    if len(iters_log) < 2:
+        return
+    for i in range(len(acc_logs)):
+        plt.plot(iters_log[1:], acc_logs[i][1:], label=error_types[i])
+    plt.xlabel("Iteration")
+    plt.ylabel("Error Scores")
+    plt.yscale('log')
+    plt.legend()
+    plt.savefig("images/accuracy_log.png")
+    plt.close()
+
+
+def plot_hist_of_errors(lists_of_error_scores, all_models, num_trials):
+    plt.figure()
+    plt.hist([np.log(errors_i) for errors_i in lists_of_error_scores],
+             label=[model.short_name for model in all_models])
+    plt.legend()
+    plt.title("Comparing errors of all methods over {} equations".format(num_trials))
+    plt.xlabel("Log of error")
+    plt.ylabel("Frequency")
+    plt.savefig("images/hist_of_errors.png")
+    plt.close()

FeynmanEqns.txt

@@ -0,0 +1,19 @@
+# eqn #; # levels; # vars; eqn str; true ops list
+ballot; 1; 2; (x1-x2)/(x1+x2); ["div"]
+sum; 1; 2; x1 + x2; ["id"]
+range; 2; 2; x1**2 * sin(2*x2) / 9.81; ["mul", "sin", "mul"]
+mm
+I.11.19; 1; 6; x1*x2 + x3*x4 + x5*x6; ["mul"]
+mmm
+kin1; 2; 3; x1*x2 + 0.5 * x3 * x2**2; ["mul", "mul", "mul"]
+mm
+mmm
+collision1; 2; 4; (2*x1*x3 - (x1-x2)*x4) / (x1 + x2); ["mul", "id", "div"]
+mmm
+I.12.1; 2; 3; x1 * x2 / (4 * pi * x3**2 / 137); ["mul", "mul", "div"]
+I.10.7a; 3; 2; x1/sqrt(1-x2**2); ["id", "id", "mul", "id", "id", "sqrt", "div"]
+mmm
+I.8.14; 2; 4; sqrt((x2-x1)**2 +(x4-x3)**2); ["mul", "mul", "sqrt"]
+I.6.20a; 2; 2; exp(-x1**2)/sqrt(2*pi); ["mul", "id", "exp"]
+I.29.16; 3; 3; sqrt(x1**2 + x2**2 - 2*x1*x2*cos(x3)); ["mul", "sin", "mul", "id", "mul", "id", "sqrt"]
+AreaTriangle; 2; 3; x1*x2*sin(x3)/2; ["mul", "sin", "mul"]

LearnRules.py

@@ -0,0 +1,146 @@
+import time
+
+import numpy as np
+
+import LearnTriangles
+import Settings as settings
+from SymbolicFunctionLearner import SFL
+
+num_triangles = 1000
+num_fake = 3000
+max_domain = 5
+num_trials = 500
+num_smp_features = 2
+settings.show_output = True
+settings.keep_logs = True
+
+settings.mode = "lr"
+# settings.initialize_ops =  ["mul", "mul", "mul", "mul", "id", "id", "id"]
+# settings.initialize_ops = ["mul", "mul", "id"]
+
+
+"""" These are the things to change"""
+var_names = LearnTriangles.get_xy_var_names()
+
+# real_data = LearnTriangles.get_right_triangle_data(num_triangles, max_domain)
+# real_data = LearnTriangles.get_angle_data(num_triangles)
+real_data = LearnTriangles.get_xy_data(num_triangles, max_domain)
+real_y = [1 for v in real_data]
+
+fake_data = LearnTriangles.get_fake_xy_data(num_triangles, max_domain)
+# fake_data = LearnTriangles.get_fake_angles(num_triangles)
+# fake_data = LearnTriangles.get_triangle_data(num_triangles, max_domain)
+fake_y = [0 for _ in fake_data]
+
+# real_test_data = LearnTriangles.get_right_triangle_data(num_triangles, max_domain * 2)
+# real_test_data = LearnTriangles.get_angle_data(num_triangles)
+real_test_data = LearnTriangles.get_xy_data(num_triangles, max_domain)
+real_test_y = [1 for v in real_test_data]
+
+# fake_test_data = LearnTriangles.get_triangle_data(num_triangles, max_domain * 2)
+# fake_test_data = LearnTriangles.get_fake_angles(num_triangles)
+fake_test_data = LearnTriangles.get_fake_xy_data(num_triangles, max_domain)
+fake_test_y = [0 for _ in fake_test_data]
+
+""" Don't change after this """
+
+print("real data: ")
+for r_d in real_data[:5]:
+    print(r_d)
+print("real y:")
+for r_d in real_y[:5]:
+    print(r_d)
+print("fake data:")
+for r_d in fake_data[:5]:
+    print(r_d)
+print("fake y:")
+for r_d in fake_y[:5]:
+    print(r_d)
+
+full_data = real_data.copy()
+full_data.extend(fake_data)
+full_labels = real_y.copy()
+full_labels.extend(fake_y)
+
+full_test_data = real_test_data.copy()
+full_test_data.extend(fake_test_data)
+full_test_labels = real_test_y.copy()
+full_test_labels.extend(fake_test_y)
+
+# print("full data:\n{}".format(full_data))
+# print("full y:\n{}".format(full_labels))
+#
+# for datum in real_data:
+#     print(datum[0]*datum[4] - datum[1]*datum[3])
+
+
+our_results = []
+
+settings.true_eqn = "0*x1"
+settings.num_features = num_smp_features
+
+model = SFL()
+
+for trial_round in range(num_trials):
+    sampled_features = np.random.choice(range(len(real_data[0])), num_smp_features, replace=True)
+
+    sampled_features = [0, 1]
+
+    data = [[row[smp_i] for smp_i in sampled_features] for row in full_data]
+    test_data = [[row[smp_i] for smp_i in sampled_features] for row in full_test_data]
+
+    smp_var_names = [var_names[smp_i] for smp_i in sampled_features]
+    print("Trial round {} of {}.".format(trial_round + 1, num_trials))
+    print("  Using variables {}.".format(smp_var_names))
+
+    settings.fixed_x = []
+    settings.fixed_y = []
+    for line in data:
+        settings.fixed_x.append(line)
+    settings.fixed_y = full_labels
+
+    # print("fixed_x: {}, {}".format(len(settings.fixed_x), len(settings.fixed_x[0])))
+    # print("fixed_y: {}".format(len(settings.fixed_y)))
+
+    model.reset(var_names=smp_var_names)
+
+    # train_X = DataUtils.generate_data(settings.train_N, n_vars=model.n_input_variables,
+    #                                       avoid_zero=settings.avoid_zero)
+    # valid_X = DataUtils.generate_data(settings.train_N, n_vars=model.n_input_variables,
+    #                                   avoid_zero=settings.avoid_zero)
+    # test_X = DataUtils.generate_data(settings.test_N, n_vars=model.n_input_variables,
+    #                                  min_x=settings.test_scope[0],
+    #                                  max_x=settings.test_scope[1])
+    train_X = np.array(data)
+    train_Y = full_labels
+
+    test_X = np.array(test_data)
+    test_Y = full_test_labels
+
+    train_X = train_X.reshape([-1, settings.num_dims_per_feature, settings.num_features])
+    test_X = test_X.reshape([-1, settings.num_dims_per_feature, settings.num_features])
+
+    start_time = time.time()
+    best_model, best_iter, best_err = model.repeat_train(train_X, train_Y,
+                                                         settings.num_train_repeat_processes,
+                                                         test_x=test_X, test_y=test_Y)
+    running_time = time.time() - start_time
+
+    print("best_model: {}".format(best_model))
+    print("----------------------")
+    print("Finished this experiment. Took {:.2f} minutes.\n".format(running_time / 60))
+
+    our_results.append([best_err, best_model, smp_var_names])
+    our_results = sorted(our_results, key=lambda entry: entry[0])
+
+    output_file = open("images/triangle_output.txt", "w")
+    for entry in our_results:
+        output_file.write("{}\n{}\n{}\n\n".format(entry[0], entry[2], entry[1]))
+    output_file.close()
+
+    print("Final solution found at attempt {}:".format(best_iter))
+    print("y = {}".format(best_model))
+    print("Test error:  {}".format(best_err))
+    if best_err < 0.02:
+        print("Attained error less than 0.02 - great!")
+    print()

LearnTriangles.py

@@ -0,0 +1,252 @@
+import numpy as np
+
+
+def get_simple_data(num_points, max_domain):
+    full_data = []
+    for i in range(num_points):
+        new_datum = [np.random.uniform(0, max_domain),
+                     np.random.uniform(0, max_domain),
+                     np.random.uniform(0, max_domain)]
+        full_data.append(new_datum)
+    return full_data
+
+
+def get_simple_var_names():
+    return ["a", "b", "c"]
+
+
+def get_xy_var_names():
+    return ["x", "y"]
+
+
+def get_angle_data(num_data):
+    full_data = []
+    for i in range(num_data):
+        theta = np.random.uniform(0, 2 * np.pi)
+        while np.abs(theta - np.pi / 2) < 1e-3 or np.abs(theta - 3 * np.pi / 2) < 1e-3:
+            theta = np.random.uniform(0, 2 * np.pi)
+        new_datum = [np.sin(theta), np.cos(theta), np.tan(theta)]
+        full_data.append(new_datum)
+    return full_data
+
+
+def get_fake_angles(num_data):
+    full_data = []
+    for i in range(num_data):
+        [s, c] = np.random.uniform(-1, 1, 2)
+        t = np.random.uniform(-5, 5)
+        new_datum = [s, c, t]
+        full_data.append(new_datum)
+    return full_data
+
+
+def get_xy_data(num_data, max_domain):
+    full_data = []
+    for i in range(num_data):
+        # t = np.random.uniform(0, 2 * np.pi)
+        # r = 2 + 3 * np.cos(t)
+        # # r = 5 * np.cos(2*t)/np.cos(t)
+        #
+        # x = r * np.cos(t)
+        # y = r * np.sin(t)
+
+        y = np.random.uniform(-3, 2)
+        sign = np.random.choice([-1, 1])
+        x = sign * np.sqrt(np.abs(2 * np.sin(y) + 5 - y ** 3))
+        full_data.append([x, y])
+    return full_data
+
+
+def get_fake_xy_data(num_data, max_domain):
+    full_data = []
+    for i in range(num_data):
+        [x, y] = np.random.uniform(0, max_domain, 2)
+        full_data.append([x, y])
+    return full_data
+
+
+
+def get_triangle_data(num_triangles, max_domain):
+    full_data = []
+    for i in range(num_triangles):
+        [x1, y1, x2, y2, x3, y3] = np.random.uniform(0, max_domain, 6)
+
+        a = np.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2)
+        b = np.sqrt((x3 - x2) ** 2 + (y3 - y2) ** 2)
+        c = np.sqrt((x1 - x3) ** 2 + (y1 - y3) ** 2)
+
+        va = np.arccos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c))
+        vb = np.arccos((c ** 2 + a ** 2 - b ** 2) / (2 * a * c))
+        vc = np.arccos((a ** 2 + b ** 2 - c ** 2) / (2 * b * a))
+
+        sa, sb, sc = np.sin([va, vb, vc])
+        ca, cb, cc = np.cos([va, vb, vc])
+        ta, tb, tc = np.tan([va, vb, vc])
+
+        new_datum = []
+        # new_datum.extend([x1, x2, x3])
+        # new_datum.extend([y1, y2, y3])
+        new_datum.extend([a, b, c])
+        # new_datum.extend([va, vb, vc])
+        # new_datum.extend([sa, sb, sc])
+        # new_datum.extend([ca, cb, cc])
+        # new_datum.extend([ta, tb, tc])
+
+        full_data.append(new_datum)
+    return full_data
+
+
+def get_right_triangle_data(num_triangles, max_domain):
+    full_data = []
+    for i in range(num_triangles):
+        [s1, s2] = np.random.uniform(0, max_domain, 2)
+        c = max(s1, s2)
+        a = min(s1, s2)
+        b = np.sqrt(c * c - a * a)
+
+        va = np.arccos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c))
+        vb = np.arccos((c ** 2 + a ** 2 - b ** 2) / (2 * a * c))
+        vc = np.arccos((a ** 2 + b ** 2 - c ** 2) / (2 * b * a))
+
+        sa, sb, sc = np.sin([va, vb, vc])
+        ca, cb, cc = np.cos([va, vb, vc])
+        ta, tb, tc = np.tan([va, vb, vc])
+
+        new_datum = []
+        new_datum.extend([a, b, c])
+        # new_datum.extend([sa, sb, sc])
+        # new_datum.extend([ca, cb, cc])
+        # new_datum.extend([ta, tb, tc])
+
+        full_data.append(new_datum)
+
+    return full_data
+
+
+def get_fake_triangle_data(num_triangles, max_domain):
+    full_data = []
+    for i in range(num_triangles):
+        [a, b, c] = np.random.uniform(0, max_domain, 3)
+        [sa, sb, sc] = np.random.uniform(-1, 1, 3)
+        [ca, cb, cc] = np.random.uniform(-1, 1, 3)
+        [ta, tb, tc] = np.random.uniform(-5, 5, 3)
+
+        new_datum = []
+        new_datum.extend([a, b, c])
+        # new_datum.extend([va, vb, vc])
+        new_datum.extend([sa, sb, sc])
+        new_datum.extend([ca, cb, cc])
+        new_datum.extend([ta, tb, tc])
+
+        # new_datum.extend([va, vb, vc])
+        # new_datum.extend([a, 2*a])
+        # new_datum.extend([ta, tb, tc])
+
+        full_data.append(new_datum)
+    return full_data
+
+
+def get_non_triangle_data(num_triangles, max_domain):
+    full_data = []
+    for i in range(num_triangles):
+        [x1, y1, x2, y2, x3, y3, x4, y4] = np.random.uniform(0, max_domain, 8)
+
+        a = np.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2)
+        b = np.sqrt((x3 - x2) ** 2 + (y3 - y2) ** 2)
+        c = np.sqrt((x4 - x3) ** 2 + (y4 - y3) ** 2)
+        d = np.sqrt((x1 - x4) ** 2 + (y1 - y4) ** 2)
+        e24 = np.sqrt((x4 - x2) ** 2 + (y4 - y2) ** 2)
+        e13 = np.sqrt((x3 - x1) ** 2 + (y3 - y1) ** 2)
+
+        va = np.arccos((b ** 2 + c ** 2 - e24 ** 2) / (2 * b * c))
+        vb = np.arccos((d ** 2 + c ** 2 - e13 ** 2) / (2 * d * c))
+        vc = np.arccos((d ** 2 + a ** 2 - e24 ** 2) / (2 * d * a))
+
+        sa, sb, sc = np.sin([va, vb, vc])
+        ca, cb, cc = np.cos([va, vb, vc])
+        ta, tb, tc = np.tan([va, vb, vc])
+
+        new_datum = []
+        new_datum.extend([a, b, c])
+        # new_datum.extend([va, vb, vc])
+        new_datum.extend([sa, sb, sc])
+        new_datum.extend([ca, cb, cc])
+        # new_datum.extend([ta, tb, tc])
+        full_data.append(new_datum)
+    return full_data
+
+def get_var_names():
+    # var_names = ["a", "b", "c"]
+    var_names = []
+    var_names.extend(["sin(x)", "cos(x)", "tan(x)"])
+    # var_names.extend(["A", "B", "C"])
+    var_names.extend(["sin(A)", "sin(B)", "sin(C)"])
+    var_names.extend(["cos(A)", "cos(B)", "cos(C)"])
+    var_names.extend(["tan(A)", "tan(B)", "tan(C)"])
+    return var_names
+#
+# our_results = []
+#
+# settings.true_eqn = "0*x1"
+# settings.num_features = num_smp_features
+# settings.show_output = False
+# settings.keep_logs = False
+#
+# model = SFL()
+#
+# for trial_round in range(num_trials):
+#     sampled_features = np.random.choice(range(len(full_data[0])), num_smp_features, replace=True)
+#
+#     # sampled_features = [3,6]
+#     data = [[row[smp_i] for smp_i in sampled_features] for row in full_data]
+#     smp_var_names = [var_names[smp_i] for smp_i in sampled_features]
+#     print("Trial round {} of {}.".format(trial_round + 1, num_trials))
+#     print("  Using variables {}.".format(smp_var_names))
+#
+#     settings.fixed_x = []
+#     settings.fixed_y = []
+#     for line in data:
+#         settings.fixed_x.append(line)
+#         settings.fixed_y.append(0)
+#
+#     model.reset(var_names=smp_var_names)
+#
+#     # train_X = DataUtils.generate_data(settings.train_N, n_vars=model.n_input_variables,
+#     #                                       avoid_zero=settings.avoid_zero)
+#     # valid_X = DataUtils.generate_data(settings.train_N, n_vars=model.n_input_variables,
+#     #                                   avoid_zero=settings.avoid_zero)
+#     # test_X = DataUtils.generate_data(settings.test_N, n_vars=model.n_input_variables,
+#     #                                  min_x=settings.test_scope[0],
+#     #                                  max_x=settings.test_scope[1])
+#     train_X = np.array(data)
+#     valid_X = np.array(data)
+#     test_X = np.array(data)
+#
+#     train_X = train_X.reshape([-1, settings.num_dims_per_feature, settings.num_features])
+#     valid_X = valid_X.reshape([-1, settings.num_dims_per_feature, settings.num_features])
+#     test_X = test_X.reshape([-1, settings.num_dims_per_feature, settings.num_features])
+#
+#     start_time = time.time()
+#     best_model, best_iter, best_err = model.repeat_train(train_X,
+#                                                          settings.num_train_repeat_processes,
+#                                                          test_x=test_X)
+#     running_time = time.time() - start_time
+#
+#     print("best_model: {}".format(best_model))
+#     print("----------------------")
+#     print("Finished this experiment. Took {:.2f} minutes.\n".format(running_time / 60))
+#
+#     our_results.append([best_err, best_model, smp_var_names])
+#     our_results = sorted(our_results, key=lambda entry: entry[0])
+#
+#     output_file = open("images/triangle_output.txt", "w")
+#     for entry in our_results:
+#         output_file.write("{}\n{}\n{}\n\n".format(entry[0], entry[2], entry[1]))
+#     output_file.close()
+#
+#     print("Final solution found at attempt {}:".format(best_iter))
+#     print("y = {}".format(best_model))
+#     print("Test error:  {}".format(best_err))
+#     if best_err < 0.02:
+#         print("Attained error less than 0.02 - great!")
+#     print()

MLP_Model.py

@@ -0,0 +1,239 @@
+import time
+
+import numpy as np
+from sklearn.neural_network import MLPRegressor
+from sklearn.utils.validation import column_or_1d
+
+import Settings as settings
+from DataUtils import make_y_multi_safe
+
+hidden_layer_sizes = (5, 5)
+max_iter_mlp = 500000
+
+
+class MLP_Model:
+    def __init__(self):
+        self.name = "MLP Model"
+        self.short_name = "MLP"
+        self.hidden_layer_sizes = hidden_layer_sizes
+        self.solver = 'adam'
+        self.max_iter = max_iter_mlp
+        self.warm_start = False
+        self.verbose = False
+        self.est_mlp = MLPRegressor(hidden_layer_sizes=self.hidden_layer_sizes,
+                                    solver=self.solver,
+                                    activation='relu',
+                                    max_iter=self.max_iter,
+                                    verbose=self.verbose,
+                                    tol=1e-7,
+                                    warm_start=self.warm_start,
+                                    n_iter_no_change=100)
+
+    def get_formula_string(self):
+        return "(neural black box)"
+        # current_inputs = ["X{}".format(i + 1) for i in range(settings.num_features)]
+        # # print(current_inputs)
+        # matrices = self.est_mlp.coefs_
+        # vectors = self.est_mlp.intercepts_
+        # for i in range(len(matrices)):
+        #     current_outputs = []
+        #
+        #     for j in range(matrices[i].shape[1]):
+        #         current_term = [vectors[i][j]]
+        #         for k in range(matrices[i].shape[0]):
+        #             sys.stdout.flush()
+        #             current_term.append(["*", matrices[i][k][j], current_inputs[k]])
+        #
+        #         current_output = current_term[-1]
+        #         for k in range(len(current_term), 1, -1):
+        #             current_output = ["+", current_term[k - 2], current_output]
+        #         current_outputs.append(current_output)
+        #     current_inputs = [["max", 0, old_out] for old_out in current_outputs]
+        #
+        # # [-1] since we don't do relu activation on the last layer.
+        # return current_inputs[0][-1]
+
+    def get_formula(self):
+        return "(neural black box)"
+        # return self.get_formula_string()
+
+    def train(self, X, Y):
+        X = np.reshape(X, [X.shape[0], -1])
+        Y = np.reshape(Y, [-1, 1])
+        Y = column_or_1d(Y)
+        self.est_mlp.fit(X, Y)
+        return None
+
+    def predict(self, X):
+        return self.est_mlp.predict(X)
+
+    # Mean square error
+    def test(self, X, Y):
+        X = np.reshape(X, [X.shape[0], -1])
+        y_hat = np.reshape(self.est_mlp.predict(X), [1, -1])[0]
+        y_gold = np.reshape(Y, [1, -1])[0]
+        our_sum = 0
+        for i in range(len(y_gold)):
+            our_sum += (y_hat[i] - y_gold[i]) ** 2
+
+        return our_sum / len(y_gold)
+
+    def reset(self):
+        self.est_mlp = MLPRegressor(hidden_layer_sizes=self.hidden_layer_sizes,
+                                    solver=self.solver,
+                                    activation='relu',
+                                    max_iter=self.max_iter,
+                                    verbose=self.verbose,
+                                    tol=1e-7,
+                                    warm_start=self.warm_start,
+                                    n_iter_no_change=100)
+
+    def soft_reset(self):
+        self.est_mlp = MLPRegressor(hidden_layer_sizes=self.hidden_layer_sizes,
+                                    solver=self.solver,
+                                    activation='relu',
+                                    max_iter=self.max_iter,
+                                    verbose=self.verbose,
+                                    tol=1e-7,
+                                    warm_start=self.warm_start,
+                                    n_iter_no_change=100)
+
+    def get_simple_formula(self, digits=None):
+        full_formula = self.get_formula_string()
+        return full_formula
+        # return DataUtils.simplify_formula(full_formula, digits=digits)
+
+    def real_repeat_train(self, x, y=None,
+                     num_repeats=settings.num_train_repeat_processes,
+                     test_x=None, test_y=None,
+                     verbose=True):
+
+        # we still reduce train set size if only 1 repeat
+        train_set_size = int(len(x) * settings.quick_train_fraction + 0.1)
+
+        x = np.array(x)
+        if y is not None:
+            y = np.array(y)
+
+        sample = np.random.choice(range(x.shape[0]), size=train_set_size, replace=False)
+        train_x = x[sample][:]
+        if y is not None:
+            train_y = y[sample]
+        else:
+            train_y = None
+
+        out_sample = [aaa for aaa in range(x.shape[0]) if aaa not in sample]
+        valid_x = x[out_sample][:]
+        if y is not None:
+            valid_y = y[out_sample]
+            # valid_y = self.make_y_multi_safe(valid_y)
+        else:
+            valid_y = None
+
+        best_formula = ""
+        best_iter = 0
+        best_validation = 999999
+        best_err = 999999
+        old_time = time.time()
+
+        if verbose:
+            print("Beginning {} repeat sessions of {} iterations each.".format(num_repeats,
+                                                                               settings.num_train_steps_in_repeat_mode))
+            print()
+            start_time = time.time()
+            old_time = start_time
+
+        for train_iter in range(1, 1 + num_repeats):
+            if verbose:
+                print("Repeated train session {} of {}.".format(train_iter, num_repeats))
+
+            self.soft_reset()
+            self.train(train_x, train_y)
+
+            valid_err = self.test(valid_x, valid_y)
+
+            current_time = time.time()
+            if verbose:
+                # print(self.get_simple_formula())
+                print("Attained validation error: {:.5f}".format(valid_err))
+
+            if valid_err < best_validation:
+                best_validation = valid_err
+                best_formula = self.get_simple_formula()
+                best_iter = train_iter
+                if test_x is not None:
+                    safe_test_y = make_y_multi_safe(test_y)
+                    best_err = self.test(test_x, safe_test_y)
+                else:
+                    best_err = valid_err
+                if verbose:
+                    print(">>> New best model!")
+                    print(best_formula)
+
+            if verbose:
+                iters_per_minute = 60.0 / (current_time - old_time)
+                print("Took {:.2f} minutes.".format((current_time - old_time) / 60))
+                print("Est. {:.2f} minutes remaining.".format((num_repeats - train_iter) / iters_per_minute))
+                print()
+                old_time = current_time
+
+        if verbose:
+            print("Total time for repeat process: {:.2f} minutes.".format((time.time() - start_time) / 60))
+
+        return best_formula, best_iter, best_err
+
+    # Does not repeat train. sorry.
+    def repeat_train(self, x, y=None,
+                     num_repeats=settings.num_train_repeat_processes,
+                     test_x=None, test_y=None,
+                     verbose=True):
+
+        # we still reduce train set size if only 1 repeat
+        train_set_size = int(len(x) * settings.quick_train_fraction + 0.1)
+
+        x = np.array(x)
+        if y is not None:
+            y = np.array(y)
+
+        sample = np.random.choice(range(x.shape[0]), size=train_set_size, replace=False)
+        train_x = x[sample][:]
+        if y is not None:
+            train_y = y[sample]
+        else:
+            train_y = None
+
+        out_sample = [aaa for aaa in range(x.shape[0]) if aaa not in sample]
+        valid_x = x[out_sample][:]
+        if y is not None:
+            valid_y = y[out_sample]
+            # valid_y = self.make_y_multi_safe(valid_y)
+        else:
+            valid_y = None
+
+        if verbose:
+            start_time = time.time()
+            old_time = start_time
+
+        self.soft_reset()
+        self.train(train_x, train_y)
+
+        current_time = time.time()
+
+        best_formula = self.get_simple_formula()
+        if test_x is not None:
+            safe_test_y = make_y_multi_safe(test_y)
+            best_err = self.test(test_x, safe_test_y)
+        else:
+            best_err = self.test(valid_x, valid_y)
+        if verbose:
+            print(">>> New best model!")
+            print(best_formula)
+
+        if verbose:
+            print("Took {:.2f} minutes.".format((current_time - old_time) / 60))
+            print()
+
+        if verbose:
+            print("Total time for repeat process: {:.2f} minutes.".format((time.time() - start_time) / 60))
+
+        return best_formula, 0, best_err

RegressorTest.py

@@ -0,0 +1,115 @@
+""""""""""""""""""""""""""""""""" 
+This file is for running.
+Do not modify this file.
+
+For running: DiffEqnSolver.py
+For modifying: settings.py
+"""""""""""""""""""""""""""""""""
+
+import os
+import time
+
+import numpy as np
+from pysr import pysr, best
+
+import DataUtils
+import Settings as settings
+from SymbolicFunctionLearner import SFL
+
+# Dataset
+X = 2 * np.random.randn(100, 5)
+y = 2 * np.cos(X[:, 3]) + X[:, 0] ** 2 - 2
+
+# Learn equations
+equations = pysr(X, y, niterations=5,
+                 binary_operators=["plus", "mult"],
+                 unary_operators=[
+                     "cos", "exp", "sin"])
+                     # Pre-defined library of operators (see https://pysr.readthedocs.io/en/latest/docs/operators/)
+# "inv(x) = 1/x"])  # Define your own operator! (Julia syntax)
+
+...  # (you can use ctl-c to exit early)
+
+print(best(equations))
+
+
+
+settings.mode = "sr"
+
+if not os.path.exists('images'):
+    os.makedirs('images')
+
+input_file = open("FeynmanEqns.txt", "r")
+input_lines = input_file.readlines()
+input_file.close()
+for line in input_lines[1:]:
+    line_parts = line.strip().split(";")
+    eqn_name = line_parts[0].strip()
+    settings.n_tree_layers = int(line_parts[1].strip())
+    settings.num_features = int(line_parts[2].strip())
+    eqn_str = line_parts[3].strip()
+    print("True equation: {}".format(eqn_str))
+    settings.true_eqn = eqn_str
+    settings.initialize_ops = eval(line_parts[4].strip())
+    print(settings.initialize_ops)
+
+    # Set up data
+    fixed_x = DataUtils.generate_data(settings.train_N, n_vars=settings.num_features)
+    print(fixed_x.shape)
+    fixed_y = DataUtils.true_function(fixed_x)
+    print(len(fixed_y))
+
+    settings.fixed_x = []
+    settings.fixed_y = []
+    for i in range(settings.train_N):
+        settings.fixed_x.append(fixed_x[i, :].tolist()[0])
+        settings.fixed_y.append(fixed_y[i])
+
+    current_model = SFL()
+
+    print('\nBeginning experiment: {}'.format(current_model.name))
+
+    print("{} tree layers.".format(settings.n_tree_layers))
+    print("{} features of {} component(s) each.".format(settings.num_features, settings.num_dims_per_feature))
+    print("{} components in output.".format(settings.n_dims_in_output))
+    print("{} operators: {}.".format(len(current_model.function_set),
+                                     current_model.function_set))
+
+    train_errors = []
+    valid_errors = []
+    test_errors = []
+    true_eqns = []
+
+    # train_X = DataUtils.generate_data(settings.train_N, n_vars=current_model.n_input_variables,
+    #                                   avoid_zero=settings.avoid_zero)
+    # valid_X = DataUtils.generate_data(settings.train_N, n_vars=current_model.n_input_variables,
+    #                                   avoid_zero=settings.avoid_zero)
+    # test_X = DataUtils.generate_data(settings.test_N, n_vars=current_model.n_input_variables,
+    #                                  min_x=settings.test_scope[0],
+    #                                  max_x=settings.test_scope[1])
+    train_X = fixed_x
+    test_X = fixed_x
+    train_Y = fixed_y
+    test_Y = fixed_y
+
+    print("\n========================")
+    print("Starting Solver.")
+    print("==========================\n")
+
+    # Train the model from scratch several times, keeping the best one.
+    start_time = time.time()
+    best_model, best_iter, best_err = current_model.repeat_train(train_X, train_Y,
+                                                                 settings.num_train_repeat_processes,
+                                                                 test_x=test_X, test_y=test_Y)
+    running_time = time.time() - start_time
+
+    print("best_model: {}".format(best_model))
+    print("----------------------")
+    print("Finished regression. Took {:.2f} minutes.\n".format(running_time / 60))
+
+    print("Final solution found at attempt {}:".format(best_iter))
+    print("y = {}".format(best_model))
+    print("Test error:  {}".format(best_err))
+    if best_err < 0.02:
+        print("Attained error less than 0.02 - great!")
+    print()

Settings.py

@@ -0,0 +1,277 @@
+""""""""""""""""""""""""""""""""" 
+This file is for modifying.
+Do not run this file.
+
+For running: RegressorTest.py
+For modifying: Settings.py
+"""""""""""""""""""""""""""""""""
+import numpy as np
+import tensorflow as tf
+
+"""""""""""""""""""""""""""
+Settings that can change
+"""""""""""""""""""""""""""
+# Determines complexity of tree
+n_tree_layers = 3
+
+# Allowable operators for tree nodes
+# function_set = ['id', 'mul', 'sqrt', 'sin', 'div', 'log']
+function_set = ["id", "mul", "sin", "sqrt"]
+
+num_features = 1
+num_dims_per_feature = 1
+n_dims_in_output = 1
+
+train_scope = [0, 5]
+test_scope = [y * 2 for y in train_scope]
+
+num_train_repeat_processes = 5
+num_train_steps_in_repeat_mode = 8000
+
+"""""""""""""""""""""""""""
+Display and log settings
+"""""""""""""""""""""""""""
+
+show_output = False
+keep_logs = False
+output_freq = 49000
+plot_frequency = 500000
+save_all_formulas = False
+
+max_formula_output_length = 400
+
+"""""""""""""""""""""""""""
+Tree settings
+"""""""""""""""""""""""""""
+use_both_for_unary = True
+non_const = False
+use_leaf_sm = False
+
+
+"""""""""""""""""""""""""""
+Domain parameters
+"""""""""""""""""""""""""""
+
+fpe_example = 0
+
+max_x = np.pi
+
+train_scope2 = [0, 5]
+test_scope2 = test_scope.copy()  # [0, 5]
+
+avoid_zero = False
+# if fpe_example == 4:
+#     avoid_zero = True
+
+
+"""""""""""""""""""""""""""
+Define the ODE here
+"""""""""""""""""""""""""""
+# options: mode = "sr", "de", "lr"
+mode = "de"
+
+# This is the "g" function that defines the ODE problem.
+def implicit_function(full_x, y, y_p, y_pp):
+    # Implicit function is 0 if we are doing symbolic regression
+    if mode == "sr":
+        return y * 0.0
+
+    y_p1 = y_p[0]
+    y_p2 = y_p[1]
+    y_p3 = y_p[2]
+
+    y_pp1 = y_pp[0]
+    y_pp2 = y_pp[1]
+    y_pp12 = y_pp[2]
+
+    x = tf.reshape(full_x[:, 0, 0], [-1, 1, 1])
+    t = tf.reshape(full_x[:, 0, -1], [-1, 1, 1])
+    if num_features > 1:
+        w = tf.reshape(full_x[:, 0, 1], [-1, 1, 1])
+
+    ret_val = None
+
+    """ Lane-Emden Equation """
+    # emden_m = 0
+    # ret_val = y_pp1 + 2.0 * tf.math.divide_no_nan(y_p1, x)
+    # ret_val += y ** emden_m
+
+    """ Bell curve integral """
+    # ret_val = tf.math.exp(-1.0 * tf.square(x)) - y_p1
+
+    """ One dimensional wave equation """
+    # c = 1.0
+    # ret_val = y_pp2 - c**2 * y_pp1
+
+    """ One dimensional heat equation """
+    # c = 1.0
+    # ret_val = y_p2 - c**2 * y_pp1 - tf.math.cos(x)
+
+    """ Inhomogeneous wave equation """
+    # ret_val = y_pp1 - y_pp2 - 2
+
+    """ Two dimensional Laplace equation """
+    # ret_val = y_pp2 + y_pp1
+
+    ret_val = y_p1 - 2 * x
+
+
+
+    """ FP Eqn """
+    if fpe_example == 1:
+        # Example 1
+        a = -1.0
+        a_p = 0.0
+        b = 1.0
+        b_p = 0.0
+        b_pp = 0.0
+    elif fpe_example == 2:
+        # Example 2
+        a = x
+        a_p = 1.0
+        b = tf.math.square(x) / 2
+        b_p = x
+        b_pp = 1.0
+    elif fpe_example == 3:
+        # Example 3
+        a = -1.0 - x
+        a_p = -1.0
+        b = tf.multiply(x ** 2, tf.math.exp(t))
+        b_p = 2 * x * tf.math.exp(t)
+        b_pp = 2 * tf.math.exp(t)
+    elif fpe_example == 4:
+        # Example 4
+        a = 4.0 * tf.math.divide_no_nan(y, x) - x / 3.0
+        a_p = 4.0 * (tf.math.divide_no_nan(y_p1, x) - tf.math.divide_no_nan(y, x ** 2)) - 1.0 / 3
+        b = y
+        b_p = y_p1
+        b_pp = y_pp1
+    elif fpe_example == 6:
+        # Example 5
+        a = 0.0
+        a_p = 0.0
+        b = 0.5
+        b_p = 0.0
+        b_pp = 0.0
+
+    if fpe_example > 0:
+        t1 = tf.multiply(y, b_pp - a_p)
+        t2 = tf.multiply(y_p1, 2 * b_p - a)
+        t3 = tf.multiply(y_pp1, b)
+        # print("a: {}".format(a.shape))
+        # print("a_p: {}".format(a_p.shape))
+        # print("b: {}".format(a.shape))
+        # print("b_p: {}".format(b_p.shape))
+        # print("b_pp: {}".format(b_pp.shape))
+        # print("t1: {}".format(t1.shape))
+        # print("t2: {}".format(t2.shape))
+        # print("t3: {}".format(t3.shape))
+
+        ret_val = y_p2 - (t1 + t2 + t3)
+
+    if fpe_example == 3:
+        ret_val = y_p2 - ((x + 1) * y_p1 + x ** 2 * tf.math.exp(t) * y_pp1)
+
+    if fpe_example == 4:
+        t1 = y * (y_pp1 * x ** 2 - 4 * y_p1 * x + 4 * y + x * x / 3.0)
+        t2 = y_p1 * (2 * x * y_p1 - 4 * x * y + x * x * x / 3.0)
+        t3 = x ** 2 * y_pp1 * y
+        ret_val = y_p2 * x * x - (t1 + t2 + t3)
+
+    if fpe_example == 5:
+        ret_val = y_p3 - (-2 * y + 3 * x * y_p1 - w * y_p2 + x ** 2 * y_pp1 + w ** 2 * y_pp2 + 2 * y_pp12)
+
+    return ret_val
+
+"""""""""""""""""""""""""""
+Initial values
+"""""""""""""""""""""""""""
+# initialize_ops are given in bottom-up order.
+initialize_ops = np.zeros([2 ** n_tree_layers - 1])
+# initialize_ops = ["mul", "mul", "id"]
+# if fpe_example in [1, 2, 3, 5]:
+#     initialize_ops = ["id", "exp", "mul"]
+# elif fpe_example in [4]:
+#     initialize_ops = ["mul", "exp", "mul"]
+
+
+# initialize_ops = ["exp", "sin", "mul"]
+#
+min_x = 0
+max_t = 5
+min_t = 0
+n_bc_points = 5
+
+# Initial values for (x, y)
+
+fixed_x = []
+fixed_y = []
+# for i in range(n_bc_points):
+#     t_i = i * (max_t - min_t)/n_bc_points + min_t
+#     fixed_x.append([0, t_i])
+#     # fixed_y.append(0)
+#     fixed_y.append(1 - np.exp(-1 * t_i))
+#     fixed_x.append([np.pi, t_i])
+#     fixed_y.append(np.exp(-1 * t_i) - 1)
+# fixed_y.append(0)
+
+
+
+# Initial values for (x, y')
+fixed_x_p1 = []
+fixed_y_p1 = []
+fixed_x_p2 = []
+fixed_y_p2 = []
+
+if mode == "de":
+    if fpe_example != 5:
+        for i in range(n_bc_points):
+            x_i = i * (max_x - min_x) / (n_bc_points - 1) + min_x
+            fixed_x.append([x_i, 0])
+            if fpe_example in [1, 2, 5]:
+                fixed_y.append(x_i)
+            elif fpe_example == 3:
+                fixed_y.append(x_i + 1)
+            elif fpe_example == 4:
+                fixed_y.append(x_i ** 2)
+            # fixed_y.append(0)
+            # fixed_y.append(x_i + np.cos(x_i))
+            # fixed_x_p2.append([x_i, 0])
+            # fixed_y_p2.append(np.sin(x_i))
+
+    if fpe_example == 5:
+        for i in range(n_bc_points):
+            x_i = i * (max_x - min_x) / (n_bc_points - 1) + min_x
+            for j in range(n_bc_points):
+                w_j = j * (max_x - min_x) / (n_bc_points - 1) + min_x
+
+                fixed_x.append([x_i, w_j, 0])
+                fixed_y.append(x_i)
+
+# print("IVP (x, y):\n{}".format([(fixed_x[i], fixed_y[i]) for i in range(len(fixed_x))]))
+# print("IVP (x, y_p1):\n{}".format([(fixed_x_p1[i], fixed_y_p1[i]) for i in range(len(fixed_x_p1))]))
+# print("IVP (x, y_p2):\n{}".format([(fixed_x_p2[i], fixed_y_p2[i]) for i in range(len(fixed_x_p2))]))
+
+# Weight to give IVP error
+ivp_lambda = 10
+
+
+"""""""""""""""""""""""""""
+Training hyperparameters
+"""""""""""""""""""""""""""
+quick_train_fraction = 0.7
+
+# Probably don't need to change any of the ones below
+max_training_batch_size = 1000
+t1_fraction = 5/20
+t2_fraction = 15 / 20
+
+train_N = 5000
+test_N = 1000
+
+eps = 1e-4
+big_eps = 1e-3
+d_eps = 2.0e-2
+learn_rate = 0.001
+w_matrix_stddev = 0.1
+init_weight_value = 5

SymbolicFunctionLearner.py

@@ -0,0 +1,1449 @@
+""""""""""""""""""""""""""""""""" 
+Do not run or modify this file.
+
+For running: DiffEqnSolver.py
+For modifying: Settings.py
+"""""""""""""""""""""""""""""""""
+
+import datetime
+import time
+
+import numpy as np
+import tensorflow as tf
+
+import DataUtils
+import Settings
+from DataUtils import choices_to_init_weight_matrix
+from DataUtils import tf_diff_sqrt, tf_diff_log, our_tanh, spike
+from Settings import implicit_function, d_eps
+
+tf.compat.v1.disable_eager_execution()
+
+
+def new_weight_matrix(n_rows, n_cols, mean=0.0, name=None):
+    initial = tf.random.normal(shape=[n_rows, n_cols], mean=mean, stddev=Settings.w_matrix_stddev)
+    if name is not None:
+        return tf.Variable(initial, name=name)
+    return tf.Variable(initial)
+
+
+def new_bias(n_cols, name=None):
+    initial = tf.zeros(shape=[1, n_cols])
+    if name is not None:
+        return tf.Variable(initial, name=name)
+    return tf.Variable(initial)
+
+
+def operate_on_tensors(tensor_A, tensor_B, fn_set, use_both_for_unary=True):
+    # print('op on tensors. input shapes: {}, {}'.format(tensor_A.shape, tensor_B.shape))
+    if use_both_for_unary:
+        w2 = 1.0
+    else:
+        w2 = 0.0
+
+    answer_vector = []
+    for operator_i in fn_set:
+        if operator_i == 'id':
+            answer_vector.extend([tensor_A[:, :, 0] + w2 * tensor_B[:, :, 0]])
+            # print("id vector shape: {}".format(answer_vector[-1].shape))
+        elif operator_i == 'add':
+            answer_vector.extend([tensor_A[:, :, 0] + tensor_B[:, :, 0]])
+        elif operator_i == 'sin':
+            answer_vector.extend([tf.sin(tensor_A[:, :, 0] + w2 * tensor_B[:, :, 0])])
+        elif operator_i == 'cos':
+            answer_vector.extend([tf.cos(tensor_A[:, :, 0] + w2 * tensor_B[:, :, 0])])
+        elif operator_i == 'sqrt':
+            answer_vector.extend([tf_diff_sqrt(tensor_A[:, :, 0] + w2 * tensor_B[:, :, 0])])
+        elif operator_i == 'mul':
+            answer_vector.extend([tf.multiply(tensor_A[:, :, 0], tensor_B[:, :, 0])])
+        elif operator_i == 'div':
+            answer_vector.extend([tf.math.divide_no_nan(tensor_A[:, :, 0], tensor_B[:, :, 0])])
+        elif operator_i == 'log':
+            answer_vector.extend([tf_diff_log(tensor_A[:, :, 0] + w2 * tensor_B[:, :, 0])])
+        elif operator_i == 'exp':
+            answer_vector.extend([tf.exp(our_tanh(tensor_A[:, :, 0] + w2 * tensor_B[:, :, 0], factor=np.log(50000)))])
+        else:
+            answer_vector.extend([None])
+
+    return tf.stack(answer_vector, axis=-1)
+
+
+def sm_no_const_selector(nonflat_input, flat_input, initial_weights):
+    # print("sm_no_const_selector---")
+    # print("initial_weights: {}".format(initial_weights.shape))
+    # print("nonflat_input: {}".format(nonflat_input.shape))
+    pre_sm_weights = new_weight_matrix(int(nonflat_input.shape[-1]), 1)
+    post_sm_weights = tf.math.softmax(pre_sm_weights + initial_weights, axis=0)
+    # print("post_sm_weights: {}".format(post_sm_weights.shape))
+    sm_result = tf.matmul(nonflat_input, post_sm_weights)
+    # print("sm_result: {}".format(sm_result.shape))
+
+    flat_weights = tf.multiply(post_sm_weights,
+                               tf.cast(tf.greater(post_sm_weights,
+                                                  tf.reduce_max(post_sm_weights) - 0.01), tf.float32))
+    flat_weights = tf.divide(flat_weights, tf.reduce_sum(flat_weights))
+    flat_result = tf.matmul(flat_input, flat_weights)
+
+    return sm_result, flat_result, pre_sm_weights+initial_weights, post_sm_weights, flat_weights
+
+
+def collect_op_inputs_str(weight_w, weight_b, input_strs):
+    num_inputs = weight_w.shape[0]
+    # print("num_inputs: {}. input_strs length: {}".format(num_inputs, len(input_strs)))
+    # print(weight_w)
+    # print(input_strs)
+    temp_answer = ''
+    has_one = False
+    has_more_than_one = False
+    for row in range(num_inputs):
+        if np.abs(weight_w[row][0]) > Settings.big_eps and input_strs[row] != '0':
+            if has_one:
+                temp_answer += ' + '
+                has_more_than_one = True
+            if np.abs(weight_w[row][0] - 1) < Settings.big_eps:
+                temp_answer += '{}'.format(input_strs[row])
+            else:
+                temp_answer += '{:.4f}*({})'.format(weight_w[row][0], input_strs[row])
+            has_one = True
+    # print('weight_b[-1]: {}'.format(weight_b))
+    if np.abs(weight_b[-1][0]) > Settings.big_eps:
+        if has_one:
+            temp_answer += ' + '
+            has_more_than_one = True
+        temp_answer += '{:.4f}'.format(weight_b[-1][0])
+    if len(temp_answer) == 0:
+        temp_answer = '0'
+    if has_more_than_one:
+        return '(' + temp_answer + ')'
+    return temp_answer
+
+def collect_minimal_op_inputs_str(weight_w, input_strs):
+    num_inputs = weight_w.shape[0]
+    temp_answer = ''
+    has_one = False
+    has_more_than_one = False
+    for row in range(num_inputs):
+        if has_one:
+            temp_answer += ' + '
+            has_more_than_one = True
+        temp_answer += '{}'.format(input_strs[row])
+        has_one = True
+
+    if len(temp_answer) == 0:
+        temp_answer = '0'
+    if has_more_than_one:
+        return '(' + temp_answer + ')'
+    return temp_answer
+
+def operation_to_str_best(weight_w, weight_b, weight_sm, input_strs1, input_strs2, fn_set,
+                          digits=None, unary_both=True, minimal=False):
+    if input_strs2 is None:
+        temp_answer = collect_op_inputs_str(weight_w, weight_b, input_strs1)
+        return [temp_answer]
+
+    answer = ['0' for _ in fn_set]
+
+    temp_answer1 = input_strs1
+    temp_answer2 = input_strs2
+
+    # Set up temp answer. Don't change this value!
+    if unary_both:
+        if temp_answer1 == '0' and temp_answer2 == '0':
+            temp_answer = '0'
+        elif temp_answer1 == '0':
+            temp_answer = str(temp_answer2)
+        elif temp_answer2 == '0':
+            temp_answer = str(temp_answer1)
+        else:
+            temp_answer = '({} + {})'.format(temp_answer1, temp_answer2)
+    else:
+        temp_answer = str(temp_answer1)
+
+    if 'id' in fn_set:
+        fn_index = fn_set.index('id')
+        answer[fn_index] = temp_answer
+
+    if 'sin' in fn_set:
+        fn_index = fn_set.index('sin')
+        if temp_answer == '0':
+            answer[fn_index] = '0'
+        else:
+            answer[fn_index] = 'sin({})'.format(temp_answer)
+
+    if 'cos' in fn_set:
+        fn_index = fn_set.index('cos')
+        answer[fn_index] = 'cos({})'.format(temp_answer)
+
+    if 'sqrt' in fn_set:
+        fn_index = fn_set.index('sqrt')
+        answer[fn_index] = '(abs({}))^(0.5)'.format(temp_answer)
+
+    if 'log' in fn_set:
+        fn_index = fn_set.index('log')
+        if temp_answer == '0':
+            answer[fn_index] = 'log(0.0001)'
+        else:
+            answer[fn_index] = 'log({})'.format(temp_answer)
+
+    if 'mul' in fn_set:
+        fn_index = fn_set.index('mul')
+        if temp_answer1 == '0' or temp_answer2 == '0':
+            prod_answer = '0'
+        else:
+            prod_answer = '({} * {})'.format(temp_answer1, temp_answer2)
+        answer[fn_index] = prod_answer
+
+    if 'add' in fn_set:
+        fn_index = fn_set.index('add')
+        if temp_answer1 == '0' and temp_answer2 == '0':
+            sum_answer = '0'
+        elif temp_answer1 == '0':
+            sum_answer = str(temp_answer2)
+        elif temp_answer2 == '0':
+            sum_answer = str(temp_answer1)
+        else:
+            sum_answer = '({} + {})'.format(temp_answer1, temp_answer2)
+        answer[fn_index] = sum_answer
+
+    if 'sub' in fn_set:
+        fn_index = fn_set.index('sub')
+        temp_answer1 = input_strs1
+        temp_answer2 = input_strs2
+
+        if temp_answer1 == '0' and temp_answer2 == '0':
+            diff_answer = '0'
+        elif temp_answer1 == '0':
+            diff_answer = "-{}".format(temp_answer2)
+        elif temp_answer2 == '0':
+            diff_answer = temp_answer1
+        else:
+            diff_answer = '({} - {})'.format(temp_answer1, temp_answer2)
+        answer[fn_index] = diff_answer
+
+    if 'max' in fn_set:
+        fn_index = fn_set.index('max')
+        answer[fn_index] = 'max({}, {})'.format(temp_answer1, temp_answer2)
+
+    if 'min' in fn_set:
+        fn_index = fn_set.index('min')
+        answer[fn_index] = 'min({}, {})'.format(temp_answer1, temp_answer2)
+
+    if 'div' in fn_set:
+        fn_index = fn_set.index('div')
+        if temp_answer2 == '0':
+            temp_answer2 = '0.001'
+        if temp_answer1 == '0':
+            div_answer = '0'
+        else:
+            div_answer = '({} / ({}))'.format(temp_answer1, temp_answer2)
+        answer[fn_index] = div_answer
+
+    if 'exp' in fn_set:
+        fn_index = fn_set.index('exp')
+        answer[fn_index] = 'exp({})'.format(temp_answer)
+
+    new_answer = [collect_op_inputs_str(weight_sm, np.zeros([1, 1]), answer)]
+    # print('New answer: {}'.format(new_answer))
+    # print('weight w, weight b: {}, {}'.format(weight_w, weight_b))
+    if minimal:
+        ret_val = collect_minimal_op_inputs_str(weight_w, new_answer)
+    else:
+        ret_val = collect_op_inputs_str(weight_w, weight_b, new_answer)
+    return ret_val
+
+
+def flattened_sm_result(input_x, sm_applied_weights, our_w, our_b):
+    # print('Create operator node. Input shapes: {}, {}'.format(input_1.shape, input_2.shape))
+    new_sm_weights = tf.multiply(sm_applied_weights,
+                                 tf.cast(tf.greater(sm_applied_weights,
+                                                    tf.reduce_max(sm_applied_weights) - 0.01), tf.float32))
+    new_sm_weights = tf.divide(new_sm_weights, tf.reduce_sum(new_sm_weights))
+
+    sm_result = tf.matmul(input_x, new_sm_weights)
+    final_result = tf.multiply(sm_result, our_w) + our_b
+    # print('  Final result shape: {}'.format(final_result.shape))
+
+    return final_result, new_sm_weights
+
+
+class SFL:
+    def __init__(self, var_names=None):
+
+        self.name = "Symbolic Function Learner"
+        self.short_name = "SFL"
+
+        # mode: in ["sr", "de", "lr"]
+        self.mode = Settings.mode
+
+        # main hyperparameters of the symbolic expression
+        self.n_tree_layers = Settings.n_tree_layers
+        self.function_set = Settings.function_set.copy()
+        self.n_input_variables = Settings.num_features
+        self.n_dims_per_variable = Settings.num_dims_per_feature
+        self.n_dims_in_output = Settings.n_dims_in_output
+        assert self.n_dims_in_output in [1, self.n_dims_per_variable]
+
+        # use_both_for_unary: decide how to handle two input children
+        # for a unary operator.
+        # True: add the two inputs.
+        # False: keep first input; discard second input.
+        self.use_both_for_unary = Settings.use_both_for_unary
+
+        # Use a softmax on leaf layer?
+        self.sm_leaf_layer = Settings.use_leaf_sm
+
+        # data_x,y: Input (x, y) values over which we are training.
+        # For symbolic regression, it's the same as fixed_x,y.
+        # For differential equations, it's random values.
+        self.data_x = tf.compat.v1.placeholder("float", [None, self.n_dims_per_variable,
+                                                         self.n_input_variables], name="data_x")
+        self.data_y = tf.compat.v1.placeholder("float", [None, self.n_dims_per_variable, 1], name="data_y")
+
+        # Fixed_x,y: these are the set of points that must be satisfied
+        # by the function that is learned. These are  used to compute
+        # the residual error in the cost function.
+        self.fixed_x = tf.compat.v1.placeholder("float", [None, self.n_dims_per_variable,
+                                                          self.n_input_variables], name="data_x")
+        self.fixed_y = tf.compat.v1.placeholder("float", [None, self.n_dims_per_variable, 1], name="data_y")
+
+        # To initialize operators in the SFL with a warm start before training
+        self.init_op_weights = tf.compat.v1.placeholder("float", [len(self.function_set), 2 ** self.n_tree_layers - 1],
+                                                        name="init_op_weights")
+        self.init_op_weight_matrix = np.zeros(shape=[len(self.function_set), 2 ** self.n_tree_layers - 1])
+
+        # To initialize variable choices in the SFL with a warm start before training
+        num_var_input_choices = self.n_input_variables
+        self.init_var_weights = tf.compat.v1.placeholder("float", [num_var_input_choices, 2 ** self.n_tree_layers],
+                                                         name="init_var_weights")
+        self.init_var_weight_matrix = np.zeros(shape=[num_var_input_choices, 2 ** self.n_tree_layers])
+
+        # variables can have default or custom names
+        if self.n_input_variables == 1 and var_names is None:
+            self.var_names = ['x']
+        elif var_names is None:
+            self.var_names = ['x{}'.format(i + 1) for i in range(self.n_input_variables)]
+        else:
+            self.var_names = var_names
+
+        self.learn_rate = Settings.learn_rate
+
+        self.y_gold = self.data_y
+        self.g_error = tf.Variable(0.0)
+        self.g_error_not_flat = tf.Variable(0.0)
+        self.mse = tf.Variable(0.0)
+        self.mse_not_flat = tf.Variable(0.0)
+        self.spike_error = tf.Variable(0.0)
+        self.ivp_error = tf.Variable(0.0)
+        self.ivp_error_not_flat = tf.Variable(0.0)
+        self.total_error = tf.Variable(0.0)
+
+        if self.mode == "de":
+            self.ivp_lambda = Settings.ivp_lambda
+        else:
+            self.ivp_lambda = 0
+
+        self.train_accuracy_log = []
+        self.valid_accuracy_log = []
+        self.test_accuracy_log = []
+
+        self.seen_eqns = []
+        self.seen_minimal_eqns = []
+        self.log_iters = []
+
+        self.best_accuracy_so_far = 9999999
+        self.best_formula_so_far = ""
+        self.best_iter = 0
+
+        self.y_hat = None
+        self.y_hat_p1 = None
+        self.y_hat_pp1 = None
+        self.y_hat_p2 = None
+        self.y_hat_pp2 = None
+        self.y_hat_p3 = None
+        self.y_hat_pp3 = None
+        self.y_hat_pp12 = None
+        self.implicit_g = None
+        self.y_hat_not_flat = None
+        self.y_hat_p_not_flat = None
+        self.y_hat_pp_not_flat = None
+        self.implicit_g_not_flat = None
+
+        self.W_matrices = []
+        self.b_matrices = []
+        self.non_sm_weights = []
+        self.leaf_sm_weights = []
+        self.sm_W_matrices = []
+        self.sm_applied_W_matrices = []
+        self.flattened_W_matrices = []
+
+        self.use_both_for_unary = Settings.use_both_for_unary
+
+        self.init = None
+        self.sess = None
+
+        self.build_sfl()
+        self.reset(var_names)
+
+    def build_sfl(self):
+        self.data_x = tf.compat.v1.placeholder("float", [None, self.n_dims_per_variable,
+                                                         self.n_input_variables], name="data_x")
+        self.data_y = tf.compat.v1.placeholder("float", [None, self.n_dims_per_variable, 1], name="data_y")
+        self.fixed_x = tf.compat.v1.placeholder("float", [None, self.n_dims_per_variable,
+                                                          self.n_input_variables], name="fixed_x")
+        self.fixed_y = tf.compat.v1.placeholder("float", [None, self.n_dims_per_variable, 1], name="fixed_y")
+
+        # To initialize operators in the SFL with a warm start before training
+        self.init_op_weights = tf.compat.v1.placeholder("float", [len(self.function_set), 2 ** self.n_tree_layers - 1],
+                                                        name="init_op_weights")
+        # To initialize variable choices in the SFL with a warm start before training
+        # Right now, only one variable is supported.
+        num_var_input_choices = self.n_input_variables
+
+        self.init_var_weights = tf.compat.v1.placeholder("float", [num_var_input_choices, 2 ** self.n_tree_layers],
+                                                         name="init_var_weights")
+
+        self.g_error = tf.Variable(0.0)
+        self.g_error_not_flat = tf.Variable(0.0)
+        self.mse = tf.Variable(0.0)
+        self.mse_not_flat = tf.Variable(0.0)
+        self.spike_error = tf.Variable(0.0)
+        self.ivp_error = tf.Variable(0.0)
+        self.ivp_error_not_flat = tf.Variable(0.0)
+        self.total_error = tf.Variable(0.0)
+
+        self.W_matrices = []
+        self.b_matrices = []
+        self.non_sm_weights = []
+        self.leaf_sm_weights = []
+        self.sm_W_matrices = []
+        self.sm_applied_W_matrices = []
+        self.flattened_W_matrices = []
+
+        previous_output = []
+        previous_flat_output = []
+        weight_layer = []
+        bias_layer = []
+
+        if Settings.show_output:
+            print("Setting up {} model.".format(self.name))
+            print("  {} tree layers.".format(self.n_tree_layers))
+            print("  {} features of {} component(s) each.".format(self.n_input_variables, self.n_dims_per_variable))
+            print("  {} component(s) in output.".format(self.n_dims_in_output))
+            print("  {} operators: {}.".format(len(self.function_set),
+                                               self.function_set))
+
+        # Set up leaf layer
+        for i in range(2 ** (Settings.n_tree_layers - 1)):
+            if self.sm_leaf_layer:
+                num_leaf_weights = 1
+            else:
+                num_leaf_weights = self.n_input_variables
+
+            new_weights1 = new_weight_matrix(num_leaf_weights, 1, mean=0.0)
+            new_b1 = new_bias(1)
+
+            new_weights2 = new_weight_matrix(num_leaf_weights, 1, mean=0.0)
+            new_b2 = new_bias(1)
+
+            # print("self.data_x.shape: {}".format(self.data_x.shape))
+
+            if self.sm_leaf_layer:
+                new_sm_weights1 = new_weight_matrix(self.n_input_variables, 1, mean=0.0)
+                new_sm_weights2 = new_weight_matrix(self.n_input_variables, 1, mean=0.0)
+
+                input_1 = tf.matmul(self.data_x, tf.math.softmax(new_sm_weights1, axis=0))
+                input_2 = tf.matmul(self.data_x, tf.math.softmax(new_sm_weights2, axis=0))
+
+                # todo: ugh
+                # new_weights1 = tf.constant([[1.0]])
+                # new_weights2 = tf.constant([[1.0]])
+            else:
+                input_1 = self.data_x
+                input_2 = self.data_x
+
+            # print("input_1.shape: {}".format(input_1.shape))
+            result_1 = tf.matmul(input_1, new_weights1) + new_b1
+            result_2 = tf.matmul(input_2, new_weights2) + new_b2
+
+            # print("result_1.shape: {}".format(result_1.shape))
+            weight_layer.extend([new_weights1, new_weights2])
+            bias_layer.extend([new_b1, new_b2])
+            if self.sm_leaf_layer:
+                self.leaf_sm_weights.extend([tf.math.softmax(new_sm_weights1, axis=0),
+                                             tf.math.softmax(new_sm_weights2, axis=0)])
+            self.non_sm_weights.extend([new_weights1, new_weights2, new_b1, new_b2])
+            # self.non_sm_weights.extend([new_weights1, new_weights2])
+
+            previous_output.extend([result_1, result_2])
+            previous_flat_output.extend([result_1, result_2])
+
+        self.W_matrices.append(weight_layer)
+        self.b_matrices.append(bias_layer)
+        self.sm_W_matrices.append([])
+        self.sm_applied_W_matrices.append([])
+        self.flattened_W_matrices.append([])
+        current_node = 0
+
+        # Set up parent layers, one at a time going up
+        for j in range(Settings.n_tree_layers):
+            sm_weight_layer = []
+            sm_applied_weight_layer = []
+            flattened_weight_layer = []
+            weight_layer = []
+            bias_layer = []
+            new_output = []
+            new_flat_output = []
+            result_layer = []
+            flattened_result_layer = []
+
+            for i in range(2 ** (Settings.n_tree_layers - j - 1)):
+                current_input_1 = previous_output[2 * i]
+                current_input_2 = previous_output[2 * i + 1]
+                nonflatten_input = operate_on_tensors(current_input_1, current_input_2, self.function_set,
+                                                      use_both_for_unary=self.use_both_for_unary)
+
+                current_flat_input_1 = previous_flat_output[2 * i]
+                current_flat_input_2 = previous_flat_output[2 * i + 1]
+                flatten_input = operate_on_tensors(current_flat_input_1, current_flat_input_2,
+                                                   self.function_set,
+                                                   use_both_for_unary=self.use_both_for_unary)
+
+                init_op_weights = tf.reshape(self.init_op_weights[:, current_node], [-1, 1])
+                sm_r, flat_r, pre_sm_w, post_sm_w, flat_w = sm_no_const_selector(nonflatten_input,
+                                                                                 flatten_input,
+                                                                                 init_op_weights)
+                new_w = new_weight_matrix(1, 1, mean=1.0)
+                new_b = new_bias(1)
+                # self.non_sm_weights.extend([new_b])
+
+                sm_r = tf.math.multiply(sm_r, new_w) + new_b
+                flat_r = tf.multiply(flat_r, new_w) + new_b
+
+                sm_weight_layer.extend([pre_sm_w])
+                sm_applied_weight_layer.extend([post_sm_w])
+                flattened_weight_layer.extend([flat_w])
+
+                new_output.extend([sm_r])
+                new_flat_output.extend([flat_r])
+
+                weight_layer.extend([new_w])
+                bias_layer.extend([new_b])
+                """ self.non_sm_weights.extend([new_w, new_b])"""
+
+                result_layer.extend([sm_r])
+                flattened_result_layer.extend([flat_r])
+
+                current_node += 1
+
+            self.sm_W_matrices.extend([sm_weight_layer])
+            self.sm_applied_W_matrices.extend([sm_applied_weight_layer])
+            self.flattened_W_matrices.extend([flattened_weight_layer])
+            self.W_matrices.extend([weight_layer])
+            self.b_matrices.extend([bias_layer])
+
+            previous_output = new_output
+            previous_flat_output = new_flat_output
+
+        if self.mode == "lr":
+            self.y_hat_not_flat = spike(previous_output[-1])
+            self.y_hat = spike(previous_flat_output[-1])
+        else:
+            self.y_hat_not_flat = our_tanh(previous_output[-1], factor=10000)
+            self.y_hat = our_tanh(previous_flat_output[-1], factor=10000)
+
+    def reset(self, var_names=None):
+        tf.compat.v1.reset_default_graph()
+        self.build_sfl()
+
+        if var_names is not None:
+            self.var_names = var_names
+
+        self.log_iters = []
+        self.train_accuracy_log = []
+        self.valid_accuracy_log = []
+        self.test_accuracy_log = []
+        self.seen_eqns = []
+        self.seen_minimal_eqns = []
+
+        self.setup_derivative_values()
+        self.setup_err_values(non_const=Settings.non_const)
+
+        # TODO: really need to sort out the whole fixed_x, fixed_y thing
+        if self.mode == "de":
+            self.ivp_error_not_flat, self.ivp_error = self.setup_ivp_values(self.fixed_x, self.fixed_y)
+
+        if self.mode == "de":
+            self.total_error = self.total_error + self.g_error
+        if Settings.non_const:
+            self.total_error = self.total_error + self.spike_error
+
+        self.total_error = self.total_error + self.mse + self.ivp_lambda * self.ivp_error
+
+        sum_of_squares = tf.reduce_sum([tf.reduce_sum(tf.square(reg_w)) for reg_w in self.non_sm_weights])
+        sum_of_squares_minus_max = sum_of_squares - tf.reduce_sum([tf.reduce_max(tf.square(reg_w))
+                                                                   for reg_w in self.non_sm_weights])
+
+        self.regularization_penalty = tf.reduce_mean([tf.reduce_sum(tf.abs(reg_w))
+                                                      for reg_w in self.non_sm_weights])
+
+        # self.regularization_penalty += sum_of_squares_minus_max
+
+        self.loss_function1 = self.mse_not_flat + self.g_error_not_flat + self.spike_error + self.ivp_lambda * self.ivp_error_not_flat
+
+        self.loss_function2 = self.mse + self.g_error + self.spike_error + self.ivp_lambda * self.ivp_error
+        self.loss_function2 += self.regularization_penalty * 0.05  # 0.1
+
+        self.loss_function3 = self.mse + self.g_error + self.spike_error + self.ivp_lambda * self.ivp_error
+        self.loss_function3 += self.regularization_penalty * 0.9  # 1.0
+
+        self.opt = tf.compat.v1.train.AdamOptimizer(self.learn_rate)
+
+        self.train_step_1 = self.opt.minimize(self.loss_function1)
+        self.train_step_2 = self.opt.minimize(self.loss_function2)
+        self.train_step_3 = self.opt.minimize(self.loss_function3)
+
+        self.init = tf.compat.v1.global_variables_initializer()
+        self.sess = tf.compat.v1.Session()
+        self.sess.run(self.init)
+
+        self.best_accuracy_so_far = 9999999
+        self.best_formula_so_far = ""
+        self.best_iter = 0
+
+    def setup_err_values(self, non_const=False):
+        if self.mode == "de":
+            self.g_error = tf.reduce_mean(tf.math.square(self.implicit_g))
+            self.g_not_flat = tf.reduce_mean(tf.math.square(self.implicit_g_not_flat))
+        else:
+            self.g_error = tf.Variable(0.0)
+            self.g_error_not_flat = tf.Variable(0.0)
+            self.mse = tf.reduce_mean(tf.math.squared_difference(self.y_hat, self.data_y))
+            self.mse_not_flat = tf.reduce_mean(tf.math.squared_difference(self.y_hat_not_flat, self.data_y))
+
+        if non_const:
+            self.spike_error = tf.reduce_mean(spike(self.y_hat_p1))
+        # tf.reduce_sum(spike(self.y_hat_p1) + spike(self.y_hat_p2) + spike(self.y_hat_p3))
+
+    def setup_ivp_values(self, fixed_x_ph, fixed_y_ph):
+        y_hat_err_not_flat = tf.Variable(0.0)
+        y_hat_err = tf.Variable(0.0)
+
+        if fixed_x_ph is not None:
+            y_hat_err_not_flat = tf.reduce_mean(tf.math.squared_difference(fixed_y_ph,
+                                                                           self.eval_formula(fixed_x_ph, flat=False)))
+            y_hat_err = tf.reduce_mean(tf.math.squared_difference(fixed_y_ph, self.eval_formula(fixed_x_ph)))
+
+        eye = tf.eye(self.n_input_variables)
+        u1 = eye[:, 0]
+        if Settings.fixed_x_p1 is not None and len(Settings.fixed_x_p1) > 0:
+            fixed_x_p1 = tf.constant(np.reshape(Settings.fixed_x_p1,
+                                                [-1, Settings.num_dims_per_feature, Settings.num_features]),
+                                     dtype="float32")
+            fixed_y_p1 = tf.constant(np.reshape(Settings.fixed_y_p1,
+                                                [-1, Settings.n_dims_in_output, 1]),
+                                     dtype="float32")
+
+            y_p1_fixed_hat = self.eval_formula(fixed_x_p1 + d_eps * u1 / 2)
+            y_p1_fixed_hat -= self.eval_formula(fixed_x_p1 - d_eps * u1 / 2)
+            y_p1_fixed_hat = y_p1_fixed_hat / d_eps
+
+            y_hat_err_not_flat += tf.reduce_mean(tf.math.squared_difference(fixed_y_p1, y_p1_fixed_hat))
+            y_hat_err += tf.reduce_mean(tf.math.squared_difference(fixed_y_p1, y_p1_fixed_hat))
+
+        if self.n_input_variables > 1:
+            u2 = eye[:, 1]
+            if Settings.fixed_x_p2 is not None and len(Settings.fixed_x_p2) > 0:
+                fixed_x_p2 = tf.constant(np.reshape(Settings.fixed_x_p2,
+                                                    [-1, Settings.num_dims_per_feature, Settings.num_features]),
+                                         dtype="float32")
+                fixed_y_p2 = tf.constant(np.reshape(Settings.fixed_y_p2,
+                                                    [-1, Settings.n_dims_in_output, 1]),
+                                         dtype="float32")
+
+                y_p2_fixed_hat = self.eval_formula(fixed_x_p2 + d_eps * u2 / 2)
+                y_p2_fixed_hat -= self.eval_formula(fixed_x_p2 - d_eps * u2 / 2)
+                y_p2_fixed_hat = y_p2_fixed_hat / d_eps
+
+                y_hat_err_not_flat += tf.reduce_mean(tf.math.squared_difference(fixed_y_p2, y_p2_fixed_hat))
+                y_hat_err += tf.reduce_mean(tf.math.squared_difference(fixed_y_p2, y_p2_fixed_hat))
+
+        return y_hat_err_not_flat, y_hat_err
+
+    def get_formula_string(self, digits=None):
+        eval_dict = {self.init_op_weights: self.init_op_weight_matrix,
+                     self.init_var_weights: self.init_var_weight_matrix}
+
+        inputs = []
+        for i in range(len(self.W_matrices[0])):
+            w_matrix = self.W_matrices[0][i].eval(session=self.sess)
+            b_vector = self.b_matrices[0][i].eval(session=self.sess)
+            if self.sm_leaf_layer:
+                sm_vector = self.leaf_sm_weights[i].eval(session=self.sess)
+                print("sm_vector: {}".format(sm_vector))
+
+                new_answer = [collect_op_inputs_str(sm_vector, np.zeros([1, 1]), self.var_names)]
+                new_input = collect_op_inputs_str(w_matrix, b_vector, new_answer)
+            else:
+                new_input = collect_op_inputs_str(w_matrix, b_vector, self.var_names)
+
+            inputs.extend([new_input])
+        for layer_i in range(1, len(self.W_matrices)):
+            sm_applied_this_layer = self.flattened_W_matrices[layer_i]
+            w_this_layer = self.W_matrices[layer_i]
+            b_this_layer = self.b_matrices[layer_i]
+            new_inputs = []
+            for iii in range(0, len(w_this_layer)):
+                new_inputs.extend([operation_to_str_best(w_this_layer[iii].eval(self.sess),
+                                                         b_this_layer[iii].eval(self.sess),
+                                                         sm_applied_this_layer[iii].eval(session=self.sess,
+                                                                                         feed_dict=eval_dict),
+                                                         inputs[2 * iii],
+                                                         inputs[2 * iii + 1],
+                                                         self.function_set,
+                                                         unary_both=self.use_both_for_unary)])
+            inputs = new_inputs
+
+        if isinstance(inputs[0], list):
+            return inputs[0][0]
+        return inputs[0]
+
+    def get_minimal_formula_string(self):
+        eval_dict = {self.init_op_weights: self.init_op_weight_matrix,
+                     self.init_var_weights: self.init_var_weight_matrix}
+
+        inputs = []
+        for i in range(len(self.W_matrices[0])):
+            # w_matrix = self.W_matrices[0][i].eval(self.sess)
+            # inputs.extend([collect_minimal_op_inputs_str(w_matrix, self.var_names)])
+            inputs.append("A{}".format(i+1))
+
+        for layer_i in range(1, len(self.W_matrices)):
+            sm_applied_this_layer = self.flattened_W_matrices[layer_i]
+            w_this_layer = self.W_matrices[layer_i]
+
+            new_inputs = []
+            for iii in range(0, len(sm_applied_this_layer)):
+                new_inputs.extend([operation_to_str_best(w_this_layer[iii].eval(self.sess),
+                                                         None,
+                                                         sm_applied_this_layer[iii].eval(session=self.sess,
+                                                                                         feed_dict=eval_dict),
+                                                         inputs[2 * iii],
+                                                         inputs[2 * iii + 1],
+                                                         self.function_set,
+                                                         unary_both=self.use_both_for_unary,
+                                                         minimal=True)])
+            inputs = new_inputs
+
+        if isinstance(inputs[0], list):
+            return inputs[0][0]
+        return inputs[0]
+
+    def eval_formula(self, input_x, flat=True):
+        inputs = []
+        for i in range(len(self.W_matrices[0])):
+            w_matrix = self.W_matrices[0][i]
+            b_vector = self.b_matrices[0][i]
+
+            if self.sm_leaf_layer:
+                post_sm_weights = self.leaf_sm_weights[i]
+                sm_result = tf.matmul(input_x, post_sm_weights)
+                result = tf.multiply(sm_result, w_matrix) + b_vector
+            else:
+                result = tf.matmul(input_x, w_matrix) + b_vector
+
+            inputs.extend([result])
+
+        for layer_i in range(1, len(self.W_matrices)):
+            sm_flat_this_layer = self.flattened_W_matrices[layer_i]
+            sm_applied_this_layer = self.sm_applied_W_matrices[layer_i]
+            w_this_layer = self.W_matrices[layer_i]
+            b_this_layer = self.b_matrices[layer_i]
+            new_inputs = []
+
+            for iii in range(0, len(w_this_layer)):
+                post_sm_weights = sm_applied_this_layer[iii]
+                flat_sm_weights = sm_flat_this_layer[iii]
+
+                op_result = operate_on_tensors(inputs[2 * iii],
+                                               inputs[2 * iii + 1],
+                                               self.function_set,
+                                               use_both_for_unary=self.use_both_for_unary)
+
+                if flat:
+                    # result, flat_sm_weights = flattened_sm_result(op_result,
+                    #                                               post_sm_weights,
+                    #                                               w_this_layer[iii],
+                    #                                               b_this_layer[iii])
+                    sm_result = tf.matmul(op_result, flat_sm_weights)
+                else:
+                    sm_result = tf.matmul(op_result, post_sm_weights)
+                result = tf.multiply(sm_result, w_this_layer[iii]) + b_this_layer[iii]
+                new_inputs.extend([result])
+
+            inputs = new_inputs
+
+        if self.mode == "lr":
+            return spike(inputs[0])
+        return inputs[0]
+
+    def setup_derivative_values(self):
+
+        d2_eps = 1e-2
+        eye = tf.eye(self.n_input_variables)
+        u1 = eye[:, 0]
+        if self.n_input_variables > 1:
+            u2 = eye[:, 1]
+        if self.n_input_variables > 2:
+            u3 = eye[:, 2]
+        # u = []
+        # for i in range(self.n_input_variables):
+        #     u_i = eye[:, i]
+
+
+        # dy / dx1
+
+        self.y_hat_p1 = self.eval_formula(self.data_x + d_eps * u1 / 2)
+        self.y_hat_p1 -= self.eval_formula(self.data_x - d_eps * u1 / 2)
+        self.y_hat_p1 = self.y_hat_p1 / d_eps
+
+        # d^2y / dx1^2
+
+        self.y_hat_pp1 = self.eval_formula(self.data_x + d2_eps * u1)
+        self.y_hat_pp1 -= (2 * self.eval_formula(self.data_x))
+        self.y_hat_pp1 += self.eval_formula(self.data_x - d2_eps * u1)
+        self.y_hat_pp1 /= (d2_eps ** 2)
+
+        if self.n_input_variables > 1:
+            # dy / dx2
+
+            self.y_hat_p2 = self.eval_formula(self.data_x + d_eps * u2 / 2)
+            self.y_hat_p2 -= self.eval_formula(self.data_x - d_eps * u2 / 2)
+            self.y_hat_p2 = self.y_hat_p2 / d_eps
+
+            # d^2y / dx2^2
+
+            self.y_hat_pp2 = self.eval_formula(self.data_x + d2_eps * u2)
+            self.y_hat_pp2 -= (2 * self.eval_formula(self.data_x))
+            self.y_hat_pp2 += self.eval_formula(self.data_x - d2_eps * u2)
+            self.y_hat_pp2 /= (d2_eps ** 2)
+
+            # d^2y / dx1 dx2
+            self.y_hat_pp12 = self.eval_formula(self.data_x + d2_eps * (u1 + u2))
+            self.y_hat_pp12 -= self.eval_formula(self.data_x - d2_eps * (u1 - u2))
+            self.y_hat_pp12 -= self.eval_formula(self.data_x - d2_eps * (u2 - u1))
+            self.y_hat_pp12 -= self.eval_formula(self.data_x + d2_eps * (-u1 - u2))
+            self.y_hat_pp12 /= (4 * d2_eps ** 2)
+        else:
+            self.y_hat_p2 = None
+            self.y_hat_pp2 = None
+            self.y_hat_pp12 = None
+
+        if self.n_input_variables > 2:
+            # dy / dx2
+
+            self.y_hat_p3 = self.eval_formula(self.data_x + d_eps * u3 / 2)
+            self.y_hat_p3 -= self.eval_formula(self.data_x - d_eps * u3 / 2)
+            self.y_hat_p3 = self.y_hat_p3 / d_eps
+        else:
+            self.y_hat_p3 = None
+
+
+        self.y_hat_p_not_flat = self.eval_formula(self.data_x + d_eps * u1 / 2, flat=False)
+        self.y_hat_p_not_flat -= self.eval_formula(self.data_x - d_eps * u1 / 2, flat=False)
+        self.y_hat_p_not_flat = self.y_hat_p_not_flat / d_eps
+
+        self.y_hat_pp_not_flat = self.eval_formula(self.data_x + d_eps * u1, flat=False)
+        self.y_hat_pp_not_flat -= 2 * self.eval_formula(self.data_x, flat=False)
+        self.y_hat_pp_not_flat += self.eval_formula(self.data_x - d_eps * u1, flat=False)
+        self.y_hat_pp_not_flat = self.y_hat_pp_not_flat / d_eps ** 2
+
+        self.implicit_g = our_tanh(implicit_function(self.data_x, self.y_hat,
+                                                     [self.y_hat_p1, self.y_hat_p2, self.y_hat_p3],
+                                                     [self.y_hat_pp1, self.y_hat_pp2, self.y_hat_pp12]))
+        self.implicit_g_not_flat = our_tanh(implicit_function(self.data_x, self.y_hat_not_flat,
+                                                              [self.y_hat_p_not_flat, self.y_hat_p2, self.y_hat_p3],
+                                                              [self.y_hat_pp_not_flat, self.y_hat_pp2,
+                                                               self.y_hat_pp12]))
+
+    """ Like reset, but does not erase records of training history. 
+            It only restarts training from a new random initialization. """
+    def soft_reset(self):
+        self.init = tf.compat.v1.global_variables_initializer()
+        self.saver = tf.compat.v1.train.Saver()
+        self.sess = tf.compat.v1.Session()
+        self.sess.run(self.init)
+
+        self.best_accuracy_so_far = 9999999
+        self.best_formula_so_far = ""
+        self.best_iter = 0
+
+    # Not needed, but don't touch
+    def set_init_op_weight_matrix(self, init_op_weight_matrix):
+        self.init_op_weight_matrix = init_op_weight_matrix
+
+    # Not needed, but don't touch
+    def set_init_var_weight_matrix(self, init_var_weight_matrix):
+        self.init_var_weight_matrix = init_var_weight_matrix
+
+    # Not 100% tested
+    def make_y_multi_safe(self, old_y):
+        if isinstance(old_y, list):
+            new_y = np.array(old_y)
+            new_y.reshape([-1, self.n_dims_in_output, 1])
+        else:
+            new_y = old_y.copy()
+        if len(new_y.shape) == 1:
+            assert (self.n_dims_in_output == 1)
+            new_y = [[[y_value] for _ in range(self.n_dims_per_variable)] for y_value in new_y]
+            new_y = np.array(new_y)
+        elif len(new_y.shape) == 2:
+            assert (self.n_dims_in_output == 1)
+            new_y = [[y_value for _ in range(self.n_dims_per_variable)] for y_value in new_y]
+            new_y = np.array(new_y)
+        elif new_y.shape[1] < self.n_dims_per_variable:
+            assert (self.n_dims_in_output == 1)
+            new_y = [[y_value[0] for _ in range(self.n_dims_per_variable)] for y_value in new_y]
+            new_y = np.array(new_y)
+        return new_y
+
+    def get_simple_formula(self, digits=None):
+        full_formula = self.get_formula_string()
+        return DataUtils.simplify_formula(full_formula, digits=digits)
+
+    # todo: want total or mean square error?
+    def test(self, x, y=None):
+        test_dict = {self.data_x: x,
+                     self.init_op_weights: self.init_op_weight_matrix,
+                     self.init_var_weights: self.init_var_weight_matrix}
+        if y is not None:
+            test_dict[self.data_y] = y
+        return self.sess.run(self.total_error, feed_dict=test_dict)
+
+    # Runs train process a number of times on a limited number of train steps.
+    # Returns the best formula found during that experience.
+    # If init_ops is given, it will start off with ops initialized accordingly.
+    #   If it is None, then ops will be initialized randomly.
+    #   If it is 0, then ops will have no initialization.
+    # Same with init_vars.
+    def train(self, x, y=None, init_op_weight_matrix=None, init_var_weight_matrix=None,
+              test_x=None, test_y=None):
+        n_rounds = Settings.num_train_steps_in_repeat_mode
+
+        batch_size = min(Settings.max_training_batch_size, int(len(x) / 2))
+
+        train_set_size = len(x)
+        train_x = np.array(x, dtype=np.float32)
+
+        if self.mode in ["de"]:
+            y = [0 for _ in range(x.shape[0])]
+            if test_x is not None:
+                test_y = [0 for _ in range(test_x.shape[0])]
+        # elif self.mode == ["sr", "lr"]:
+        #     y = DataUtils.true_function(x)
+        #     if test_x is not None:
+        #         test_y = DataUtils.true_function(test_x)
+
+        train_y = self.make_y_multi_safe(y)
+
+        if test_y is not None:
+            test_y = self.make_y_multi_safe(test_y)
+
+        if init_op_weight_matrix is not None:
+            self.set_init_op_weight_matrix(init_op_weight_matrix)
+
+        if init_var_weight_matrix is not None:
+            self.set_init_var_weight_matrix(init_var_weight_matrix)
+
+        target_y = self.y_hat
+        show_gt = False
+
+        if Settings.show_output:
+            print("Starting actual training!")
+        start_time = time.time()
+        old_time = time.time()
+        time_spent_training = 0
+        time_getting_formulas = 0
+        time_getting_scores = 0
+        time_plotting = 0
+        other_time = 0
+
+        for i in range(1, n_rounds + 1):
+            mini_start_time = time.time()
+            train_batch_x, train_batch_y, valid_batch_x, valid_batch_y = DataUtils.get_samples(train_set_size,
+                                                                                               batch_size,
+                                                                                               train_x, train_y)
+            other_time += time.time() - mini_start_time
+
+            training_dict = {self.data_x: train_batch_x,
+                             self.data_y: train_batch_y,
+                             self.init_op_weights: self.init_op_weight_matrix,
+                             self.init_var_weights: self.init_var_weight_matrix}
+
+            valid_batch_dict = {self.data_x: valid_batch_x,
+                                self.data_y: valid_batch_y,
+                                self.init_op_weights: self.init_op_weight_matrix,
+                                self.init_var_weights: self.init_var_weight_matrix}
+
+            test_dict = {self.data_x: test_x, self.data_y: test_y,
+                         self.init_op_weights: self.init_op_weight_matrix,
+                         self.init_var_weights: self.init_var_weight_matrix}
+
+            """ Actual training happens here """
+            mini_start_time = time.time()
+            if i < n_rounds * Settings.t1_fraction:
+                self.sess.run(self.train_step_1, feed_dict=training_dict)
+            elif i < n_rounds * Settings.t2_fraction:
+                self.sess.run(self.train_step_2, feed_dict=training_dict)
+            else:
+                self.sess.run(self.train_step_3, feed_dict=training_dict)
+
+            time_spent_training += (time.time() - mini_start_time)
+
+            """ Save formulas, accuracy, etc. """
+            if (i % Settings.plot_frequency == 0 or i % Settings.output_freq == 0) and Settings.keep_logs:
+
+                # Save current formula to make list of all formulas seen
+                current_formula = "(Formula not saved)"
+                if Settings.save_all_formulas:
+                    mini_start_time = time.time()
+                    current_formula = self.get_simple_formula(digits=4)
+                    time_getting_formulas += (time.time() - mini_start_time)
+
+                    if current_formula not in self.seen_eqns:
+                        self.seen_eqns.append(current_formula)
+
+                # Get results from validation set.
+                mini_start_time = time.time()
+                [valid_acc, y_pr_v] = self.sess.run([self.total_error, target_y], feed_dict=valid_batch_dict)
+                # Get results from test set.
+                if test_x is not None:
+                    [test_acc, y_pr_test] = self.sess.run([self.total_error, target_y], feed_dict=test_dict)
+
+                y_gold_v = valid_batch_y.reshape([-1, self.n_dims_per_variable, 1])[0].tolist()
+                y_hat_v = y_pr_v.reshape([-1, self.n_dims_per_variable, 1]).tolist()
+
+                time_getting_scores += (time.time() - mini_start_time)
+
+                mini_start_time = time.time()
+                [valid_acc, g_pr_v] = self.sess.run([self.total_error, self.implicit_g], feed_dict=valid_batch_dict)
+                g_hat_val = g_pr_v.reshape([-1, self.n_dims_per_variable, 1]).tolist()
+                g_hat_1d_val = [y_value[0][0] for y_value in g_hat_val]
+                g_tru_1d_val = [y_value[0][0] for y_value in valid_batch_y]
+                g_hat_1d_test = None
+                g_tru_1d_test = None
+
+                [yp_v, ypp_v] = self.sess.run([self.y_hat_p1, self.y_hat_pp1], feed_dict=valid_batch_dict)
+                y_p1_v = yp_v.reshape([-1, self.n_dims_per_variable, 1]).tolist()
+                y_pp1_v = ypp_v.reshape([-1, self.n_dims_per_variable, 1]).tolist()
+
+                [yp2_v, ypp2_v] = self.sess.run([self.y_hat_p2, self.y_hat_pp2], feed_dict=valid_batch_dict)
+                y_p2_v = yp2_v.reshape([-1, self.n_dims_per_variable, 1]).tolist()
+                y_pp2_v = ypp2_v.reshape([-1, self.n_dims_per_variable, 1]).tolist()
+
+                time_getting_scores += (time.time() - mini_start_time)
+
+                if test_x is not None:
+                    mini_start_time = time.time()
+                    [test_acc, g_pr_test] = self.sess.run([self.total_error, self.implicit_g], feed_dict=test_dict)
+
+                    g_hat_test = g_pr_test.reshape([-1, self.n_dims_per_variable, 1]).tolist()
+                    g_hat_1d_test = [g_value[0][0] for g_value in g_hat_test]
+                    g_tru_1d_test = [g_value[0][0] for g_value in test_y]
+                    time_getting_scores += (time.time() - mini_start_time)
+
+                # Update best formula seen based on validation error.
+                if Settings.save_all_formulas:
+                    if valid_acc < self.best_accuracy_so_far:
+                        self.best_accuracy_so_far = valid_acc
+                        self.best_formula_so_far = current_formula
+                        self.best_iter = i
+
+                # We only can make plots using y values if y is 1d.
+                if self.n_dims_in_output == 1:
+
+                    mini_start_time = time.time()
+
+                    y_hat_1d_val = [y_value[0][0] for y_value in y_hat_v]
+                    y_tru_1d_val = [y_value[0][0] for y_value in valid_batch_y]
+                    y_hat_1d_test = None
+                    y_tru_1d_test = None
+
+                    if test_x is not None:
+                        y_hat_test = y_pr_test.reshape([-1, self.n_dims_per_variable, 1]).tolist()
+                        y_hat_1d_test = [y_value[0][0] for y_value in y_hat_test]
+                        y_tru_1d_test = [y_value[0][0] for y_value in test_y]
+                    other_time += (time.time() - mini_start_time)
+
+                    if self.mode in ["sr", "lr"]:
+                        # Plot predicted y value against actual y value.
+                        mini_start_time = time.time()
+                        DataUtils.plot_predicted_vs_actual(y_hat_1d_val, y_tru_1d_val,
+                                                           y_hat_1d_test, y_tru_1d_test,
+                                                           self.name,
+                                                           set_name="Iteration {}".format(i))
+                        time_plotting += (time.time() - mini_start_time)
+
+                    # DataUtils.plot_2d_curve(x_1d_val, y_tru_1d_val, y_hat_1d_val, None, None, None)
+
+                    # If x is also 1d, we can plot the function itself.
+                    if self.n_input_variables == 1:
+
+                        # Plot the actual function we learned.
+                        mini_start_time = time.time()
+                        x_1d_val = [x_value[0][0] for x_value in valid_batch_x]
+                        x_1d_test = None
+                        if test_x is not None:
+                            x_1d_test = [x_value[0][0] for x_value in test_x]
+                        other_time += (time.time() - mini_start_time)
+
+                        mini_start_time = time.time()
+                        DataUtils.plot_1d_curve(x_1d_val, y_tru_1d_val, y_hat_1d_val,
+                                                 x_1d_test, y_tru_1d_test, y_hat_1d_test,
+                                                 file_suffix="_y",
+                                                 title="Learned function: Iteration {}".format(i),
+                                                 show_ground_truth=show_gt)
+                        time_plotting += (time.time() - mini_start_time)
+
+                        # Plot the g output values, in implicit case
+                        if test_x is not None:
+                            mini_start_time = time.time()
+                            DataUtils.plot_1d_curve(x_1d_val, g_tru_1d_val, g_hat_1d_val,
+                                                     x_1d_test, g_tru_1d_test, g_hat_1d_test,
+                                                     file_suffix="_g",
+                                                     title="Output of g: Iteration {}".format(i))
+
+                            time_plotting += time.time() - mini_start_time
+
+                    elif self.n_input_variables == 2:
+                        # Plot the actual function we learned.
+                        mini_start_time = time.time()
+
+                        plot2d_x1 = np.arange(Settings.test_scope[0], Settings.test_scope[1], 0.1)
+                        plot2d_x2 = np.arange(Settings.test_scope[0], Settings.test_scope[1], 0.1)
+
+                        plot2d_x1_m, plot2d_x2_m = np.meshgrid(plot2d_x1, plot2d_x2)
+                        plot2d_x1 = np.reshape(plot2d_x1_m, [-1, 1, 1])
+                        plot2d_x2 = np.reshape(plot2d_x2_m, [-1, 1, 1])
+                        plot2d_x1x2 = np.concatenate([plot2d_x1, plot2d_x2], axis=-1)
+                        [plot2d_y, plot2d_g] = self.sess.run([target_y, self.implicit_g],
+                                                             feed_dict={self.data_x: plot2d_x1x2,
+                                                                        self.init_op_weights: self.init_op_weight_matrix,
+                                                                        self.init_var_weights: self.init_var_weight_matrix})
+
+                        if self.mode == "sr":
+                            plot2d_g = DataUtils.true_function(plot2d_x1x2)
+
+                        plot2d_y_m = np.reshape(plot2d_y, plot2d_x1_m.shape)
+                        plot2d_g_m = np.reshape(plot2d_g, plot2d_x1_m.shape)
+
+                        DataUtils.plot_2d_curve(plot2d_x1_m, plot2d_x2_m, plot2d_y_m, plot2d_g_m)
+
+                        time_plotting += (time.time() - mini_start_time)
+
+                if Settings.keep_logs:
+                    mini_start_time = time.time()
+                    self.train_accuracy_log.append(self.test(train_x, train_y))
+                    # self.valid_accuracy_log.append(valid_acc)
+                    self.valid_accuracy_log.append(self.test(valid_batch_x, valid_batch_y))
+
+                    # self.log_iters.append(i)
+                    if len(self.log_iters) == 0:
+                        self.log_iters.append(i)
+                    else:
+                        self.log_iters.append(self.log_iters[-1] + Settings.plot_frequency)
+
+                    accuracies_to_plot = [self.train_accuracy_log,
+                                          self.valid_accuracy_log]
+                    accuracy_type_names = ["Training Error", "Validation Error"]
+                    if test_x is not None:
+                        self.test_accuracy_log.append(test_acc)
+                        accuracies_to_plot.append(self.test_accuracy_log)
+                        accuracy_type_names.append("Test Error")
+
+                    time_getting_scores += (time.time() - mini_start_time)
+
+                    mini_start_time = time.time()
+                    DataUtils.plot_accuracy_over_time(self.log_iters, accuracies_to_plot, accuracy_type_names)
+
+                    time_plotting += time.time() - mini_start_time
+
+            if i % Settings.output_freq == 0 and Settings.show_output:
+
+                if not Settings.keep_logs:
+                    # Get results from validation set.
+                    mini_start_time = time.time()
+                    [valid_acc, y_pr_v] = self.sess.run([self.total_error, target_y], feed_dict=valid_batch_dict)
+                    # Get results from test set.
+                    if test_x is not None:
+                        [test_acc, y_pr_test] = self.sess.run([self.total_error, target_y], feed_dict=test_dict)
+                    y_hat_v = y_pr_v.reshape([-1, self.n_dims_per_variable, 1]).tolist()
+
+                    g_pr_v = self.sess.run(self.implicit_g, feed_dict=valid_batch_dict)
+                    g_hat_val = g_pr_v.reshape([-1, self.n_dims_per_variable, 1]).tolist()
+
+                    [yp1_v, ypp1_v] = self.sess.run([self.y_hat_p1, self.y_hat_pp1],
+                                                     feed_dict={self.data_x: valid_batch_x,
+                                                                self.init_op_weights: self.init_op_weight_matrix,
+                                                                self.init_var_weights: self.init_var_weight_matrix})
+                    y_p1_v = yp1_v.reshape([-1, self.n_dims_per_variable, 1]).tolist()
+                    y_pp1_v = ypp1_v.reshape([-1, self.n_dims_per_variable, 1]).tolist()
+
+                    [yp2_v, ypp2_v] = self.sess.run([self.y_hat_p2, self.y_hat_pp2],
+                                                    feed_dict={self.data_x: valid_batch_x,
+                                                               self.init_op_weights: self.init_op_weight_matrix,
+                                                               self.init_var_weights: self.init_var_weight_matrix})
+                    y_p2_v = yp2_v.reshape([-1, self.n_dims_per_variable, 1]).tolist()
+                    y_pp2_v = ypp2_v.reshape([-1, self.n_dims_per_variable, 1]).tolist()
+
+                    time_getting_scores += (time.time() - mini_start_time)
+
+                print()
+
+                print('Iteration {}:'.format(i))
+
+                mini_start_time = time.time()
+                formula_as_string = self.get_formula_string(digits=4)
+                dotdot = ""
+                if len(formula_as_string) > Settings.max_formula_output_length:
+                    dotdot = " ..."
+                print("# Current Model: {}{}".format(formula_as_string[:Settings.max_formula_output_length], dotdot))
+
+                simple_formula = self.get_simple_formula(digits=4)
+                dotdot = ""
+                if len(simple_formula) > Settings.max_formula_output_length:
+                    dotdot = " ..."
+                print("# AKA:           {}{}".format(simple_formula[:Settings.max_formula_output_length], dotdot))
+
+                minimal_eqn = self.get_minimal_formula_string()
+                print("# Simple:        {}".format(minimal_eqn))
+                if minimal_eqn not in self.seen_minimal_eqns:
+                    self.seen_minimal_eqns.append(minimal_eqn)
+
+                if "**" in simple_formula:
+                    print("(Has a power)")
+
+                time_getting_formulas += (time.time() - mini_start_time)
+
+                print(
+                    "  Length:           {} ({})".format(len(formula_as_string), len("{}".format(simple_formula))))
+
+                print("  Train batch size: {}".format(train_batch_x.shape))
+                print("  Valid batch size: {}".format(valid_batch_x.shape))
+                print("  # Mnml eqns seen: {}".format(len(self.seen_minimal_eqns)))
+
+                iters_per_min = Settings.output_freq * 60 / (time.time() - old_time)
+                print('  Iters per minute: {:.2f}'.format(iters_per_min))
+                total_time = time.time() - start_time
+                print('  Time so far:      {:.2f} minutes'.format(total_time / 60.0))
+                print('    ({:.1%} training, {:.1%} scoring, {:.1%} formulas)'.format(
+                    time_spent_training / total_time,
+                    time_getting_scores / total_time,
+                    time_getting_formulas / total_time))
+                print('    ({:.1%} plotting, {:.1%} other)'.format(time_plotting / total_time,
+                                                                   other_time / total_time))
+                print('  Est. time left:   {:.2f} minutes'.format((n_rounds - i) / iters_per_min))
+
+                print('Error values:')
+                mini_start_time = time.time()
+
+                curr_errs = self.sess.run([self.g_error, self.ivp_error,
+                                           self.spike_error, self.total_error],
+                                          feed_dict=valid_batch_dict)
+                if self.mode == "de":
+                    print('   g-err Valid: {}'.format(curr_errs[0]))
+                print('   IVP Valid:   {}'.format(curr_errs[1]))
+                if Settings.non_const:
+                    print('   Spike err:   {}'.format(curr_errs[2]))
+                print('   Tot. Val.:   {}'.format(curr_errs[3]))
+                if np.abs(curr_errs[3] - (curr_errs[0] + self.ivp_lambda * curr_errs[1] + curr_errs[2])) > 1e-4:
+                    print("Something is wrong.")
+
+
+                # Hope we don't get nans, but break out if we do.
+                nans = np.isnan(curr_errs[0])
+                if nans:
+                    break
+
+                if test_x is not None:
+                    print('   Tot. Test:   {}'.format(test_acc))
+
+                time_getting_scores += (time.time() - mini_start_time)
+
+                print('Performance on sample validation data:')
+
+                print_str = ""
+                for feature_i in range(Settings.num_features):
+                    print_str += "{}\t\t".format(self.var_names[feature_i])
+                print_str += "|\t"
+                # print_str += "y_tru\t"
+                if self.mode == "de":
+                    print_str += "g_hat\t"
+                elif self.mode == "sr" or self.mode == "lr":
+                    print_str += "y_tru\t"
+                print_str += "y_hat\t"
+
+                if self.mode == "de":
+                    print_str += "y_p1\t"
+                    print_str += "y_pp1\t"
+                    print_str += "y_p2\t"
+                    print_str += "y_pp2\t"
+                print(print_str)
+                line_len = len(print_str) + 16
+                print("=" * line_len)
+
+                var_range = range(self.n_input_variables)
+                num_pts_to_show = 5
+                if self.mode in ["sr", "lr"]:
+                    y_tru_v = valid_batch_y[:num_pts_to_show, :, :]
+                    # y_tru_v = DataUtils.predict_from_formula(Settings.true_eqn, valid_batch_x[:num_pts_to_show, :, :])
+
+                for datapoint_i in range(min(valid_batch_x.shape[0], num_pts_to_show)):
+                    comps_to_show = range(self.n_dims_per_variable)
+                    if self.n_dims_per_variable > 9:
+                        comps_to_show = [0, 1, 2, -1]
+                    for component_j in comps_to_show:
+                        if component_j == -1:
+                            print(" ... ")
+                        print_str = ""
+                        for var_k in var_range:
+                            x_ijk = valid_batch_x[datapoint_i, component_j, var_k]
+                            print_str += "{:.3f}\t".format(x_ijk)
+                        print_str += "|\t"
+                        # print_str += "{:.3f}\t".format(valid_batch_y[datapoint_i, component_j, 0])
+
+                        if self.mode == "de":
+                            print_str += "{:.3f}\t".format(g_hat_val[datapoint_i][component_j][0])
+                        elif self.mode in ["sr", "lr"]:
+                            print_str += "{:.3f}\t".format(y_tru_v[datapoint_i][0][0])
+
+                        print_str += "{:.3f}\t".format(y_hat_v[datapoint_i][component_j][0])
+
+                        if self.mode == "de":
+                            print_str += "{:.3f}\t".format(y_p1_v[datapoint_i][component_j][0])
+                            print_str += "{:.3f}\t".format(y_pp1_v[datapoint_i][component_j][0])
+                            print_str += "{:.3f}\t".format(y_p2_v[datapoint_i][component_j][0])
+                            print_str += "{:.3f}\t".format(y_pp2_v[datapoint_i][component_j][0])
+
+                        print(print_str)
+                    print("-" * line_len)
+                print()
+
+                old_time = time.time()
+
+        if Settings.show_output:
+            print('Finished training at {:%H:%M:%S}.\n'.format(datetime.datetime.now()))
+            end_time = time.time()
+            total_time = end_time - start_time
+
+            print('Took {:.2f} seconds to finish.'.format(total_time))
+            print('    ({:.1%} training, {:.1%} scoring, {:.1%} formulas)'.format(time_spent_training / total_time,
+                                                                                  time_getting_scores / total_time,
+                                                                                  time_getting_formulas / total_time))
+            print('    ({:.1%} plotting, {:.1%} other)'.format(time_plotting / total_time,
+                                                               other_time / total_time))
+            print('Average of {:.2f} training steps per minute.'.format(
+                60 * n_rounds / total_time))
+            print('Average of {:.2f} minutes per 10000 training steps.'.format(
+                10000 * total_time / (60 * n_rounds)))
+
+            print()
+            if Settings.save_all_formulas:
+                print("Best formula had accuracy {:.3f} and was seen at iteration {}:".format(
+                    self.best_accuracy_so_far,
+                    self.best_iter))
+                print("{}".format(self.best_formula_so_far)[:1000])
+            else:
+                final_acc = self.sess.run(self.total_error, feed_dict={self.data_x: train_x,
+                                                                             self.data_y: train_y,
+                                                                             self.init_op_weights: self.init_op_weight_matrix,
+                                                                             self.init_var_weights: self.init_var_weight_matrix})
+                print("Final formula had accuracy {:.3f}:".format(final_acc))
+
+                print("{}".format(self.get_simple_formula(digits=4))[:1000])
+            print()
+
+        return self.get_simple_formula(digits=4)
+
+    def repeat_train(self, x, y=None,
+                     num_repeats=Settings.num_train_repeat_processes,
+                     test_x=None, test_y=None,
+                     verbose=True):
+
+        # we still reduce train set size if only 1 repeat
+        train_set_size = int(len(x) * Settings.quick_train_fraction + 0.1)
+
+        x = np.array(x)
+
+        if y is not None:
+            y = np.array(y)
+
+        sample = np.random.choice(range(x.shape[0]), size=train_set_size, replace=False)
+        train_x = x[sample][:]
+        if y is not None:
+            train_y = y[sample]
+
+        out_sample = [aaa for aaa in range(x.shape[0]) if aaa not in sample]
+        valid_x = x[out_sample][:]
+        if y is not None:
+            valid_y = y[out_sample]
+            valid_y = self.make_y_multi_safe(valid_y)
+
+        best_formula = ""
+        best_iter = 0
+        best_validation = 999999
+        best_err = 999999
+        old_time = time.time()
+
+        if verbose:
+            print("Beginning {} repeat sessions of {} iterations each.".format(num_repeats,
+                                                                               Settings.num_train_steps_in_repeat_mode))
+            print()
+            start_time = time.time()
+            old_time = start_time
+
+        for train_iter in range(1, 1 + num_repeats):
+            if verbose:
+                print("Repeated train session {} of {}.".format(train_iter, num_repeats))
+
+
+            self.soft_reset()
+
+            self.set_init_op_weight_matrix(choices_to_init_weight_matrix(Settings.initialize_ops,
+                                                                         self.function_set))
+            self.set_init_var_weight_matrix(choices_to_init_weight_matrix(np.zeros([2 ** self.n_tree_layers]),
+                                                                          self.var_names))
+
+
+            self.train(train_x, train_y, test_x=test_x, test_y=test_y)
+
+            valid_err = self.test(valid_x, valid_y)
+
+            current_time = time.time()
+            if verbose:
+                # print(self.get_simple_formula())
+                print("Attained validation error: {:.5f}".format(valid_err))
+
+            if valid_err < best_validation:
+                best_validation = valid_err
+                best_formula = self.get_simple_formula()
+                best_iter = train_iter
+                if test_x is not None:
+                    safe_test_y = self.make_y_multi_safe(test_y)
+                    best_err = self.test(test_x, safe_test_y)
+                else:
+                    best_err = valid_err
+                if verbose:
+                    print(">>> New best model!")
+                    print(best_formula)
+
+            if verbose:
+                iters_per_minute = 60.0 / (current_time - old_time)
+                print("Took {:.2f} minutes.".format((current_time - old_time) / 60))
+                print("Est. {:.2f} minutes remaining.".format((num_repeats - train_iter) / iters_per_minute))
+                print()
+                old_time = current_time
+
+        if verbose:
+            print("Total time for repeat process: {:.2f} minutes.".format((time.time() - start_time) / 60))
+
+        return best_formula, best_iter, best_err

gp_model.py

@@ -0,0 +1,163 @@
+import time
+
+import numpy as np
+from gplearn.genetic import SymbolicRegressor
+from sklearn.utils.validation import column_or_1d
+
+import Settings as settings
+from DataUtils import make_y_multi_safe
+
+pop_size = 5000
+generations = 20
+p_crossover = 0.7
+warm_start = False
+
+
+class Genetic_Model:
+    def __init__(self):
+        self.name = "Genetic Model"
+        self.short_name = "GP"
+        self.function_set = settings.function_set.copy()
+        if "id" in self.function_set:
+            self.function_set.remove("id")
+
+        self.est_gp = SymbolicRegressor(population_size=pop_size,
+                                        generations=generations, stopping_criteria=0.01,  # 20 gen
+                                        p_crossover=p_crossover, p_subtree_mutation=0.1,
+                                        p_hoist_mutation=0.05, p_point_mutation=0.1,
+                                        warm_start=warm_start,
+                                        max_samples=0.9, verbose=False,
+                                        parsimony_coefficient=0.01,
+                                        function_set=self.function_set)
+
+    def reset(self):
+        del self.est_gp
+        self.est_gp = SymbolicRegressor(population_size=pop_size,
+                                        generations=generations, stopping_criteria=0.01,  # 20 gen
+                                        p_crossover=p_crossover, p_subtree_mutation=0.1,
+                                        p_hoist_mutation=0.05, p_point_mutation=0.1,
+                                        warm_start=warm_start,
+                                        max_samples=0.9, verbose=False,
+                                        parsimony_coefficient=0.01,
+                                        function_set=self.function_set)
+
+    def soft_reset(self):
+        del self.est_gp
+        self.est_gp = SymbolicRegressor(population_size=pop_size,
+                                        generations=generations, stopping_criteria=0.01,  # 20 gen
+                                        p_crossover=p_crossover, p_subtree_mutation=0.1,
+                                        p_hoist_mutation=0.05, p_point_mutation=0.1,
+                                        warm_start=warm_start,
+                                        max_samples=0.9, verbose=False,
+                                        parsimony_coefficient=0.01,
+                                        function_set=self.function_set)
+
+    def predict(self, X):
+        return self.est_gp.predict(X)
+
+    def get_formula(self):
+        return self.est_gp._program
+
+    def get_simple_formula(self, digits=None):
+        return self.get_formula()
+
+    def get_big_formula(self):
+        formula_string = str(self.get_formula())
+        nested_list_string = formula_string.replace("sqrt(", "[\'sqrt\', ")
+        nested_list_string = nested_list_string.replace("add(", "[\'+\', ")
+        nested_list_string = nested_list_string.replace("mul(", "[\'*\', ")
+        nested_list_string = nested_list_string.replace("sub(", "[\'-\', ")
+        nested_list_string = nested_list_string.replace("sin(", "[\'sin\', ")
+        nested_list_string = nested_list_string.replace(")", "]")
+        nested_list_string = nested_list_string.replace("X", "Y")
+
+        retval = ""
+        currently_digits = False
+        current_number = ""
+        for current_char in nested_list_string:
+            if current_char == 'Y':
+                retval += "\'x"
+                currently_digits = True
+                current_number = ""
+            elif currently_digits:
+                if current_char.isdigit():
+                    # retval += "{}".format(current_char)
+                    current_number += "{}".format(current_char)
+                else:
+                    currently_digits = False
+                    retval += "{}".format(int(current_number) + 1)
+                    retval += "\'{}".format(current_char)
+            else:
+                retval += "{}".format(current_char)
+
+        if "Y" in retval:
+            print("ERROR: formula still contains a Y...")
+            print("   formula string: {}\n   nested list string: {}".format(formula_string, nested_list_string))
+
+        return eval(retval)
+
+    def train(self, X, Y):
+        X = np.reshape(X, [X.shape[0], -1])
+        Y = np.reshape(Y, [-1, 1])
+        Y = column_or_1d(Y)
+        self.est_gp.fit(X, Y)
+        return None
+
+    # Does not repeat train. Sorry.
+    def repeat_train(self, x, y, test_x=None, test_y=None,
+                     num_repeats=settings.num_train_repeat_processes,
+                     num_steps_to_train=settings.num_train_steps_in_repeat_mode,
+                     verbose=True):
+        train_set_size = int(len(x) * settings.quick_train_fraction + 0.1)
+        x = np.array(x)
+        y = np.reshape(np.array(y), [-1, ])
+        sample = np.random.choice(range(x.shape[0]), size=train_set_size, replace=False)
+        out_sample = [yyy for yyy in range(x.shape[0]) if yyy not in sample]
+
+        train_x = x[sample][:]
+        train_y = y[sample][:]
+        valid_x = x[out_sample][:]
+        valid_y = y[out_sample][:]
+
+        old_time = time.time()
+
+        if verbose:
+            print("Beginning {} repeat sessions of {} iterations each.".format(num_repeats,
+                                                                               settings.num_train_steps_in_repeat_mode))
+            print()
+            start_time = time.time()
+            old_time = start_time
+
+        self.soft_reset()
+        self.train(train_x, train_y)
+
+        current_time = time.time()
+        if verbose:
+            # print(self.get_simple_formula())
+            print("Attained validation error: {:.5f}".format(valid_err))
+
+        best_formula = self.get_simple_formula()
+        if test_x is not None:
+            safe_test_y = make_y_multi_safe(test_y)
+            best_err = self.test(test_x, safe_test_y)
+        else:
+            best_err = self.test(valid_x, valid_y)
+
+        if verbose:
+            iters_per_minute = 60.0 / (current_time - old_time)
+            print("Took {:.2f} minutes.".format((current_time - old_time) / 60))
+            print("Est. {:.2f} minutes remaining.".format((num_repeats - train_iter) / iters_per_minute))
+            print()
+
+        return best_formula, 0, best_err
+
+    # Mean square error
+    def test(self, x, y):
+        x = np.reshape(x, [x.shape[0], -1])
+        y_hat = np.reshape(self.est_gp.predict(x), [1, -1])[0]
+        y_gold = np.reshape(y, [1, -1])[0]
+        our_sum = 0
+        for i in range(len(y_gold)):
+            our_sum += (y_hat[i] - y_gold[i]) ** 2
+
+        return our_sum / len(y_gold)