Hi. Based on this code to test various models, i found the average of the following two to have the best accuracy. Including only the necessary code, write the python to create an ensemble of the two (average of their predictions) and code to predict values #65, 66, 67.

Checking: No Outlier Removal, Log Transformation, Degree 1, Linear Regression
	Predicted  : 302d7423bc9ff0000 in Puzzle 66
	Real Value:  2832ed74f2b5e35ee ✔️

Checking: No Outlier Removal, Log Transformation, Degree 1, Ridge Regression (alpha=1.0)
	Predicted  : 26529912fc1c24000 in Puzzle 66
	Real Value:  2832ed74f2b5e35ee ✔️


Code: import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression, Ridge, Lasso
from sklearn.preprocessing import PolynomialFeatures, StandardScaler
from sklearn.model_selection import LeaveOneOut
from sklearn.metrics import mean_squared_error
from sklearn.pipeline import make_pipeline
import joblib
import warnings
from sklearn.exceptions import ConvergenceWarning

# Assume 'data' is your DataFrame containing the columns '#' and 'Target'
x = data['#'] - 1  # x values (# = x + 1)
y = data['Target']  # Target values

# Actual value for comparison
actual_value = 46346217550346335726
x_to_predict = 65  # The x value you want to predict for

# Initialize variables to track the best model
smallest_distance = float('inf')
best_prediction = None
best_model_info = ""
best_pipeline = None  # To store the best pipeline

# Possible preprocessing techniques
transformations = [np.log2, np.sqrt, lambda y: y]  # Log, square root, and no transformation
transformation_names = ['Log Transformation', 'Square Root Transformation', 'No Transformation']

# Possible regression models with adjusted parameters
models = [
    ('Linear Regression', LinearRegression()),
    ('Ridge Regression (alpha=1.0)', Ridge(alpha=1.0, max_iter=100000, tol=1e-3)),
    ('Ridge Regression (alpha=10.0)', Ridge(alpha=10.0, max_iter=100000, tol=1e-3)),
    ('Lasso Regression (alpha=0.1)', Lasso(alpha=0.1, max_iter=1000000, tol=1e-3)),
    ('Lasso Regression (alpha=1.0)', Lasso(alpha=1.0, max_iter=1000000, tol=1e-3)),
    ('Lasso Regression (alpha=10.0)', Lasso(alpha=10.0, max_iter=1000000, tol=1e-3))
]

inrange = []
# Polynomial degrees to try
degrees = [1, 2, 3, 4, 5, 6]

# Outlier removal methods
def remove_outliers_zscore(x, y):
    z_scores = np.abs((y - y.mean()) / y.std())
    mask = z_scores < 3
    return x[mask], y[mask]

def remove_outliers_iqr(x, y):
    Q1 = np.percentile(y, 10)
    Q3 = np.percentile(y, 90)
    IQR = Q3 - Q1
    mask = (y >= Q1 - 1.5 * IQR) & (y <= Q3 + 1.5 * IQR)
    return x[mask], y[mask]

outlier_methods = [
    ('No Outlier Removal', lambda x, y: (x, y)),
    ('Z-Score Outlier Removal', remove_outliers_zscore),
    ('IQR Outlier Removal', remove_outliers_iqr)
]

# Suppress convergence warnings for cleaner output
warnings.filterwarnings("ignore", category=ConvergenceWarning)
best_model_params = {}

# Begin automated testing
for outlier_name, outlier_func in outlier_methods:
    # Prepare data
    x = data['#'] - 1  # x values
    y = data['Target']  # Target values

    # Apply outlier removal
    x_clean, y_clean = outlier_func(x, y)

    X = x_clean.values.reshape(-1, 1)
    shape = X.shape
    y_values = y_clean.values

    for transform_name, transform_func in zip(transformation_names, transformations):
        # Transform y
        if transform_name != 'No Transformation':
            y_transformed = transform_func(y_values)
        else:
            y_transformed = y_values.copy()

        # Scale y_transformed if necessary (e.g., for Lasso and Ridge stability)
        # Applying StandardScaler to y_transformed
        y_mean = y_transformed.mean()
        y_std = y_transformed.std()
        y_transformed_scaled = (y_transformed - y_mean) / y_std

        for degree in degrees:
            for model_name, model in models:
                # Create polynomial features if degree > 1
                if degree > 1:
                    pipeline = make_pipeline(
                        PolynomialFeatures(degree, include_bias=False),
                        StandardScaler(),
                        model
                    )
                else:
                    pipeline = make_pipeline(
                        StandardScaler(),
                        model
                    )

                # Leave-One-Out Cross-Validation
                loo = LeaveOneOut()
                mse_list = []

                for train_index, test_index in loo.split(X):
                    X_train, X_test = X[train_index], X[test_index]
                    y_train, y_test = y_transformed_scaled[train_index], y_transformed_scaled[test_index]

                    # Fit model
                    pipeline.fit(X_train, y_train)

                    # Predict
                    y_pred_transformed_scaled = pipeline.predict(X_test)

                    # Inverse scaling
                    y_pred_transformed = y_pred_transformed_scaled * y_std + y_mean
                    y_test_transformed = y_test * y_std + y_mean

                    # Inverse transform prediction
                    if transform_name == 'Log Transformation':
                        y_pred = 2 ** y_pred_transformed
                        y_test_actual = 2 ** y_test_transformed
                    elif transform_name == 'Square Root Transformation':
                        y_pred = y_pred_transformed ** 2
                        y_test_actual = y_test_transformed ** 2
                    else:
                        y_pred = y_pred_transformed
                        y_test_actual = y_test_transformed

                    # Calculate MSE
                    mse = mean_squared_error(y_test_actual, y_pred)
                    mse_list.append(mse)

                # Average MSE
                avg_mse = np.mean(mse_list)

                # Fit model on all data
                pipeline.fit(X, y_transformed_scaled)

                # Predict for x = x_to_predict
                x_new = np.array([[x_to_predict]])
                y_pred_transformed_scaled = pipeline.predict(x_new)
                y_pred_transformed = y_pred_transformed_scaled * y_std + y_mean

                # Inverse transform prediction
                if transform_name == 'Log Transformation':
                    predicted_target = 2 ** y_pred_transformed[0]
                elif transform_name == 'Square Root Transformation':
                    predicted_target = y_pred_transformed[0] ** 2
                else:
                    predicted_target = y_pred_transformed[0]

                # Calculate distance from actual
                distance = abs(predicted_target - actual_value)

                # Check if this is the best prediction so far
                is_best = False
                if distance < smallest_distance:
                    smallest_distance = distance
                    best_prediction = predicted_target
                    try:
                      X_poly = pipeline.named_steps['polynomialfeatures'].transform(X).shape
                      best_model_info = f"{outlier_name} + {transform_name} + Degree {degree} + {model_name} + shape_before {shape} + shapeAfter = {X_poly}"
                    except:
                      pass
                    is_best = True
                    best_pipeline = pipeline  # Save the best pipeline

                    # Save transformation parameters
                    best_model_params = {
                        'y_mean': y_mean,
                        'y_std': y_std,
                        'transform_name': transform_name
                    }


                # Print output
                pzle = puzzle(predicted_target)
                if pzle == x_to_predict+1:
                  inrange.append(predicted_target)
                  print(f"Checking: {outlier_name}, {transform_name}, Degree {degree}, {model_name}")
                  print(f"\tPredicted  : {hex(int(predicted_target))[2:]} in Puzzle {pzle}")
                  print(f"\tReal Value:  {hex(actual_value)[2:]} {'✔️' if is_best else ''}\n")

print("Best model found:")
print(best_model_info)
print(f"Predicted Target for x={x_to_predict}: {best_prediction:.0f} (hex: {hex(int(best_prediction))[2:]})")
print(f"Actual value: {actual_value} (hex: {hex(int(actual_value))[2:]})")
print(f"Smallest distance: {smallest_distance:.0f} (hex: {hex(int(smallest_distance))[2:]})")

#print average of inrange as a hex
print(f"Average of inrange: {hex(int(np.mean(inrange)))[2:]} Puzzle ")


# Save the best model and parameters
joblib.dump({'model': best_pipeline, 'params': best_model_params}, 'best_model.pkl')