Improve this reverse proxy script, and add distributed roleplay learning, Fine-tuning with Evolutionary Strategies (ES), and Custom Reward Functions Based on Proxy Metrics (Saved in a config folder):


```
import g4f
import asyncio
import argparse
from flask import Flask, request, jsonify, Response, stream_with_context
from flask_cors import CORS
from pyngrok import ngrok
import random
import json
import time
import os
import numpy as np
from langchain_community.llms import OpenAI
from langchain.chains import ConversationChain
from langchain.memory import ConversationBufferMemory
from langchain.prompts import PromptTemplate
from stable_baselines3 import PPO
from stable_baselines3.common.vec_env import DummyVecEnv
from gym import spaces

app = Flask(__name__)
CORS(app)

# Available providers
PROVIDERS = [
    g4f.Provider.FreeChatgpt,
]

# Default configuration
DEFAULT_CONFIG = {
    'provider': random.choice(PROVIDERS),
    'model': 'chatglm3-6B',
    'max_tokens': 1400,
    'temperature': 0.9,
    'top_p': 1.0,
    'context_size': 100000,
    'system_prompt': (
        "You are a lively, imaginative conversational AI. You respond in an immersive roleplay style, "
        "using *actions* to depict behaviors and emotions, and you speak with a creative flair, often using storytelling elements. "
        "Ensure responses are helpful and accurate, maintaining the roleplay theme while delivering information."
    )
}

# Global configuration (can be updated dynamically)
config = {}

# LangChain setup
llm = OpenAI(temperature=0.9)
conversation = ConversationChain(
    llm=llm, 
    memory=ConversationBufferMemory(),
    prompt=PromptTemplate(
        input_variables=["history", "input"],
        template="{history}\nHuman: {input}\nAI:"
    )
)

# PPO Environment
class ChatEnvironment(gym.Env):
    def __init__(self):
        super(ChatEnvironment, self).__init__()
        self.action_space = spaces.Box(low=-1, high=1, shape=(10,), dtype=np.float32)
        self.observation_space = spaces.Box(low=-np.inf, high=np.inf, shape=(100,), dtype=np.float32)

    def step(self, action):
        # Implement the step function
        observation = np.random.rand(100)  # Example observation
        reward = np.random.rand()  # Example reward
        done = False
        info = {}
        return observation, reward, done, info

    def reset(self):
        return np.random.rand(100)  # Example initial observation

# Initialize PPO model
env = DummyVecEnv([lambda: ChatEnvironment()])
ppo_model = PPO("MlpPolicy", env, verbose=1)

def get_config_with_defaults(request_config):
    """Merge request-specific config with default values."""
    final_config = DEFAULT_CONFIG.copy()
    final_config.update(request_config)
    return final_config

@app.route('/v1/chat/completions', methods=['POST'])
def chat_completions():
    """Handles chat completion requests with structured, clean, and refined responses."""
    data = request.json
    messages = data.get('messages', [])
    request_config = data.get('config', {})

    if not messages:
        return jsonify({'error': 'No messages provided'}), 400

    final_config = get_config_with_defaults(request_config)
    system_prompt = data.get('system', final_config['system_prompt'])
    stream = data.get('stream', False)

    # Prepend system message to maintain the roleplay theme
    messages.insert(0, {'role': 'system', 'content': system_prompt})

    try:
        if stream:
            return Response(stream_with_context(stream_response(messages, final_config)), content_type='text/event-stream')
        else:
            # First refinement - checking the incoming request for consistency
            refined_messages = refine_messages(messages)

            # Use LangChain for response generation
            user_input = refined_messages[-1]['content']
            response = conversation.predict(input=user_input)

            # Apply PPO model to refine the response
            ppo_action, _ = ppo_model.predict(np.random.rand(100))  # Example observation
            response = apply_ppo_action(response, ppo_action)

            # Double refinement - polishing the response content
            refined_response = refine_response(response)
            
            # Truncate the response if it exceeds length
            truncated_response = refined_response[:1000] + '...' if len(refined_response) > 1000 else refined_response

            return jsonify({
                'choices': [{'message': {'content': truncated_response}}],
                'usage': {'total_tokens': len(truncated_response.split())}
            })
    except Exception as e:
        return jsonify({'error': str(e)}), 500

def apply_ppo_action(response, action):
    # Implement how PPO action modifies the response
    # This is a placeholder implementation
    return response

def refine_messages(messages):
    """Performs validation and refinement of incoming messages for coherence."""
    refined = []
    for message in messages:
        # Simple validation and trimming for potential malformed input
        if 'content' in message and len(message['content']) > 0:
            refined.append({
                'role': message['role'],
                'content': message['content'].strip()
            })
    return refined

def refine_response(response):
    """Refines the generated response by filtering and restructuring it."""
    # In this case, let's assume response refinement could involve filtering certain phrases
    return response.replace("generic", "creative").replace("boring", "engaging")

def stream_response(messages, config):
    """Yields chunks of response in streaming mode, with roleplay flavor added."""
    try:
        user_input = messages[-1]['content']
        response = conversation.predict(input=user_input)
        
        # Simulate streaming by splitting the response into chunks
        chunks = [response[i:i+10] for i in range(0, len(response), 10)]
        
        for chunk in chunks:
            # Refine chunk and maintain immersive flow with structured responses
            refined_chunk = refine_response(chunk)
            yield f"data: {json.dumps({'choices': [{'delta': {'content': refined_chunk}}]})}\n\n"
            time.sleep(0.1)  # Simulate streaming delay with a lively pause
    except Exception as e:
        yield f"data: {json.dumps({'error': str(e)})}\n\n"
    finally:
        yield "data: [DONE]\n\n"

@app.route('/config', methods=['GET', 'POST'])
def handle_config():
    """Handles configuration updates and retrievals."""
    global config
    if request.method == 'POST':
        new_config = request.json
        config.update(new_config)
        return jsonify({'message': 'Configuration updated', 'config': config})
    else:
        return jsonify(config)

@app.route('/providers', methods=['GET'])
def list_providers():
    """Lists available providers."""
    return jsonify({'providers': [provider.__name__ for provider in PROVIDERS]})

def start_ngrok(port):
    """Starts ngrok tunnel."""
    public_url = ngrok.connect(port).public_url
    print(f" * ngrok tunnel '{public_url}' -> 'http://localhost:{port}'")
    return public_url

async def check_provider_status():
    """Periodically checks the status of providers."""
    while True:
        for provider in PROVIDERS:
            try:
                response = await g4f.ChatCompletion.create_async(
                    model=config.get('model', DEFAULT_CONFIG['model']),
                    provider=provider,
                    messages=[{"role": "user", "content": "Hello"}],
                    max_tokens=10,
                )
                print(f"Provider {provider.__name__} is working")
            except Exception as e:
                print(f"Provider {provider.__name__} is not working: {str(e)}")
        await asyncio.sleep(3600)  # Check every hour

def load_custom_reward_functions():
    """Load custom reward functions from the config folder."""
    reward_functions = {}
    config_folder = 'config'
    for filename in os.listdir(config_folder):
        if filename.endswith('.py'):
            module_name = filename[:-3]
            module = __import__(f'{config_folder}.{module_name}', fromlist=['reward_function'])
            if hasattr(module, 'reward_function'):
                reward_functions[module_name] = module.reward_function
    return reward_functions

def evolutionary_strategy_update(ppo_model):
    """Implement Evolutionary Strategy for fine-tuning."""
    # This is a placeholder implementation
    noise = np.random.normal(0, 0.1, ppo_model.get_parameters().shape)
    new_params = ppo_model.get_parameters() + noise
    ppo_model.set_parameters(new_params)

def main():
    """Main entry point to start the server and ngrok tunnel."""
    parser = argparse.ArgumentParser(description="GPT4Free Advanced Local Reverse Proxy with LangChain")
    parser.add_argument('--port', type=int, default=5000, help='Port to run the server on')
    args = parser.parse_args()

    # Start ngrok
    public_url = start_ngrok(args.port)

    print(f"Server is running on http://localhost:{args.port}")
    print(f"Public URL: {public_url}")
    print("Available endpoints:")
    print(f"  - Chat completions: {public_url}/v1/chat/completions")
    print(f"  - Configuration: {public_url}/config")
    print(f"  - Providers list: {public_url}/providers")
    print("\nCurrent configuration:")
    for key, value in DEFAULT_CONFIG.items():
        print(f"  {key}: {value}")

    # Load custom reward functions
    reward_functions = load_custom_reward_functions()
    print(f"Loaded {len(reward_functions)} custom reward functions")

    # Start the provider status checking task
    asyncio.get_event_loop().create_task(check_provider_status())

    # Periodically update PPO model using Evolutionary Strategy
    def update_ppo_model():
        while True:
            evolutionary_strategy_update(ppo_model)
            time.sleep(3600)  # Update every hour

    import threading
    threading.Thread(target=update_ppo_model, daemon=True).start()

    app.run(port=args.port, threaded=True)

if __name__ == '__main__':
    main()