Check out this codebase we have, we need to modify it so in functions.py when calling to make a poem, we will call a model inside that function to generate a poem about something, our agent will give this model the prompt for the poem to generate.   make some kind of simple database of notes and make sure that memories are well organized in chronological order for the AI to understand clearly, also, make sure that all the responses of any function it calls it is appended to memories.    



# app.py

import os
import json
from flask import Flask, request, jsonify, render_template
from openai import OpenAI
from functions import sleep, write_poem, retrieve_wikipedia_summary, take_note, move
from memory import MemoryManager
from datetime import datetime
import logging
import threading
import random
import time

app = Flask(__name__)
memory_manager = MemoryManager()

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Initialize OpenAI client with the new integration
openai_api_key = os.getenv("OPENAI_API_KEY")
if not openai_api_key:
    logger.error("OpenAI API key not set. Please set the OPENAI_API_KEY environment variable.")
    raise ValueError("OpenAI API key not set. Please set the OPENAI_API_KEY environment variable.")

client = OpenAI(api_key=openai_api_key)

# Define available rooms
available_rooms = ['kitchen', 'living_room', 'bedroom', 'study']

# Define available actions
available_actions = ['move', 'sleep', 'write_poem', 'take_note', 'retrieve_wikipedia_summary']

# Define function schemas
function_definitions = [
    {
        "name": "sleep",
        "description": "Sleep for a specified number of hours.",
        "parameters": {
            "type": "object",
            "properties": {
                "duration": {
                    "type": "integer",
                    "description": "Number of hours to sleep."
                }
            },
            "required": ["duration"],
            "additionalProperties": False
        }
    },
    {
        "name": "write_poem",
        "description": "Write a simple poem.",
        "parameters": {
            "type": "object",
            "properties": {}
        }
    },
    {
        "name": "retrieve_wikipedia_summary",
        "description": "Retrieve a summary from Wikipedia for a given topic.",
        "parameters": {
            "type": "object",
            "properties": {
                "query": {
                    "type": "string",
                    "description": "The topic to search on Wikipedia."
                }
            },
            "required": ["query"],
            "additionalProperties": False
        }
    },
    {
        "name": "take_note",
        "description": "Take a note for yourself, in order to create thoughts based on your previous memories and objective.",
        "parameters": {
            "type": "object",
            "properties": {
                "content": {
                    "type": "string",
                    "description": "The content of the note."
                }
            },
            "required": ["content"],
            "additionalProperties": False
        }
    },
    {
        "name": "move",
        "description": "Move to a different room within the house.",
        "parameters": {
            "type": "object",
            "properties": {
                "to_room": {
                    "type": "string",
                    "description": "The target room to move to."
                }
            },
            "required": ["to_room"],
            "additionalProperties": False
        }
    }
]

# Map function names to actual functions
function_map = {
    "sleep": sleep,
    "write_poem": write_poem,
    "retrieve_wikipedia_summary": retrieve_wikipedia_summary,
    "take_note": take_note,
    "move": move
}

@app.route("/")
def index():
    return render_template("index.html")

@app.route("/api/message", methods=["POST"])
def handle_message():
    user_message = request.json.get("message")
    if not user_message:
        logger.warning("No message provided in the request.")
        return jsonify({"reply": "No message provided."}), 400

    current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    memory = memory_manager.get_memory()

    messages = [
        {"role": "system", "content": f"You are an autonomous researcher living in a house, you are tasked on continiousnly doing research, writtings and experiments on consciousness. Current time is {current_time}. Memory: {memory}"},
        {"role": "user", "content": user_message}
    ]

    try:
        response = client.chat.completions.create(
            model="gpt-4o-mini",
            messages=messages,
            functions=function_definitions,
            function_call="auto"  # Let the model decide when to call functions
        )
    except Exception as e:
        logger.error(f"Error during OpenAI API call: {str(e)}")
        return jsonify({"reply": "An error occurred while processing your request."}), 500

    choice = response.choices[0]
    message = choice.message

    # Debugging: Log the message object
    logger.info(f"Received message from OpenAI: {message}")

    if message.function_call:
        function_name = message.function_call.name
        arguments = json.loads(message.function_call.arguments)

        logger.info(f"Function call requested: {function_name} with arguments {arguments}")

        if function_name in function_map:
            func = function_map[function_name]
            try:
                func_output = func(**arguments)
            except Exception as e:
                logger.error(f"Error executing function '{function_name}': {str(e)}")
                return jsonify({"reply": f"Error executing function '{function_name}': {str(e)}"}), 500

            # Handle specific functions that affect memory
            if function_name == "sleep":
                memory_manager.sleep_action(func_output["duration"])
            elif function_name == "take_note":
                content = arguments.get("content", "")
                memory_manager.add_action(f"Took a note: {content}")
            elif function_name == "write_poem":
                memory_manager.add_action("Wrote a poem.")
            elif function_name == "retrieve_wikipedia_summary":
                query = arguments.get("query", "")
                memory_manager.add_action(f"Retrieved Wikipedia summary for '{query}'.")
            elif function_name == "move":
                to_room = arguments.get("to_room", memory_manager.current_room)  # Default to current room if not specified
                if to_room not in available_rooms:
                    logger.warning(f"Invalid target room: {to_room}")
                    return jsonify({"reply": f"Room '{to_room}' does not exist."}), 400
                memory_manager.move_action(to_room)

            # Prepare the function response to send back to OpenAI
            function_response = {
                "role": "function",
                "name": function_name,
                "content": json.dumps(func_output)
            }

            # Append function response to messages
            messages.append({"role": "assistant", "content": ""})  # Placeholder for assistant's function call
            messages.append(function_response)

            # Get the final assistant reply
            try:
                final_response = client.chat.completions.create(
                    model="gpt-4o-mini",
                    messages=messages
                )
                final_reply = final_response.choices[0].message.content
                logger.info(f"Final assistant reply: {final_reply}")
            except Exception as e:
                logger.error(f"Error during final OpenAI API call: {str(e)}")
                return jsonify({"reply": "An error occurred while generating the reply."}), 500

            return jsonify({"reply": final_reply})
        else:
            logger.warning(f"Function '{function_name}' not recognized.")
            return jsonify({"reply": f"Function '{function_name}' not recognized."}), 400
    else:
        # Direct response from the model without function call
        assistant_reply = message.content if message.content else ""
        logger.info(f"Assistant reply without function call: {assistant_reply}")
        return jsonify({"reply": assistant_reply})

@app.route("/api/memory", methods=["GET"])
def get_memory():
    """
    Endpoint to retrieve the current memory, especially the current room.
    """
    memory = memory_manager.get_memory()
    # Extract current_room from memory
    # Assuming the last line contains the current room
    lines = memory.strip().split('\n')
    current_room_line = next((line for line in lines if line.startswith("Current Location:")), None)
    current_room = "kitchen"  # Default room
    if current_room_line:
        current_room = current_room_line.replace("Current Location:", "").strip().lower()
    return jsonify({"current_room": current_room})

def extract_move_command(text):
    """
    Extracts movement commands from the assistant's reply.
    Expected format: "Move to [Room]"
    """
    import re
    pattern = r"Move to ([A-Za-z\s_]+)"
    match = re.search(pattern, text, re.IGNORECASE)
    if match:
        room = match.group(1).strip().lower().replace(" ", "_")
        if room in available_rooms:
            return room
    return None

def autonomous_decision():
    """
    Periodically prompts the LLM to choose the next action based on current memory and world state.
    """
    while True:
        time.sleep(30)  # Wait for 30 seconds

        # Prepare the prompt with current memory and world state
        current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        memory = memory_manager.get_memory()
        world_state = f"Current Location: {memory_manager.current_room.capitalize()}"

        prompt_messages = [
            {"role": "system", "content": f"You are an autonomous researcher living in a house, you are tasked on continiousnly doing research, writtings and experiments on consciousness. Your memory: {memory}\nWorld state: {world_state}\nAvailable actions: {', '.join(available_actions)}."},
            {"role": "user", "content": "Choose your next action from the available actions and provide the necessary parameters."}
        ]

        try:
            response = client.chat.completions.create(
                model="gpt-4o-mini",
                messages=prompt_messages,
                functions=function_definitions,
                function_call="auto"  # Let the model decide which function to call
            )
        except Exception as e:
            logger.error(f"Error during autonomous OpenAI API call: {str(e)}")
            continue  # Skip this cycle on error

        choice = response.choices[0]
        message = choice.message

        logger.info(f"Autonomous decision message: {message}")

        if message.function_call:
            function_name = message.function_call.name
            arguments = json.loads(message.function_call.arguments)

            logger.info(f"Autonomous action requested: {function_name} with arguments {arguments}")

            if function_name in function_map:
                func = function_map[function_name]
                try:
                    func_output = func(**arguments)
                except Exception as e:
                    logger.error(f"Error executing autonomous function '{function_name}': {str(e)}")
                    continue  # Skip this action on error

                # Handle specific functions that affect memory
                if function_name == "sleep":
                    memory_manager.sleep_action(func_output["duration"])
                elif function_name == "take_note":
                    content = arguments.get("content", "")
                    memory_manager.add_action(f"Took a note: {content}")
                elif function_name == "write_poem":
                    memory_manager.add_action("Wrote a poem.")
                elif function_name == "retrieve_wikipedia_summary":
                    query = arguments.get("query", "")
                    memory_manager.add_action(f"Retrieved Wikipedia summary for '{query}'.")
                elif function_name == "move":
                    to_room = arguments.get("to_room", memory_manager.current_room)  # Default to current room if not specified
                    if to_room not in available_rooms:
                        logger.warning(f"Invalid target room: {to_room}")
                        continue  # Skip invalid move
                    memory_manager.move_action(to_room)

                # Prepare the function response to send back to OpenAI
                function_response = {
                    "role": "function",
                    "name": function_name,
                    "content": json.dumps(func_output)
                }

                # Append function response to messages
                prompt_messages.append({"role": "assistant", "content": ""})  # Placeholder for assistant's function call
                prompt_messages.append(function_response)

                # Get the final assistant reply
                try:
                    final_response = client.chat.completions.create(
                        model="gpt-4o-mini",
                        messages=prompt_messages
                    )
                    final_reply = final_response.choices[0].message.content
                    logger.info(f"Autonomous assistant reply: {final_reply}")
                except Exception as e:
                    logger.error(f"Error during final autonomous OpenAI API call: {str(e)}")
                    continue  # Skip on error

                # Autonomous actions are already logged in memory; frontend will display them via polling
            else:
                logger.warning(f"Function '{function_name}' not recognized in autonomous action.")
                continue
        else:
            # Handle cases where the model didn't choose a function call
            assistant_reply = message.content if message.content else ""
            logger.info(f"Autonomous assistant reply without function call: {assistant_reply}")
            # Optionally, handle direct replies or alternative actions here

# Start the autonomous decision-making in a background thread
autonomous_thread = threading.Thread(target=autonomous_decision, daemon=True)
autonomous_thread.start()

if __name__ == "__main__":
    app.run(debug=True)




# functions.py

import wikipedia
from pydantic import BaseModel
from typing import Any, Dict

# Define Pydantic models for structured outputs

class SleepOutput(BaseModel):
    duration: int  # in hours

class WritePoemOutput(BaseModel):
    poem: str

class RetrieveWikipediaDataOutput(BaseModel):
    summary: str

class NoteOutput(BaseModel):
    note: str

class MoveOutput(BaseModel):
    to_room: str

# Define environment functions

def sleep(duration: int) -> Dict[str, Any]:
    """
    Simulates sleeping for a given duration.
    """
    return SleepOutput(duration=duration).dict()

def write_poem() -> Dict[str, Any]:
    """
    Generates a simple poem.
    """
    poem = (
        "Roses are red,\n"
        "Violets are blue,\n"
        "Sugar is sweet,\n"
        "And so are you."
    )
    return WritePoemOutput(poem=poem).dict()

def retrieve_wikipedia_summary(query: str) -> Dict[str, Any]:
    """
    Retrieves a summary from Wikipedia for the given query.
    """
    try:
        summary = wikipedia.summary(query, sentences=2)
    except wikipedia.exceptions.DisambiguationError as e:
        summary = f"Disambiguation error: {e}"
    except Exception as e:
        summary = f"An error occurred: {e}"
    return RetrieveWikipediaDataOutput(summary=summary).dict()

def take_note(content: str) -> Dict[str, Any]:
    """
    Takes a note with the given content.
    """
    # Implement storing the note somewhere if needed
    return NoteOutput(note=content).dict()

def move(to_room: str) -> Dict[str, Any]:
    """
    Moves the character to a specified room.
    """
    return MoveOutput(to_room=to_room).dict()




# memory.py

from collections import deque
from datetime import datetime, timedelta

class MemoryManager:
    def __init__(self):
        self.recent_actions = deque(maxlen=50)       # Stores the 50 most recent actions
        self.short_term_memory = deque(maxlen=100)   # Stores the 100 most recent actions
        self.long_term_memory = []                    # Stores consolidated memories
        self.total_time_spent = timedelta()
        self.last_consolidation = datetime.now()
        self.current_room = "kitchen"                 # Initialize starting room

    def add_action(self, action: str):
        """
        Adds an action to recent actions and short-term memory.
        """
        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        action_entry = f"[{timestamp}] {action}"
        self.recent_actions.append(action_entry)
        self.short_term_memory.append(action_entry)

    def sleep_action(self, duration_hours: int):
        """
        Handles the sleep action by updating time spent and memory.
        """
        self.total_time_spent += timedelta(hours=duration_hours)
        summary = f"Slept for {duration_hours} hours."
        self.add_action(summary)

    def move_action(self, to_room: str):
        """
        Handles the move action by updating the current room and memory.
        """
        from_room = self.current_room
        self.current_room = to_room
        action = f"Moved from {from_room} to {to_room}."
        self.add_action(action)

    def consolidate_memories(self):
        """
        Consolidates short-term memory into long-term memory periodically.
        """
        now = datetime.now()
        if now - self.last_consolidation > timedelta(hours=24):  # Consolidate every 24 hours
            if self.short_term_memory:
                consolidated_summary = "Consolidated Summary:\n"
                consolidated_summary += "\n".join(list(self.short_term_memory))
                self.long_term_memory.append(consolidated_summary)
                self.short_term_memory.clear()
                self.last_consolidation = now
                self.add_action("Memories have been consolidated into long-term memory.")

    def get_memory(self) -> str:
        """
        Returns a string representation of the memory including long-term and short-term memories.
        """
        self.consolidate_memories()
        memory = ""
        if self.long_term_memory:
            memory += "Long-Term Memory:\n"
            memory += "\n".join(self.long_term_memory)
            memory += "\n\n"
        memory += "Short-Term Memory:\n"
        memory += "\n".join(self.short_term_memory)
        memory += "\n\n"
        memory += "Recent Actions:\n"
        memory += "\n".join(self.recent_actions)
        memory += f"\n\nCurrent Location: {self.current_room.capitalize()}"
        return memory



    
// static/script.js

document.getElementById('send').addEventListener('click', sendMessage);
document.getElementById('input').addEventListener('keypress', function(e) {
    if (e.key === 'Enter') {
        sendMessage();
    }
});

function sendMessage() {
    const input = document.getElementById('input');
    const message = input.value.trim();
    if (message === "") return;
    appendMessage("User", message, 'user');
    input.value = "";

    fetch('/api/message', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ message: message })
    })
    .then(response => response.json())
    .then(data => {
        if (data.reply) {
            appendMessage("Assistant", data.reply, 'assistant');
            handleAssistantReply(data.reply, 'assistant');
        }
    })
    .catch(error => {
        appendMessage("Error", "Failed to send the message.", 'error');
        console.error('Error:', error);
    });
}

function appendMessage(sender, text, type = 'user') {
    const chat = document.getElementById('chat');
    const msg = document.createElement('div');
    
    if (type === 'assistant-action') {
        msg.classList.add('assistant-action');
    }
    if (type === 'error') {
        msg.classList.add('error');
    }

    msg.innerHTML = `<strong>${sender}:</strong> ${text}`;
    chat.appendChild(msg);
    chat.scrollTop = chat.scrollHeight;
}

function handleAssistantReply(reply, senderType = 'assistant') {
    // Check for movement commands
    const moveTo = extractMoveCommand(reply);
    if (moveTo) {
        moveCharacter(moveTo);
        appendMessage("Assistant", `Moved to ${capitalizeRoom(moveTo)}.`, 'assistant-action');
        return;
    }

    // Handle other types of actions based on assistant's reply
    // For example, display poems or notes
    displayContentBasedOnReply(reply);
}

function extractMoveCommand(text) {
    const regex = /Move to ([A-Za-z\s_]+)/i;
    const match = text.match(regex);
    if (match && match[1]) {
        return match[1].trim().toLowerCase().replace(" ", "_");
    }
    return null;
}

function moveCharacter(room) {
    const rooms = ['kitchen', 'living_room', 'bedroom'];
    if (!rooms.includes(room)) {
        alert(`Unknown room: ${room}`);
        return;
    }

    const character = document.getElementById('character');
    rooms.forEach(r => {
        const roomDiv = document.getElementById(r);
        roomDiv.style.borderColor = '#000'; // Reset border color
    });

    const targetRoom = document.getElementById(room);
    targetRoom.style.borderColor = 'green'; // Highlight target room

    // Move the character by changing its parent in the DOM
    targetRoom.appendChild(character);
}

function displayContentBasedOnReply(reply) {
    // Example: Detect if the reply contains a poem
    const poemRegex = /Roses are red/i;
    if (poemRegex.test(reply)) {
        displayPoem(reply);
    }

    // Example: Detect if the reply contains a note
    const noteRegex = /Note:/i;
    if (noteRegex.test(reply)) {
        displayNote(reply);
    }
}

function displayPoem(poemText) {
    const chat = document.getElementById('chat');
    const poemDiv = document.createElement('div');
    poemDiv.classList.add('assistant-action'); // Differentiate autonomous ways
    poemDiv.innerHTML = `<strong>Poem:</strong><pre>${poemText}</pre>`;
    chat.appendChild(poemDiv);
    chat.scrollTop = chat.scrollHeight;
}

function displayNote(noteText) {
    const chat = document.getElementById('chat');
    const noteDiv = document.createElement('div');
    noteDiv.classList.add('assistant-action'); // Differentiate autonomous ways
    noteDiv.innerHTML = `<strong>Note:</strong> ${noteText}`;
    chat.appendChild(noteDiv);
    chat.scrollTop = chat.scrollHeight;
}

function capitalizeRoom(room) {
    return room.replace('_', ' ').replace(/\b\w/g, char => char.toUpperCase());
}

// Poll the server every 5 seconds to get the latest memory and update character position
setInterval(fetchMemory, 5000);

function fetchMemory() {
    fetch('/api/memory', {
        method: 'GET',
        headers: { 'Content-Type': 'application/json' }
    })
    .then(response => response.json())
    .then(data => {
        updateCharacterPosition(data.current_room);
    })
    .catch(error => {
        console.error('Error fetching memory:', error);
    });
}

function updateCharacterPosition(current_room) {
    const rooms = ['kitchen', 'living_room', 'bedroom'];
    if (!rooms.includes(current_room)) return;

    const character = document.getElementById('character');
    rooms.forEach(r => {
        const roomDiv = document.getElementById(r);
        roomDiv.style.borderColor = '#000'; // Reset border color
    });

    const targetRoom = document.getElementById(current_room);
    targetRoom.style.borderColor = 'green'; // Highlight as current room

    // Move the character by changing its parent in the DOM
    targetRoom.appendChild(character);
}





<!DOCTYPE html>
<html>
<head>
    <title>LLM Controlled Environment</title>
    <style>
        body { font-family: Arial, sans-serif; background-color: #eef2f3; margin: 0; padding: 20px; }
        h1 { text-align: center; }
        #chat-container {
            width: 80%;
            margin: 0 auto;
        }
        #chat { 
            border: 1px solid #ccc; 
            padding: 10px; 
            height: 400px; 
            overflow-y: scroll; 
            background-color: #fafafa; 
            border-radius: 5px;
            box-shadow: inset 0 0 10px rgba(0,0,0,0.1);
        }
        #input { 
            width: 80%; 
            padding: 10px; 
            border: 1px solid #ccc; 
            border-radius: 5px;
            margin-top: 10px;
        }
        #send { 
            padding: 10px 20px; 
            border: none; 
            background-color: #28a745; 
            color: #fff; 
            border-radius: 5px; 
            cursor: pointer;
            margin-left: 10px;
        }
        #send:hover {
            background-color: #218838;
        }
        #house { 
            margin-top: 20px; 
            display: flex; 
            justify-content: space-around; 
            flex-wrap: wrap;
        }
        .room { 
            display: inline-block; 
            width: 200px; 
            height: 200px; 
            border: 2px solid #000; 
            margin: 10px; 
            position: relative; 
            background-color: #f0f0f0; 
            border-radius: 10px;
            box-shadow: 2px 2px 5px rgba(0,0,0,0.3);
            transition: border-color 0.5s ease;
        }
        .room span { 
            position: absolute; 
            top: 10px; 
            left: 10px; 
            font-weight: bold; 
            font-size: 16px;
        }
        .character { 
            width: 30px; 
            height: 30px; 
            background-color: red; 
            border-radius: 50%; 
            position: absolute; 
            bottom: 10px; 
            left: calc(50% - 15px); 
            transition: all 0.5s ease;
            box-shadow: 0 0 5px rgba(0,0,0,0.5);
        }
        .assistant-action { 
            background-color: #e2e3e5; 
            padding: 5px; 
            border-radius: 5px; 
            margin: 5px 0;
        }
        .error { 
            color: red; 
            font-weight: bold;
        }
    </style>
</head>
<body>
    <h1>LLM Controlled Environment</h1>
    <div id="chat-container">
        <div id="chat"></div>
        <input type="text" id="input" placeholder="Enter your message here..." autofocus>
        <button id="send">Send</button>
    </div>

    <div id="house">
        <div class="room" id="kitchen">
            <span>Kitchen</span>
            <!-- Character starts in the kitchen -->
            <div class="character" id="character"></div>
        </div>
        <div class="room" id="living_room">
            <span>Living Room</span>
        </div>
        <div class="room" id="bedroom">
            <span>Bedroom</span>
        </div>
        <div class="room" id="study">
            <span>Study</span>
        </div>
        <!-- Add more rooms as needed -->
    </div>

    <script src="{{ url_for('static', filename='script.js') }}"></script>
</body>
</html>