improve so its a general agent that can take screenshots, think and execute actions using mouse  & keyboard : import os
import time
import pyautogui
import google.generativeai as genai
import cv2
import numpy as np
from ultralytics import YOLO
import logging
import threading
import random
import string
from dotenv import load_dotenv
from abc import ABC, abstractmethod
from typing import Dict, List, Any
import json
from concurrent.futures import ThreadPoolExecutor
from PIL import Image
import io

# Load environment variables
load_dotenv()

# Set up logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
GOOGLE_API_KEY = os.environ["GOOGLE_API_KEY"]
if not GOOGLE_API_KEY:
    raise ValueError("GOOGLE_API_KEY environment variable is not set")

genai.configure(api_key=GOOGLE_API_KEY)

# Initialize Gemini
model = genai.GenerativeModel('gemini-1.5-flash')

ALLOWED_EXTENSIONS = {"png", "jpg", "jpeg"}

def allowed_file(filename):
    """Returns if a filename is supported via its extension"""
    _, ext = os.path.splitext(filename)
    return ext.lstrip('.').lower() in ALLOWED_EXTENSIONS

class Agent(ABC):
    def __init__(self, name: str, role: str, system_prompt: str):
        self.name = name
        self.role = role
        self.system_prompt = system_prompt
        self.chat = model.start_chat(history=[])
        self.knowledge_base: Dict[str, Any] = {}

    def send_message(self, message: str, image: Image.Image = None) -> str:
        logger.info(f"{self.name}: Sending message to Gemini: {message}")
        try:
            if image:
                response = self.chat.send_message([message, image])
            else:
                response = self.chat.send_message(message)
            logger.info(f"{self.name}: Received response from Gemini: {response.text}")
            return response.text
        except Exception as e:
            logger.error(f"{self.name}: Error sending message to Gemini: {e}")
            return f"Error: {str(e)}"

    @abstractmethod
    def think(self) -> None:
        pass

    def update_knowledge(self, key: str, value: Any) -> None:
        self.knowledge_base[key] = value

    def get_knowledge(self, key: str) -> Any:
        return self.knowledge_base.get(key)

    def share_knowledge(self, other_agent: 'Agent') -> None:
        other_agent.knowledge_base.update(self.knowledge_base)

class MasterAgent(Agent):
    def __init__(self):
        super().__init__("Master", "Control, delegate, and train", 
                         "You are the master agent. Your role is to analyze tasks, delegate them to appropriate sub-agents, and manage the training of YOLO models.")
        self.sub_agents: Dict[str, Agent] = {}
        self.screenshot_thread: threading.Thread = None
        self.screenshot_counter: int = 0
        self.yolo_models: Dict[str, str] = {}
        self.task_queue: List[str] = []
        self.executor = ThreadPoolExecutor(max_workers=5)
        self.screenshot_interval: int = 60  # Screenshot interval in seconds

    def delegate_task(self, task_description: str) -> str:
        logger.info(f"{self.name}: Received task: {task_description}")

        if "screenshot twitter" in task_description.lower():
            return self.get_or_create_agent('twitter').take_screenshot()
        elif "train yolo" in task_description.lower():
            return self.train_yolo_model(task_description)
        else:
            self.task_queue.append(task_description)
            return "Task added to queue for processing."

    def get_or_create_agent(self, agent_type: str) -> Agent:
        if agent_type not in self.sub_agents:
            if agent_type == 'twitter':
                self.sub_agents[agent_type] = TwitterAgent()
        return self.sub_agents[agent_type]

    def train_yolo_model(self, task_description: str) -> str:
        logger.info(f"{self.name}: Initiating YOLO model training based on: {task_description}")
        try:
            object_to_detect = task_description.split("train yolo to detect ")[1].strip()
            dataset_path = self.create_sample_dataset(object_to_detect)
            model_name = ''.join(random.choices(string.ascii_lowercase, k=5))
            model = YOLO("yolov8n.yaml")
            results = model.train(data=dataset_path, epochs=10, imgsz=640, batch=16)
            model_path = f"{model_name}.pt"
            model.save(model_path)
            self.yolo_models[object_to_detect] = model_path
            self.update_knowledge(f"yolo_model_{object_to_detect}", model_path)
            return f"YOLO model trained to detect '{object_to_detect}' and saved as {model_path}"
        except Exception as e:
            logger.error(f"Error during YOLO model training: {e}")
            return f"Error training YOLO model: {e}"

    def create_sample_dataset(self, object_name: str) -> str:
        dataset_path = f"dataset_{object_name}"
        os.makedirs(dataset_path, exist_ok=True)
        os.makedirs(os.path.join(dataset_path, "images"), exist_ok=True)
        os.makedirs(os.path.join(dataset_path, "labels"), exist_ok=True)

        for i in range(100):  # Increased dataset size
            image = np.random.randint(0, 255, (416, 416, 3), dtype=np.uint8)
            cv2.imwrite(os.path.join(dataset_path, "images", f"image_{i}.jpg"), image)
            with open(os.path.join(dataset_path, "labels", f"image_{i}.txt"), "w") as label_file:
                x, y, w, h = np.random.rand(4)
                label_file.write(f"0 {x} {y} {w} {h}\n")

        with open(os.path.join(dataset_path, "data.yaml"), "w") as yaml_file:
            yaml_file.write(f"""
train: {os.path.join(dataset_path, "images")}
val: {os.path.join(dataset_path, "images")}

nc: 1
names: ['{object_name}']
            """)

        return dataset_path

    def start_screenshot_thread(self) -> None:
        logger.info(f"{self.name}: Starting screenshot thread")
        self.screenshot_thread = threading.Thread(target=self.take_screenshots)
        self.screenshot_thread.daemon = True
        self.screenshot_thread.start()

    def take_screenshots(self) -> None:
        logger.info(f"{self.name}: Screenshot thread started")
        while True:
            try:
                logger.info(f"{self.name}: Attempting to take a screenshot")
                screenshot = pyautogui.screenshot()
                filename = f"screenshot_{self.screenshot_counter}.png"
                screenshot.save(filename)
                self.screenshot_counter += 1
                logger.info(f"{self.name}: Screenshot saved as {filename}")
                self.analyze_screenshot(filename)
                time.sleep(self.screenshot_interval)
            except Exception as e:
                logger.error(f"{self.name}: Error taking screenshot: {e}")
                time.sleep(1)  # Wait a bit before retrying

    def analyze_screenshot(self, image_path: str) -> None:
        logger.info(f"{self.name}: Analyzing screenshot: {image_path}")
        try:
            with Image.open(image_path) as img:
                analysis = self.send_message("Analyze this screenshot and describe its contents:", img)
            logger.info(f"{self.name}: Screenshot analysis: {analysis}")
            self.update_knowledge(f"screenshot_analysis_{image_path}", analysis)
            self.task_queue.append(f"Process screenshot analysis: {analysis}")
        except Exception as e:
            logger.error(f"{self.name}: Error analyzing screenshot: {e}")
            self.task_queue.append(f"Handle failed screenshot analysis: {image_path}")

    def think(self) -> None:
        logger.info(f"{self.name}: Thinking...")
        self.process_task_queue()
        self.coordinate_sub_agents()

    def process_task_queue(self) -> None:
        logger.info(f"{self.name}: Processing task queue. Current queue size: {len(self.task_queue)}")
        while self.task_queue:
            task = self.task_queue.pop(0)
            logger.info(f"{self.name}: Processing task: {task}")
            self.executor.submit(self.process_task, task)

    def process_task(self, task: str) -> None:
        if task.startswith("Handle failed screenshot analysis:"):
            image_path = task.split(": ")[1]
            self.handle_failed_analysis(image_path)
        else:
            analysis = self.send_message(f"Analyze this task and suggest actions: {task}")
            logger.info(f"{self.name}: Task analysis: {analysis}")
            if "twitter" in analysis.lower():
                self.get_or_create_agent('twitter').take_screenshot()
            elif "train" in analysis.lower() and "yolo" in analysis.lower():
                object_to_detect = analysis.split("train yolo to detect ")[1].split(".")[0].strip()
                self.train_yolo_model(f"Train YOLO to detect {object_to_detect}")

    def handle_failed_analysis(self, image_path: str) -> None:
        logger.info(f"{self.name}: Handling failed analysis for {image_path}")
        try:
            with Image.open(image_path) as img:
                retry_analysis = self.send_message("Retry analysis of this screenshot:", img)
            logger.info(f"{self.name}: Retry analysis result: {retry_analysis}")
            self.update_knowledge(f"retry_analysis_{image_path}", retry_analysis)
            self.task_queue.append(f"Process screenshot analysis: {retry_analysis}")
        except Exception as e:
            logger.error(f"{self.name}: Error in retry analysis: {e}")
            self.update_knowledge(f"failed_analysis_{image_path}", "Analysis failed after retry")

    def coordinate_sub_agents(self) -> None:
        for agent in self.sub_agents.values():
            agent.think()
            agent.share_knowledge(self)

    def activate_new_master(self) -> 'MasterAgent':
        logger.info(f"{self.name}: Activating a new Master Agent.")
        new_master = MasterAgent()
        self.share_knowledge(new_master)
        return new_master

class TwitterAgent(Agent):
    def __init__(self):
        super().__init__("TwitterAgent", "Interact with Twitter",
                         "You are the Twitter agent. You are responsible for interacting with the Twitter application.")
        self.yolo_model_path = "twitter_window_detector.pt"

    def take_screenshot(self) -> str:
        try:
            screenshot = pyautogui.screenshot()
            results = YOLO(self.yolo_model_path)(np.array(screenshot))

            if results[0].boxes.xyxy.numel() > 0:
                x1, y1, x2, y2 = results[0].boxes.xyxy[0].tolist()
                twitter_screenshot = pyautogui.screenshot(region=(x1, y1, x2 - x1, y2 - y1))
                filename = f"twitter_screenshot_{int(time.time())}.png"
                twitter_screenshot.save(filename)
                self.analyze_content(filename)
                return f"Twitter screenshot taken and saved as {filename}."
            else:
                return "Twitter window not found."
        except Exception as e:
            logger.error(f"Error during Twitter screenshot: {e}")
            return f"Error taking Twitter screenshot: {e}"

    def analyze_content(self, image_path: str) -> None:
        try:
            with Image.open(image_path) as img:
                analysis = self.send_message("Analyze the content of this Twitter screenshot:", img)
            logger.info(f"{self.name}: Twitter content analysis: {analysis}")
            self.update_knowledge(f"twitter_analysis_{image_path}", analysis)
        except Exception as e:
            logger.error(f"{self.name}: Error analyzing Twitter content: {e}")

    def think(self) -> None:
        logger.info(f"{self.name}: Thinking about Twitter interactions...")
        recent_analyses = [v for k, v in self.knowledge_base.items() if k.startswith("twitter_analysis_")]
        if recent_analyses:
            latest_analysis = recent_analyses[-1]
            response = self.send_message(f"Based on this Twitter analysis, suggest potential actions or insights:\n{latest_analysis}")
            logger.info(f"{self.name}: Action suggestions: {response}")
            self.update_knowledge("latest_twitter_insights", response)

class AgentCoordinator:
    def __init__(self):
        self.master_agents: List[MasterAgent] = [MasterAgent()]
        self.active_master_index: int = 0

    def run(self) -> None:
        try:
            self.master_agents[0].start_screenshot_thread()
            logger.info("Screenshot thread started for initial Master Agent")
            
            while True:
                active_master = self.master_agents[self.active_master_index]
                active_master.think()
                
                # if random.random() < 0.05:  # 5% chance to activate a new master
                #     new_master = active_master.activate_new_master()
                #     new_master.start_screenshot_thread()
                #     self.master_agents.append(new_master)
                #     logger.info(f"New Master Agent activated and screenshot thread started. Total masters: {len(self.master_agents)}")
                
                # if random.random() < 0.02:  # 2% chance to switch active master
                #     self.active_master_index = random.randint(0, len(self.master_agents) - 1)
                #     logger.info(f"Switched to Master Agent {self.active_master_index}")
                
                self.share_knowledge_across_masters()
                
                time.sleep(60)  # Sleep for 10 seconds between cycles
        except KeyboardInterrupt:
            logger.info("Program terminated by user.")
        except Exception as e:
            logger.error(f"Unexpected error in AgentCoordinator: {e}")
        finally:
            self.cleanup()

    def share_knowledge_across_masters(self) -> None:
        for i, master in enumerate(self.master_agents):
            for j, other_master in enumerate(self.master_agents):
                if i != j:
                    master.share_knowledge(other_master)
        logger.info("Knowledge shared across all Master Agents.")

    def cleanup(self) -> None:
        for master in self.master_agents:
            master.executor.shutdown(wait=True)
        logger.info("Cleanup completed. All executors shut down.")

def main() -> None:
    coordinator = AgentCoordinator()
    coordinator.run()

if __name__ == "__main__":
    main()