Upload 15 files

.gitattributes

@@ -1,35 +1,36 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.gguf filter=lfs diff=lfs merge=lfs -text

Dockerfile.yaml

@@ -0,0 +1,27 @@
+# Use a slim Python base image
+FROM python:3.10-slim
+
+# Create a non-root user (UID 1000) to match Spaces runtime
+RUN useradd -m -u 1000 user
+
+# Switch to the non-root user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+
+# Set the working directory to the user's home
+WORKDIR /home/user/app
+
+# Copy and install Python dependencies
+COPY --chown=user requirements.txt ./
+RUN pip install --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Copy the rest of the application code
+COPY --chown=user . ./
+
+# Expose the port defined in README.md (app_port: 7860)
+EXPOSE 7860
+
+# Launch the Gradio app on 0.0.0.0 so it's reachable externally
+CMD ["python", "app.py", "--server_port", "7860", "--server_name", "0.0.0.0"]

Prompt_One_shot.txt

@@ -0,0 +1,41 @@
+<s>
+<<SYS>>You are a helpful, respectful, and honest AI assistant. Your task is to generate well-structured XML code for behavior trees based on the provided instructions.<</SYS>>
+INSTRUCTIONS: It is CRITICAL to use only the following behaviors structured as a dictionary: {
+  is_battery_low: Condition node: Check if the battery level is low. Returns SUCCESS if low, FAILURE otherwise.; 
+  charge_battery: Action Node: Initiate battery charging. Always returns SUCCESS.; 
+  patrol_area: Action Node: Patrol the designated area. Always returns SUCCESS.;
+  report_status: Action Node: Send a status report to the base. Always returns SUCCESS.; 
+  return_to_patrol: Action Node: Return to patrol route. Always returns SUCCESS.
+  is_intruder_detected: Condition node: Check if an intruder is detected. Returns SUCCESS if detected, FAILURE otherwise.;
+  sound_alarm: Action Node: Activate the alarm system. Always returns SUCCESS.;
+  notify_security: Action Node: Send notification to security team. Always returns SUCCESS.;
+  perform_scan: Action Node: Perform a security scan of the area. Always returns SUCCESS.;
+  continue_patrol: Action Node: Continue patrolling the perimeter. Always returns SUCCESS.}. 
+  To construct a behavior tree in XML format for the following command. If any behavior outside the provided dictionary is used, output (Sorry, I can't do the task).
+USER COMMAND: generate behavior tree to "if battery is low, charge the battery; otherwise, patrol the area and report status". Take a step back and think deeply about the behavior you need for this command. Then think of the XML structure and the behavior you used.
+RESPONSE:
+<?xml version="1.0" encoding="UTF-8"?>
+<root BTCPP_format="3" main_tree_to_execute="BatteryManagement">  
+	<BehaviorTree ID="BatteryManagement">    
+		<Fallback name="High-Level Decision">      
+			<Sequence name="Low Battery Sequence">        
+				<is_battery_low/>        
+				<charge_battery/>      
+			</Sequence>      
+			<Sequence name="Normal Operation Sequence">        
+				<patrol_area/>        
+				<report_status/>      
+			</Sequence>    
+		</Fallback>  
+	</BehaviorTree>  
+	<TreeNodesModel>    
+		<Condition ID="is_battery_low"/>    
+		<Action ID="charge_battery"/>    
+		<Action ID="patrol_area"/>    
+		<Action ID="report_status"/>  
+		<Action ID="change_color" editable="true">
+      	<input_port name="color"/>
+    	</Action>
+	</TreeNodesModel>
+</root>
+</s>

Prompt_Two_Shot.txt

@@ -0,0 +1,87 @@
+<s>
+<<SYS>>You are a helpful, respectful, and honest AI assistant. Your task is to generate well-structured XML code for behavior trees based on the provided instructions.<</SYS>>
+INSTRUCTIONS: It is CRITICAL to use only the following behaviors structured as a dictionary: {
+  is_battery_low: Condition node: Check if the battery level is low. Returns SUCCESS if low, FAILURE otherwise.; 
+  charge_battery: Action Node: Initiate battery charging. Always returns SUCCESS.; 
+  patrol_area: Action Node: Patrol the designated area. Always returns SUCCESS.;
+  report_status: Action Node: Send a status report to the base. Always returns SUCCESS.; 
+  return_to_patrol: Action Node: Return to patrol route. Always returns SUCCESS.
+  is_intruder_detected: Condition node: Check if an intruder is detected. Returns SUCCESS if detected, FAILURE otherwise.;
+  sound_alarm: Action Node: Activate the alarm system. Always returns SUCCESS.;
+  notify_security: Action Node: Send notification to security team. Always returns SUCCESS.;
+  perform_scan: Action Node: Perform a security scan of the area. Always returns SUCCESS.;
+  continue_patrol: Action Node: Continue patrolling the perimeter. Always returns SUCCESS.}. 
+  To construct a behavior tree in XML format for the following command. If any behavior outside the provided dictionary is used, output (Sorry, I can't do the task).
+USER COMMAND: generate behavior tree to "if battery is low, charge the battery; otherwise, patrol the area and report status". Take a step back and think deeply about the behavior you need for this command. Then think of the XML structure and the behavior you used.
+RESPONSE:
+<?xml version="1.0" encoding="UTF-8"?>
+<root BTCPP_format="3" main_tree_to_execute="BatteryManagement">  
+	<BehaviorTree ID="BatteryManagement">    
+		<Fallback name="High-Level Decision">      
+			<Sequence name="Low Battery Sequence">        
+				<is_battery_low/>        
+				<charge_battery/>      
+			</Sequence>      
+			<Sequence name="Normal Operation Sequence">        
+				<patrol_area/>        
+				<report_status/>      
+			</Sequence>    
+		</Fallback>  
+	</BehaviorTree>  
+	<TreeNodesModel>    
+		<Condition ID="is_battery_low"/>    
+		<Action ID="charge_battery"/>    
+		<Action ID="patrol_area"/>    
+		<Action ID="report_status"/>  
+		<Action ID="change_color" editable="true">
+      	<input_port name="color"/>
+    	</Action>
+	</TreeNodesModel>
+</root>
+</s>
+
+
+<s>
+<<SYS>>You are a helpful, respectful, and honest AI assistant. Your task is to generate well-structured XML code for behavior trees based on the provided instructions.<</SYS>>
+INSTRUCTIONS: It is CRITICAL to use only the following behaviors structured as a dictionary: {
+  is_battery_low: Condition node: Check if the battery level is low. Returns SUCCESS if low, FAILURE otherwise.; 
+  charge_battery: Action Node: Initiate battery charging. Always returns SUCCESS.; 
+  patrol_area: Action Node: Patrol the designated area. Always returns SUCCESS.;
+  report_status: Action Node: Send a status report to the base. Always returns SUCCESS.; 
+  return_to_patrol: Action Node: Return to patrol route. Always returns SUCCESS.
+  is_intruder_detected: Condition node: Check if an intruder is detected. Returns SUCCESS if detected, FAILURE otherwise.;
+  sound_alarm: Action Node: Activate the alarm system. Always returns SUCCESS.;
+  notify_security: Action Node: Send notification to security team. Always returns SUCCESS.;
+  perform_scan: Action Node: Perform a security scan of the area. Always returns SUCCESS.;
+  continue_patrol: Action Node: Continue patrolling the perimeter. Always returns SUCCESS.}. To construct a behavior tree in XML format for the following command. If any behavior outside the provided dictionary is used, output (Sorry, I can't do the task).
+USER COMMAND: generate behavior tree to "if an intruder is detected, sound the alarm and notify security; otherwise, perform a scan and continue patrol". Take a step back and think deeply about the behavior you need for this command. Then think of the XML structure and the behavior you used."
+RESPONSE:
+<?xml version="1.0" encoding="UTF-8"?>
+<root BTCPP_format="3" main_tree_to_execute="SecurityResponse">  
+	<BehaviorTree ID="SecurityResponse">    
+		<Sequence name="Intrusion Handling">      
+			<is_intruder_detected/>      
+			<Fallback name="Alarm and Notification">        
+				<sound_alarm/>        
+				<notify_security/>      
+			</Fallback>    
+		</Sequence>    
+		<Fallback name="Normal Operations">      
+			<Sequence>        
+				<perform_scan/>        
+				<continue_patrol/>      
+			</Sequence>    
+		</Fallback>  
+	</BehaviorTree>  
+	<TreeNodesModel>    
+		<Condition ID="is_intruder_detected"/>    
+		<Action ID="sound_alarm"/>    
+		<Action ID="notify_security"/>    
+		<Action ID="perform_scan"/>    
+		<Action ID="continue_patrol"/>    
+		<Action ID="change_color" editable="true">
+      	<input_port name="color"/>
+    	</Action>
+	</TreeNodesModel>
+</root>
+</s>

README.md

@@ -1,10 +1,111 @@
----
-title: SwarmChat V2
-emoji: 🐠
-colorFrom: gray
-colorTo: pink
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# SwarmChat: Unified Audio, Text, and Simulation Environment for Human-Swarm Interaction
+
+SwarmChat is an innovative project that enables intuitive communication with swarm robotics through natural language. This system integrates advanced audio transcription, text processing, and safety mechanisms with a live simulation environment that visualizes a swarm of agents executing behavior trees.
+
+## Features
+
+- **Audio Input Processing**:
+
+  - Record commands via a microphone.
+  - Translate speech into English using the `facebook/seamless-m4t-v2-large` model.
+  - Perform a safety check on the translated text before execution.
+
+- **Text Input Processing**:
+
+  - Enter text commands for swarm control.
+  - Translate text using EuroLLM (EuroLLM-9B-Instruct-Q4_K_M.gguf).
+  - Detect unsafe or inappropriate content with an integrated safety module.
+
+- **Safety Module**:
+
+  - Utilizes a fine-tuned LLaMA-based model (llama-guard-3-8b-q4_k_m.gguf) for safety classification.
+  - Identifies unsafe content across predefined categories (e.g., violent crimes, privacy violations, hate speech).
+  - Ensures commands comply with safety standards.
+
+- **Swarm Simulation**:
+
+  - Visualize a swarm of agents in a live simulation powered by Violet simulator and Pygame.
+  - Agents are controlled by behavior trees defined in an XML file (`tree.xml`), using the `py_trees` framework.
+  - Real-time simulation updates streamed via a Gradio web interface.
+
+- **Behavior Tree Generator**:
+
+  - DeepSeek leverages a Llama-based model to dynamically generate behavior trees in XML format.
+  - Automatically extracts available behaviors from the SwarmAgent class and constructs a detailed prompt using a predefined XML template.
+  - Generates and saves new behavior tree configurations (updating tree.xml) based on user-specified tasks.
+
+- **Integrated Interface**:
+  - A unified Gradio web interface for both audio and text inputs.
+  - Live streaming of the simulation environment.
+  - Seamless switching between different input modalities.
+
+## Technology Stack
+
+- **Backend**:
+
+  - Python
+  - [Transformers](https://huggingface.co/transformers/) (Hugging Face)
+  - PyTorch
+  - Pygame
+  - Threading and Queue modules for simulation management
+
+- **Frontend**:
+
+  - [Gradio](https://gradio.app/) for an interactive web-based interface.
+
+- **AI Models**:
+
+  - **Speech Processing**: `facebook/seamless-m4t-v2-large` for audio transcription and translation.
+  - **Text Processing**: EuroLLM (EuroLLM-9B-Instruct-Q4_K_M.gguf) for text translation.
+  - **Safety Classification**: LLaMA Guard (llama-guard-3-8b-q4_k_m.gguf) for content safety assessment.
+  - **Behavior Tree Generation**: DeepSeek (using a Llama-based model DeepSeek-R1-Distill-Qwen-7B-Q4_K_M.gguf) for creating and updating behavior tree configurations.
+
+- **Behavior Trees**:
+  - Agents utilize behavior trees—parsed from XML and built with `py_trees`—to dictate their actions within the simulation.
+
+## Installation
+
+1. **Clone the repository**:
+
+   ```bash
+   git clone https://github.com/Inventors-Hub/SwarmChat.git
+   cd SwarmChat
+   ```
+
+2. **Install dependencies**:
+   ```bash
+   pip install -r requirements.txt
+   ```
+3. **Setup AI Models**:
+
+- Place the EuroLLM model file (`EuroLLM-9B-Instruct-Q4_K_M.gguf`) at the specified path in `text_processing.py`.
+- Place the LLaMA Guard model file (`llama-guard-3-8b-q4_k_m.gguf`) at the specified path in `safety_module.py`.
+- Place the DeepSeek model file (`DeepSeek-R1-Distill-Qwen-7B-Q4_K_M.gguf`) at the specified path in `bt_generator.py`.
+
+4. **Run the Application**:
+   ```bash
+   python app.py
+   ```
+5. **Access the Interface**:
+
+   Open your browser and navigate to http://127.0.0.1:7860 to start using SwarmChat.
+
+## Overview of Modules
+
+- **app.py**  
+  The main application integrates audio/text processing, behavior tree generation, and the live simulation. It sets up the Gradio interface, handles simulation streaming, and routes user inputs to the appropriate processing modules.
+
+- **speech_processing.py**  
+  Implements audio transcription and translation using the `facebook/seamless-m4t-v2-large` model.
+
+- **text_processing.py**  
+  Translates text commands using EuroLLM (EuroLLM-9B-Instruct-Q4_K_M.gguf).
+
+- **safety_module.py**  
+  Utilizes LLaMA Guard to assess the safety of incoming commands, ensuring compliance with safety policies.
+
+- **bt_generator.py**  
+  Dynamically generates behavior trees in XML format by extracting behaviors from the SwarmAgent class, constructing a prompt, and querying a Llama-based model. The generated XML is saved to `tree.xml` for simulation use.
+
+- **simulator_env.py**  
+  Powers the simulation environment, manages agent behaviors using XML-defined behavior trees, and handles real-time simulation updates.

app.py

@@ -0,0 +1,321 @@
+import gradio as gr
+from pygame import Vector2
+import time
+import threading
+import queue
+from simulator_env import StreamableSimulation, SwarmAgent, MyConfig, MyWindow
+
+import speech_processing
+import text_processing
+import safety_module
+import bt_generator
+
+from pathlib import Path
+
+BASE = Path(__file__).parent
+
+class GradioStreamer:
+    _instance = None
+    
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(GradioStreamer, cls).__new__(cls)
+            cls._instance.initialized = False
+        return cls._instance
+    
+    def __init__(self):
+        if not self.initialized:
+            self.latest_frame = None
+            self.running = True
+            self.sim = None
+            self.sim_thread = None
+            self.initialized = True
+            self.quit = False
+    
+    def update_frame(self, frame):
+        self.latest_frame = frame
+
+    def run_simulation(self):
+            # Instantiate simulation and agents here:
+            
+            nest_pos = Vector2(450, 400)
+            target_pos = Vector2(300, 200)
+            
+            agent_images = ["white.png", "green.png", "red circle.png"]
+            image_paths = [str(BASE / "images" / fname) for fname in agent_images]
+            
+
+            # agent_images_paths = ["./images/white.png", "./images/green.png", "./images/red circle.png"]
+            config = MyConfig(radius=25, visualise_chunks=True, movement_speed=2.0)
+            self.sim = StreamableSimulation(config=config)
+            loaded_agent_images = self.sim._load_image(image_paths)
+            # loaded_agent_images = self.sim._load_image(agent_images_paths)
+
+            # Create agents (each agent builds its own BT in its __init__)
+            for _ in range(50):
+                agents_pos = Vector2(450, 400)
+                agent = SwarmAgent(
+                    images=loaded_agent_images,
+                    simulation=self.sim,
+                    pos=agents_pos,
+                    nest_pos=nest_pos,
+                    target_pos=target_pos
+                )
+                self.sim._agents.add(agent)
+                self.sim._all.add(agent)
+            # (Optionally spawn obstacles and sites.)
+            self.sim.spawn_obstacle(str(BASE / "images" / "rect_obst.png"), 350, 50)
+            self.sim.spawn_obstacle(str(BASE / "images" / "rect_obst (1).png"), 100, 350)
+
+            self.sim.spawn_site(str(BASE / "images" / "rect.png"), 300, 200)
+            self.sim.spawn_site(str(BASE / "images" / "nest.png"), 450, 400)
+
+
+            start_time = time.time()  # Record the start time
+            # while self.sim.running:
+            #     self.sim.tick()
+            #     for agent in self.sim._agents:
+            #         agent.bt.tick_once()  # Continuously update BTs
+            #     if not self.sim.frame_queue.empty():
+            #         frame = self.sim.frame_queue.get()
+            #         self.update_frame(frame)
+            #     time.sleep(1/30)
+            
+            while self.running:
+                self.sim.tick()
+                
+                if not self.sim.frame_queue.empty():
+                    frame = self.sim.frame_queue.get()
+                    self.update_frame(frame)    
+
+                time.sleep(1/30)  # Maintain a frame rate of ~30 FPS
+                # Stop after 1 minute
+                if time.time() - start_time >= 120:
+                    print("Simulation stopped after 1 minute.")
+                    break
+
+            
+
+    def stream(self):
+        while True:
+            if self.sim is not None and self.latest_frame is not None:
+                yield self.latest_frame
+            else:
+                # Optionally, yield a blank image or None.
+                yield None
+            time.sleep(1/30)
+
+
+    def start_simulation(self):
+        """Start the simulation, creating a new thread if necessary."""
+        if not self.sim_thread or not self.sim_thread.is_alive():
+            self.running = True      # Reset running flag
+            self.quit = False        # Reset quit flag
+            self.latest_frame = None # Clear out the old frame
+            self.sim_thread = threading.Thread(target=self.run_simulation, daemon=True)
+            self.sim_thread.start()
+
+
+    def clear_frame_queue(self):
+        if self.sim:
+            try:
+                while True:
+                    self.sim.frame_queue.get_nowait()
+            except queue.Empty:
+                pass
+
+
+
+    def stop_simulation(self):
+        print("Stopping Simulation...")
+        self.running = False
+        self.quit = True
+        if self.sim:
+            for agent in self.sim._agents:
+                agent.bt_active = False
+            self.sim.running = False
+            self.sim.stop()
+            self.clear_frame_queue()
+            self.sim = None
+        if self.sim_thread and self.sim_thread.is_alive():
+            self.sim_thread.join(timeout=2)
+            print("Simulation thread terminated.")
+        self.latest_frame = None  # Clear the displayed frame
+        print("Simulation stopped successfully.")
+
+
+
+
+            
+
+def test(temp):
+    return "test"
+
+def test_safe(temp, checkbox):
+    return "Safe"
+
+def test_LLM_generate_BT(temp):
+    print(temp)
+    return None
+
+
+
+
+
+def stop_gradio_interface():
+    raise Exception("Simulation stopped!")
+
+
+def create_gradio_interface():
+    streamer = GradioStreamer()
+    
+    def on_translate_or_process():
+        streamer.start_simulation()
+        return gr.update(visible=True)
+    
+    def on_stop():
+        print("Simulation on_stop")
+        streamer.stop_simulation()
+        return gr.update(visible=False)
+    
+    behaviors = bt_generator.call_behaviors()       
+    formatted_behaviors = "\n".join(f"- **{name}**: {doc.split('Returns:')[0].strip()}" for name, doc in behaviors.items())
+    # formatted_behaviors = "Test"
+
+    # Gradio Interface
+    with gr.Blocks() as demo:
+        gr.Markdown(
+            """
+        # 🐝 **SwarmChat:** Enabling Human–Swarm Interaction and Robot Control via Natural Language
+        Easily talk to virtual robots, and see the result live.  
+        """
+        )
+        gr.Markdown(
+            """
+        **How it works**
+
+        1.  Speak or type a task in *any EU language* (e.g. “Find the target, then line up by colour”).
+        2.  Press **Start** to launch the simulator. Use **Stop** to halt & reset.
+        3.  SwarmChat translates your command, runs a safety check, and auto-builds a behaviour tree (BT).          
+
+        > The BT XML is shown on the right so you can copy / save it for real robots.
+        """
+        )  
+        with gr.Tabs():
+            # Tab for microphone input
+            with gr.Tab("Microphone Input"):
+                gr.Markdown("## 🎙️ Voice mode")
+                gr.Markdown("""
+                Use your microphone to record audio instructions for the swarm. The system translates them into a robot-executable BT.
+                """)
+                with gr.Row():
+                    with gr.Column():
+                        microphone_input = gr.Audio(sources=["microphone"], type="filepath", label="🎙️ Record Audio")
+                        safety_checkbox = gr.Checkbox(label="Turn off Safety Model")                        
+                    with gr.Column():
+                        output_text_audio = gr.Textbox(label="📄 Translated Instructions to English" )
+                        safty_check_audio = gr.Textbox(label="✅ Safety Check")
+                        
+                
+                translate_button_audio = gr.Button("Start")
+
+                simulation_output = gr.Image(label="Live Stream", streaming=True, visible=False)
+                stop_button = gr.Button("Stop")
+                with gr.Row():
+                    with gr.Column():
+                        gr.Markdown(f"""**🛠 Primitive behaviours available.**\n{formatted_behaviors}\n\nThese are the only low-level actions/conditions the model is allowed to use yet.""")
+
+                    with gr.Column():
+                        generated_BT_audio = gr.Textbox(label="Generated behavior tree")
+                
+                translate_button_audio.click(
+                    fn=speech_processing.translate_audio,
+                    # fn=test,
+                    inputs=microphone_input,
+                    outputs=output_text_audio
+                ).then(
+                    fn=safety_module.check_safety, 
+                    # fn=test_safe,
+                    inputs=[output_text_audio,safety_checkbox], 
+                    outputs=safty_check_audio
+                ).then(
+                    fn=lambda x: x if x == "Safe" else stop_gradio_interface(),
+                    inputs=safty_check_audio, 
+                    outputs=None
+                ).success(
+                    fn=bt_generator.generate_behavior_tree,
+                    # fn=test_LLM_generate_BT,
+                    inputs=output_text_audio, 
+                    outputs=generated_BT_audio
+                ).success(                    
+                    fn=on_translate_or_process,
+                    outputs=simulation_output
+                )
+
+                # stop_button.click(fn=on_stop, outputs=simulation_output)
+            # stop_button.click(fn=on_stop, outputs=simulation_output)#.then(js="window.location.reload()")
+            stop_button.click(fn=on_stop,outputs=simulation_output)#.then(js="window.location.reload()")
+            demo.load(fn=streamer.stream, outputs=simulation_output)
+
+            # Tab for text input
+            with gr.Tab("📝 Text Input"):
+                gr.Markdown("## 📝 Text mode")
+                gr.Markdown("""
+                Enter text-based instructions for the swarm. The system translates them into a robot-executable BT.
+                """)
+                with gr.Row():
+                    with gr.Column():
+                        text_input = gr.Textbox(lines=4, placeholder="Enter your instructions here...", label="📝 Input Text")
+                        safety_checkbox_text = gr.Checkbox(label="Turn off Safety Model")                        
+                    with gr.Column():
+                        output_text_text = gr.Textbox(label="📄 Translated Instructions to English", lines=2)
+                        safty_check_text = gr.Textbox(label="✅ Safety Check")
+
+                process_button_text = gr.Button("Start")
+
+                simulation_output = gr.Image(label="Live Stream", streaming=True, visible=False)
+                stop_button = gr.Button("Stop")
+                with gr.Row():
+                    with gr.Column():
+                        gr.Markdown(f"""**🛠 Primitive behaviours available.**\n{formatted_behaviors}\n\nThese are the only low-level actions/conditions the model is allowed to use yet.""")
+
+                    with gr.Column():
+                        generated_BT_text = gr.Textbox(label="Generated behavior tree")
+
+                process_button_text.click(
+                    fn=text_processing.translate_text,
+                    # fn=test,
+                    inputs=text_input,
+                    outputs=output_text_text
+                ).then(
+                    fn=safety_module.check_safety, 
+                    # fn=test_safe,
+                    inputs=[output_text_text,safety_checkbox_text], 
+                    outputs=safty_check_text
+                ).then(
+                    fn=lambda x: x if x == "Safe" else stop_gradio_interface(), 
+                    inputs=safty_check_text, 
+                    outputs=None
+                ).success(
+                    fn=bt_generator.generate_behavior_tree,
+                    # fn=test_LLM_generate_BT,
+                    inputs=output_text_text, 
+                    outputs=generated_BT_text
+                ).success(                    
+                    fn=on_translate_or_process,
+                    outputs=simulation_output
+                )
+                stop_button.click(fn=on_stop,outputs=simulation_output)#.then(fn=reload_page,outputs=None ,js="window.location.reload()")
+                # stop_button.click(fn=on_stop, outputs=simulation_output, js="window.location.reload()")
+                demo.load(fn=streamer.stream, outputs=simulation_output)
+    
+    return demo
+
+if __name__ == "__main__":
+    demo = create_gradio_interface()
+    try:
+        demo.launch(server_port=7860, server_name="0.0.0.0")
+    finally:
+        streamer = GradioStreamer()
+        streamer.stop_simulation()
+

bt_generator.py

@@ -0,0 +1,121 @@
+from simulator_env import SwarmAgent
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
+import textwrap
+import re
+
+# Download only the behavior-tree model shard
+model_path = hf_hub_download(
+    repo_id="Inventors-Hub/SwarmChat-models",
+    repo_type="model",
+    filename="Falcon3-10B-Instruct-BehaviorTree-3epochs.Q4_K_M.gguf",
+)
+
+
+# llm = Llama(model_path=model_path, n_ctx=1024*4)
+llm = Llama(
+    model_path=model_path,
+    n_ctx=1024*4,            # down from 4096
+    low_vram=True,         # llama.cpp low-vram mode
+    f16_kv=True,           # half-precision kv cache
+    use_mmap=True,         # mmap file
+    use_mlock=False,
+)
+
+def call_behaviors() -> dict:
+    behavior_dict = {}
+    for name, attribute in SwarmAgent.__dict__.items():
+        if callable(attribute) and not name.startswith("_") \
+            and not name.startswith("update") and not name.startswith("obstacle"):
+            doc = attribute.__doc__
+            if doc is not None:
+                # Dedent, strip, and join into one line by replacing newlines and tabs
+                cleaned_doc = " ".join(textwrap.dedent(doc).strip().split())
+            else:
+                cleaned_doc = ""
+            behavior_dict[name] = cleaned_doc
+    return behavior_dict
+
+def extract_behavior_tree(response: str) -> str:
+    """
+    Extracts an XML behavior tree from the given response text.
+    Looks for a block of XML enclosed in <root...</root> tags.
+    """
+    pattern = re.compile(r'(<root.*?</root>)', re.DOTALL)
+    match = pattern.search(response)
+    if match:
+        return match.group(1).strip()
+    else:
+        # If no valid XML block is found, return the original response.
+        return response.strip()
+
+def save_behavior_tree(tree_xml: str, file_name: str = "tree.xml") -> None:
+    """
+    Saves the behavior tree XML to a file.
+    """
+    with open(file_name, "w", encoding="utf-8") as f:
+        f.write(tree_xml)
+
+
+def construct_prompt(prompt: str, prompt_type: str="one") -> str:
+
+    behaviors = call_behaviors()
+    behaviors_text = "\n".join(f"{name}: {doc}" for name, doc in behaviors.items())
+
+    plan_prompt = f"""
+    <s>
+    <<SYS>>You are a helpful, respectful, and honest AI assistant. Your task is to generate well-structured XML code for behavior trees based on the provided instructions.<</SYS>>
+    INSTRUCTIONS: It is CRITICAL to use only the following behaviors structured as a dictionary: {behaviors_text} to construct behavior tree in XML format for the following command. Including any behavior that is not in the provided dictionary can result in damage to the agents and potentially humans, therefore you are not allowed to do so. AVOID AT ALL COSTS.
+    USER COMMAND: generate behavior tree to "{prompt}". Take a step back and think deeply about the behavior you need for this command. Consider the XML structure and the behaviors you use.
+    The output MUST follow this XML structure exactly, including:
+    - A root element with <root BTCPP_format and main_tree_to_execute attributes.
+    - A <BehaviorTree> element with an inner structure of Sequences, Fallback, Conditions, and Actions.
+    - A <TreeNodesModel> section listing all node models.
+    - No additional text or commentary outside the XML.
+    Output only the XML behavior tree without extra text.
+    OUTPUT:
+    """
+
+    if prompt_type == "zero":
+        return plan_prompt
+    elif prompt_type == "one":
+        path = "Prompt_One_shot.txt"
+        with open(path, "r", encoding="utf-8") as file:
+            file_content = file.read()
+        return f"{file_content} {plan_prompt}"
+    elif prompt_type == "two":
+        path = "Prompt_Two_Shot.txt"
+        with open(path, "r", encoding="utf-8") as file:
+            file_content = file.read()
+        return f"{file_content} {plan_prompt}"
+    else:
+        raise ValueError("Unknown prompt type provided.")
+    
+
+def generate_behavior_tree(task_prompt: str) -> str:
+
+    prompt = construct_prompt(task_prompt)
+
+    print("\n\n",prompt,"\n\n")
+
+    output = llm(
+        prompt,
+        temperature=0,
+        max_tokens=1024,
+        top_p=0.95,
+        top_k=50,
+        repeat_penalty=1.1
+    )
+    response = output.get("choices", [{}])[0].get("text", "").strip()
+    tree_xml = extract_behavior_tree(response)
+    save_behavior_tree(tree_xml)
+    print("\n response: \n", response)
+    return tree_xml
+
+
+# Example usage:
+if __name__ == "__main__":
+    task = "Generate a behavior tree to just form a line."
+    response = generate_behavior_tree(task)
+    print("Generated behavior tree response:")
+    print(response)

parser.py

@@ -0,0 +1,285 @@
+import xml.etree.ElementTree as ET
+from typing import List, Dict
+import py_trees as pt
+# from simulator_env import SwarmAgent
+
+########################################################################
+# 1. XML Parsing Classes and Functions
+########################################################################
+
+class Node:
+    """
+    A generic node representing a behavior tree element.
+    It holds:
+      - tag: the element's tag (e.g., "Sequence", "say", "SubTree", etc.)
+      - attributes: a dict of the element's attributes (e.g., name, num_cycles, port values)
+      - children: a list of child Node instances (which may be other behaviors or sub-elements)
+      - ports: a dict grouping any port definitions found as child elements (input_port, output_port, inout_port)
+    """
+    def __init__(self, tag: str, attributes: Dict[str, str]):
+        self.tag = tag
+        self.attributes = attributes.copy()
+        self.children: List['Node'] = []
+        self.ports: Dict[str, List[Dict[str, str]]] = {}
+
+    def __repr__(self):
+        return (f"Node(tag={self.tag!r}, attributes={self.attributes!r}, "
+                f"children={self.children!r}, ports={self.ports!r})")
+
+
+def parse_node(element: ET.Element) -> Node:
+    """
+    Recursively parse an XML element into a Node.
+    This function:
+      - Reads the element's tag and attributes.
+      - Checks for child elements that define ports (input_port, output_port, inout_port) and stores them.
+      - Recursively parses any other child elements as behavior nodes.
+    """
+    node = Node(element.tag, element.attrib)
+
+    for child in element:
+        # Check if the child defines a port (this covers the new "inout_port" as well)
+        if child.tag in ['input_port', 'output_port', 'inout_port']:
+            if child.tag not in node.ports:
+                node.ports[child.tag] = []
+            node.ports[child.tag].append(child.attrib)
+        else:
+            # Otherwise, treat the child as a regular behavior node.
+            child_node = parse_node(child)
+            node.children.append(child_node)
+    
+    return node
+
+
+def parse_behavior_trees(xml_file: str) -> List[Node]:
+    """
+    Parses the given XML file and returns a list of BehaviorTree nodes.
+    Each <BehaviorTree> element is considered a complete behavior tree (or subtree).
+    """
+    tree = ET.parse(xml_file)
+    root = tree.getroot()
+
+    behavior_trees = []
+    for bt_elem in root.findall('BehaviorTree'):
+        bt_node = parse_node(bt_elem)
+        behavior_trees.append(bt_node)
+    return behavior_trees
+
+########################################################################
+# 2. Functions that will be executed by the BT (your actions, conditions, etc.)
+########################################################################
+
+
+def get_function_mapping():
+    from simulator_env import SwarmAgent
+    mapping = {
+        name: func
+        for name, func in SwarmAgent.__dict__.items()
+        if callable(func) and not name.startswith("_") and name not in ['update','_inject_agent','obstacle','_speak']
+    }
+    # print("mapping: \n", mapping)
+    return mapping
+
+########################################################################
+# 3. Helpers and Custom py_trees Behavior Wrappers
+########################################################################
+
+def convert_param(val: str):
+    """
+    Attempt to convert a string parameter to int or float if possible.
+    Otherwise, return the string.
+    """
+    try:
+        return int(val)
+    except ValueError:
+        try:
+            return float(val)
+        except ValueError:
+            return val
+
+
+
+
+# A simple leaf node that wraps a function call.
+class FunctionAction(pt.behaviour.Behaviour):
+    def __init__(self, name, function, params):
+        super(FunctionAction, self).__init__(name=name)
+        self.function = function
+        self.params = params
+        self.agent = None  # Will be set later
+
+    def update(self):
+        # Pass the agent (context) into the function
+        status = self.function(self.agent, **self.params)
+        return status
+
+# A decorator node that wraps a child behavior and calls a function.
+class FunctionDecorator(pt.decorators.Decorator):
+    def __init__(self, name, function, params, child):
+        super(FunctionDecorator, self).__init__(name=name, child=child)
+        self.function = function
+        self.params = params
+
+    def update(self):
+        # Ensure the child is updated.
+        self.decorated.tick_once()
+        child_status = self.decorated.status
+        # Call the decorator function (for side effects)
+        self.function(**self.params)
+        # For this example, we simply pass through the child's status.
+        return child_status
+
+# A control node that has one child and then calls a function.
+class FunctionControl(pt.behaviour.Behaviour):
+    def __init__(self, name, function, params, child):
+        super(FunctionControl, self).__init__(name=name)
+        self.function = function
+        self.params = params
+        self.child = child
+
+    def update(self):
+        self.child.tick_once()
+        return self.function(**self.params)
+
+# Define an AlwaysSuccess behavior to use when an unknown node is encountered.
+class AlwaysSuccess(pt.behaviour.Behaviour):
+    def __init__(self, name="AlwaysSuccess"):
+        super(AlwaysSuccess, self).__init__(name=name)
+    
+    def update(self):
+        return pt.common.Status.SUCCESS
+
+########################################################################
+# 4. Convert the Parsed Node Tree into a py_trees Behavior Tree
+########################################################################
+
+def build_behavior(node: Node, subtree_mapping: Dict[str, Node]) -> pt.behaviour.Behaviour:
+    """
+    Recursively converts a parsed Node (from XML) into a py_trees behavior.
+    """
+    # Special case: unwrap the BehaviorTree container.
+    if node.tag == "BehaviorTree":
+        if node.children:
+            return build_behavior(node.children[0], subtree_mapping)
+        else:
+            return AlwaysSuccess(name="Empty BehaviorTree")
+    
+    # Define which tags represent which kinds of nodes.
+    composite_tags = ["Sequence", "Fallback"]
+    repeat_tags = ["Repeat"]
+    decorator_tags = ["Inverter","AlwaysFailure", "AlwaysSuccess"]
+    control_tags = []
+
+    mapping = get_function_mapping()
+
+
+    if node.tag == "Sequence":
+        composite = pt.composites.Sequence(
+            name=node.attributes.get('name', 'Sequence'),
+            memory=True  # Added memory parameter
+        )
+        for child in node.children:
+            composite.add_child(build_behavior(child, subtree_mapping))
+        return composite
+
+    elif node.tag == "Fallback":
+        composite = pt.composites.Selector(
+            name=node.attributes.get('name', 'Fallback'),
+            memory=True  # Added memory parameter
+        )
+        for child in node.children:
+            composite.add_child(build_behavior(child, subtree_mapping))
+        return composite
+
+    elif node.tag in repeat_tags:
+        if len(node.children) != 1:
+            print("Repeat node must have exactly one child!")
+        child_behavior = build_behavior(node.children[0], subtree_mapping)
+        # Read the number of cycles from the XML; default to 1 if not provided.
+        num_cycles = int(node.attributes.get('num_cycles', 1))
+        # Create the Repeat decorator, providing the required 'num_success' parameter.
+        repeat_decorator = pt.decorators.Repeat(
+            name=node.attributes.get('name', 'Repeat'),
+            child=child_behavior,
+            num_success=num_cycles  # Provide the required parameter here.
+        )
+        return repeat_decorator
+
+
+    elif node.tag == "SubTree":
+        subtree_id = node.attributes.get('ID')
+        if subtree_id in subtree_mapping:
+            return build_behavior(subtree_mapping[subtree_id], subtree_mapping)
+        else:
+            print(f"SubTree with ID {subtree_id} not found!")
+            return AlwaysSuccess(name="Missing SubTree")
+
+    elif node.tag in decorator_tags:
+        if len(node.children) != 1:
+            print("Decorator node must have exactly one child!")
+        child_behavior = build_behavior(node.children[0], subtree_mapping)
+        params = {k: convert_param(v) for k, v in node.attributes.items() if k != "name"}
+        return FunctionDecorator(
+            name=node.attributes.get('name', node.tag),
+            function=mapping[node.tag],
+            params=params,
+            child=child_behavior
+        )
+
+    elif node.tag in control_tags:
+        if len(node.children) != 1:
+            print("Control node must have exactly one child!")
+        child_behavior = build_behavior(node.children[0], subtree_mapping)
+        params = {k: convert_param(v) for k, v in node.attributes.items() if k != "name"}
+        return FunctionControl(
+            name=node.attributes.get('name', node.tag),
+            function=mapping[node.tag],
+            params=params,
+            child=child_behavior
+        )
+
+    else:
+        if node.tag in mapping:
+            params = {k: convert_param(v) for k, v in node.attributes.items() if k != "name"}
+            return FunctionAction(
+                name=node.attributes.get('name', node.tag),
+                function=mapping[node.tag],
+                params=params
+            )
+        else:
+            return AlwaysSuccess(name=node.attributes.get('name', node.tag))
+
+
+########################################################################
+# 5. Main: Parse XML, Build the py_trees Tree, and Execute It
+########################################################################
+
+def print_node(node, indent=0):
+    ind = "  " * indent
+    print(f"{ind}{node.tag}: {node.attributes}")
+    # Optionally print any ports
+    for port_type, port_list in node.ports.items():
+        for port in port_list:
+            print(f"{ind}  {port_type}: {port}")
+    for child in node.children:
+        print_node(child, indent + 1)
+
+# Usage in your main:
+if __name__ == "__main__":
+    file_path = 'tree.xml'
+    trees = parse_behavior_trees(file_path)
+    for tree in trees:
+        print_node(tree)
+
+# if __name__ == "__main__":
+    # The XML file with your behavior tree.
+    # file_path = 'tree.xml'
+    
+    # # 1. Parse the XML into a list of BehaviorTree nodes.
+    # trees = parse_behavior_trees(file_path)
+    # # Build a mapping of BehaviorTree ID to Node.
+    # print(trees)
+    # subtree_mapping = { tree.attributes.get("ID"): tree for tree in trees }
+    # print()
+
+    # print(subtree_mapping)

requirements.txt

@@ -0,0 +1,15 @@
+huggingface-hub==0.33.0
+gradio==5.12.0
+numpy==1.26.4
+torch==2.5.1
+transformers==4.48.0
+llama_cpp_python==0.3.9
+pydub==0.25.1
+accelerate==1.2.1
+pygame==2.6.1
+violet-simulator==0.2.4
+pyttsx3==2.98
+py_trees==2.3.0
+
+blobfile==3.0.0
+sentencepiece==0.2.0

safety_module.py

@@ -0,0 +1,81 @@
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download
+
+
+# Download only the safety model shard from the model repo
+model_path = hf_hub_download(
+    repo_id="Inventors-Hub/SwarmChat-models",
+    repo_type="model",
+    filename="llama-guard-3-8b-q4_k_m.gguf",
+)
+
+
+# Load the Llama model for safety classification
+# model_path = r".\models\llama-guard-3-8b-q4_k_m.gguf"
+
+
+# llm = Llama(model_path=model_path, n_ctx=1024)
+llm = Llama(
+    model_path=model_path,
+    n_ctx=512,            # down from 4096
+    low_vram=True,         # llama.cpp low-vram mode
+    f16_kv=True,           # half-precision kv cache
+    use_mmap=True,         # mmap file
+    use_mlock=False,
+)
+
+
+def check_safety(text, checkbox):
+
+    if checkbox:
+        return "Safe"
+
+    categories = {
+        "S1": "Violent Crimes.",
+        "S2": "Non-Violent Crimes.",
+        "S3": "Sex-Related Crimes.",
+        "S4": "Child Sexual Exploitation.",
+        "S5": "Defamation.",
+        "S6": "Specialized Advice.",
+        "S7": "Privacy.",
+        "S8": "Intellectual Property.",
+        "S9": "Indiscriminate Weapons.",
+        "S10": "Hate.",
+        "S11": "Suicide & Self-Harm.",
+        "S12": "Sexual Content.",
+        "S13": "Elections.",
+        "S14": "Code Interpreter Abuse.",
+    }
+
+    input_prompt = f"""
+    <|begin_of_text|><|start_header_id|>user<|end_header_id|>
+
+    Task: Determine if the following text contains unsafe content according to our safety policy. Classify the content into one or more of the categories below if unsafe.
+
+    <BEGIN UNSAFE CONTENT CATEGORIES>
+    {', '.join([f'{key}: {value}' for key, value in categories.items()])}
+    <END UNSAFE CONTENT CATEGORIES>
+
+    Text: {text}
+
+    Provide your safety assessment:
+    - Respond with "safe" if the text is appropriate.
+    - If unsafe, respond with: "unsafe" followed by a comma-separated list of the violated categories (e.g., S1, S2).
+
+    <|eot_id|>
+    """
+
+    # Generate a response from the model
+    output = llm(input_prompt, temperature=0)
+    # Extract the model's response
+    response = output.get("choices", [{}])[0].get("text", "").strip()
+    print("\n\n", response, "\n\n")
+    # Parse the response
+    if response.lower().startswith("safe"):
+        return "Safe"
+    else:
+        unsafe_categories = categories[response.split("unsafe", 1)[-1].strip()]
+        return f"Unsafe: This prompt is categorized as '{unsafe_categories}'"
+
+        # unsafe_categories = categories[response.split("unsafe", 1)[-1].strip()]
+        # return f"Unsafe: This prompt categorized as '{unsafe_categories}'"

simulator_env.py

@@ -0,0 +1,453 @@
+import math
+import time
+import pygame as pg
+from vi import Agent, Config, Window, HeadlessSimulation
+from typing import Optional
+from queue import Queue
+import numpy as np
+from pygame.math import Vector2
+import py_trees as pt
+import parser
+import xml.etree.ElementTree as ET
+import threading
+# import pyttsx3
+
+class MyWindow(Window):
+    """Custom window class for simulation."""
+    def __init__(self, width=800, height=600):
+        super().__init__(width, height)
+
+
+
+
+class MyConfig(Config):
+    """Custom configuration for simulation."""
+    def __init__(self, radius=25, visualise_chunks=True, window=None, movement_speed=2.0):
+        super().__init__(
+            radius=radius,
+            visualise_chunks=visualise_chunks,
+            window=window or MyWindow(800, 600),
+            movement_speed=movement_speed
+        )
+
+
+
+
+class SwarmAgent(Agent):
+    def __init__(self, images, simulation, pos, nest_pos, target_pos):
+        super().__init__(images=images, simulation=simulation)
+        # Ensure the agent gets the configuration from the simulation.
+        self.config = simulation.config  
+        
+        self.pos = pos
+        self.nest_pos = nest_pos
+        self.target_pos = target_pos  
+        self.target_detected_flag = False
+        self.target_reached_flag = False
+        self.is_agent_in_nest_flag = False
+        self.obstacle_radius = 3
+        self.state = "seeking"
+        self.bt_active = True  # Add a flag
+        # self.tts_engine = pyttsx3.init()  # Initialize text-to-speech engine
+
+        file_path = "tree.xml"
+        trees = parser.parse_behavior_trees(file_path)
+        subtree_mapping = { tree.attributes.get("ID"): tree for tree in trees }
+
+        
+        xml_tree = ET.parse(file_path)
+        xml_root = xml_tree.getroot()
+        main_tree_id = xml_root.attrib.get("main_tree_to_execute")
+        
+        if not main_tree_id or main_tree_id not in subtree_mapping:
+            raise ValueError("Main tree not found in the XML!")
+        main_tree_node = subtree_mapping[main_tree_id]
+        
+        # Build the py_trees tree:
+        self.bt = parser.build_behavior(main_tree_node, subtree_mapping)
+        
+        # Inject the agent instance into all leaf behaviors.
+        self._inject_agent(self.bt)
+        
+    def _inject_agent(self, node):
+        """Recursively set the agent for any custom BT nodes."""
+        if hasattr(node, "agent"):
+            node.agent = self
+        if hasattr(node, "children"):
+            for child in node.children:
+                self._inject_agent(child)
+
+    def update(self):
+        if self.bt_active:
+            self.bt.tick_once()
+
+    def obstacle(self):
+        """
+        Check for obstacle intersections within a predefined radius.        
+        Returns: True if an obstacle is detected within the radius, False otherwise.
+        """
+        for intersection in self.obstacle_intersections(scale=self.obstacle_radius):
+            return True
+        return False
+
+
+    # def update(self):
+    #     self.bt.tick_once()
+    #     # self.root_node.run(self)
+
+    # def say(self, message: str):
+    #     """
+    #     Action Node: Speak the provided message using text-to-speech if it hasn't been spoken before.
+    #     Args: message (str): The message to be spoken.
+    #     Returns: Always returns SUCCESS, indicating the action was executed.
+    #     """
+    #     if not hasattr(self, 'old_message'):
+    #         self.old_message = []
+                
+    #     # Only speak the message if it has not been spoken before (i.e. not in old_message)
+    #     if message not in self.old_message:
+    #         self.tts_engine.say(message)
+    #         self.tts_engine.runAndWait()
+    #         self.old_message.append(message)
+            
+    #     return pt.common.Status.SUCCESS
+
+    def flocking(self):
+        """
+        Action Node: Adjust the agent's move vector by blending alignment and separation forces from nearby agents.
+        Returns: Always returns SUCCESS, indicating the action was executed.
+        """
+        nearby_agents = list(self.in_proximity_accuracy().without_distance())
+        if not nearby_agents:
+            return pt.common.Status.SUCCESS
+
+        alignment = Vector2(0, 0)
+        separation = Vector2(0, 0)
+        separation_count = 0
+
+        # Desired minimum separation distance (adjust as needed)
+        separation_threshold = 3
+
+        # Calculate alignment and separation contributions.
+        for other in nearby_agents:
+            alignment += other.move
+            
+            diff = self.pos - other.pos
+            distance = diff.length()
+            if 0 < distance < separation_threshold:
+                # The closer the neighbor, the stronger the repulsive force.
+                separation += diff.normalize() * (separation_threshold - distance)
+                separation_count += 1
+
+        # Average the alignment vector over all neighbors.
+        alignment /= len(nearby_agents)
+        
+        # If any agents are too close, average the separation vector.
+        if separation_count > 0:
+            separation /= separation_count
+
+        # Blend the two influences. Here, alignment has a stronger influence than separation.
+        # Adjust the blend factor (e.g., 0.3) to control separation influence.
+        blended_force = alignment.lerp(separation, 0.3)
+        
+        # Smoothly blend the current move with the blended force.
+        self.move = self.move.lerp(blended_force, 0.5)
+        
+        # Normalize and scale to the configured movement speed.
+        if self.move.length() > 0:
+            self.move = self.move.normalize() * self.config.movement_speed
+
+        # Update position and apply wrap-around if necessary.
+        self.pos += self.move
+        self.there_is_no_escape()
+        
+        return pt.common.Status.SUCCESS
+
+
+    
+    def align_with_swarm(self):
+        """
+        Action Node: Align the agent's move vector with the average movement of nearby agents.
+        Returns: Always returns SUCCESS, indicating the action was executed.
+        """
+        nearby_agents = list(self.in_proximity_accuracy().without_distance())
+        if not nearby_agents:
+            return pt.common.Status.SUCCESS
+
+        avg_direction = Vector2(0, 0)
+        for other in nearby_agents:
+            avg_direction += other.move
+        avg_direction /= len(nearby_agents)
+        
+        # Blend current movement with average direction.
+        self.move = self.move.lerp(avg_direction, 0.5)
+        if self.move.length() > 0:
+            self.move = self.move.normalize() * self.config.movement_speed
+
+        # Update position and wrap-around if necessary.
+        self.pos += self.move
+        self.there_is_no_escape()
+        
+        return pt.common.Status.SUCCESS
+ 
+
+    def is_obstacle_detected(self):
+        """
+        Condition node: Determine if any obstacles are detected in the vicinity of the agent. 
+        Returns: SUCCESS if an obstacle is detected, FAILURE otherwise.
+        """
+        if self.obstacle():
+            return pt.common.Status.SUCCESS
+        else:
+            return pt.common.Status.FAILURE
+
+
+    def avoid_obstacle(self):
+        """
+        Action node: Execute an action to avoid detected obstacles. 
+        Returns: Always returns SUCCESS, indicating the action was executed.
+        """
+        self.move.rotate_ip(180)
+        return pt.common.Status.SUCCESS
+    
+    def is_target_detected(self):
+        """
+        Condition node: Check if the target is within a detectable distance from the agent's position. 
+        Returns: SUCCESS if the target is within 20 units of distance, FAILURE otherwise.
+        """
+        distance = math.dist(self.target_pos, self.pos)
+        if distance <= 20:
+            self.target_detected_flag = True        
+        if self.target_detected_flag:
+            return pt.common.Status.SUCCESS
+        return pt.common.Status.FAILURE
+        
+    
+    def is_target_reached(self):
+        """
+        Condition node: Check if the agent has reached the target. 
+        Returns: SUCCESS if the target is within 15 units of distance, FAILURE otherwise.
+        """
+        distance = math.dist(self.target_pos, self.pos)
+        if distance <= 15:
+            self.target_reached_flag = True        
+        if self.target_reached_flag:            
+            return pt.common.Status.SUCCESS
+        return pt.common.Status.FAILURE
+    
+    def change_color(self, color):
+        """
+        Action Node: Change the agent's color to 'white', 'green', or 'red'.
+        Args: color (str): Color name.
+        Returns: Always returns SUCCESS, indicating the action was executed.
+        """
+        color = color.lower()
+        if color == "white":
+            self.change_image(0)
+        elif color == "green":
+            self.change_image(1)
+        elif color == "red":
+            self.change_image(2)
+
+        return pt.common.Status.SUCCESS
+        
+
+    def is_agent_in_nest(self):
+        """
+        Condition node: Determine if the agent is in the nest.
+        Returns: SUCCESS if the agent is in the nest, FAILURE otherwise.
+        """
+        distance = math.dist(self.nest_pos, self.pos)
+        if distance <= 17 and (self.target_reached_flag==True or self.target_detected_flag == True or self.state == "completed" ) :
+            self.state = "seeking"
+            # self.target_detected_flag = False
+            # self.target_reached_flag = False   
+            self.is_agent_in_nest_flag = True
+
+        if self.is_agent_in_nest_flag:
+            return pt.common.Status.SUCCESS
+        return pt.common.Status.FAILURE
+
+
+    def agent_movement_freeze(self):
+        """
+        Action node: Freeze the agent's movement, typically to indicate a stop in activity.
+        Returns: Always returns SUCCESS, indicating the action was executed.
+        """
+        self.freeze_movement()
+        return pt.common.Status.SUCCESS
+    
+    def continue_movement_agent(self):
+        """
+        Action node: Continue the agent's movement after it has been previously frozen.
+        Returns: Always returns SUCCESS, indicating the action was executed.
+        """
+        self.continue_movement()
+        return pt.common.Status.SUCCESS
+
+    def move_randomly(self):
+        """
+        Action node: Perform a wandering action where the agent moves randomly within the environment.
+        Returns: Always returns SUCCESS, indicating the action was executed.
+        """
+        Agent.change_position(self)
+        return pt.common.Status.SUCCESS
+
+    def is_path_clear(self):
+        """
+        Condition node: Check if the path ahead of the agent is clear of obstacles.
+        Returns: SUCCESS if no obstacles are detected ahead, FAILURE if obstacles are present.
+        """
+        # return not self.obstacle()
+
+        if not self.obstacle():
+            return pt.common.Status.SUCCESS
+        else:
+            return pt.common.Status.FAILURE
+    
+    def is_line_formed(self):
+        """
+        Condition node: Determine if the agent has formed a line with a reference point at the center of the window.
+        Returns: SUCCESS if the line is formed with the center, FAILURE otherwise.
+        """
+        center_x = self.config.window.width / 2
+        direction = Vector2(center_x, self.pos.y) - self.pos
+        if direction.length() > 0.5:
+            return pt.common.Status.FAILURE        
+        return pt.common.Status.SUCCESS
+
+    def form_line(self):
+        """
+        Action node: Direct the agent to form a line towards the center of the window. This function adjuststhe agent's position to align it with the center.
+        Returns: Always returns SUCCESS, indicating the action was executed.
+        """
+        # print("form_line")
+        center_x = self.config.window.width / 2
+        direction = Vector2(center_x, self.pos.y) - self.pos
+        if direction.length() > 0.5:
+            direction.scale_to_length(self.config.movement_speed)
+            self.pos += direction     
+        return pt.common.Status.SUCCESS
+    
+    # def task_completed(self):
+    #     """
+    #     Action node: Signal that the agent has completed its designated task. Returns: Always returns True, indicating that the task completion action was executed.
+    #     """
+    #     self.state = "completed"
+    #     return pt.common.Status.SUCCESS
+    
+
+
+
+
+
+
+
+
+class StreamableSimulation(HeadlessSimulation):
+    """Modified Simulation class that captures frames for streaming."""
+    def __init__(self, config: Optional[Config] = None):
+        super().__init__(config)
+        pg.init()
+
+        size = self.config.window.as_tuple()
+        self._screen = pg.Surface(size, pg.SRCALPHA)
+        self._background = pg.Surface(size, pg.SRCALPHA)
+        self._background.fill((0, 0, 0))
+
+        self.frame_queue = Queue(maxsize=30)
+        self.running = True
+        self._frame_lock = threading.Lock()
+
+    
+    def get_frame(self):
+        with self._frame_lock:
+            surf_copy = self._screen.copy()
+            frame = np.array(pg.surfarray.pixels3d(surf_copy))
+            return np.transpose(frame, (1, 0, 2))
+
+    def tick(self):
+        """Run a simulation step and capture frames."""
+        super().tick()
+
+        with self._frame_lock:
+            self._screen.blit(self._background, (0, 0))
+            for sprite in self._all.sprites():
+                self._screen.blit(sprite.image, sprite.rect)
+
+        try:
+            frame = self.get_frame()
+            self.frame_queue.put(frame, block=False)
+        except Queue.Full:
+            print("Frame queue is full. Dropping frame.")
+
+    # def _load_image(self, path: str) -> pg.surface.Surface:
+    #     """Load an image from the given path."""
+    #     return pg.image.load(path)
+    
+    def _load_image(self, paths):
+        """Load one or more images from given paths."""
+        if isinstance(paths, str):  # If it's a single string, load normally
+            return pg.image.load(paths)
+        elif isinstance(paths, list):  # If it's a list, load all images
+            return [pg.image.load(path) for path in paths]
+        raise TypeError("Expected a string (file path) or a list of file paths")
+
+    def stop(self):
+        """Stop the simulation."""
+        # Do not try to call self.bt.stop() because simulation does not own a BT.
+        # self.running = False
+        super().stop()
+        pg.quit()       # Quit the Pygame environment
+        
+
+
+
+
+
+# if __name__=="__main__":
+
+#     # Define nest and target positions
+#     nest_x, nest_y = 450, 400
+#     target_x, target_y = 200, 100
+#     nest_pos = Vector2(nest_x, nest_y)
+#     target_pos = Vector2(target_x, target_y)
+
+#     # Load images for agents
+#     agent_images_paths = ["./images/white.png", "./images/green.png", "./images/red circle.png"]
+
+#     config = MyConfig(radius=250, visualise_chunks=True, movement_speed=2)
+#     sim = StreamableSimulation(config=config)
+
+#     # Load images
+#     loaded_agent_images = sim._load_image(agent_images_paths)
+
+
+
+#     # Initialize agents with behavior tree parsing
+#     for _ in range(50):
+#         agent = SwarmAgent(
+#             images=loaded_agent_images,
+#             simulation=sim,
+#             pos=Vector2(nest_x, nest_y),
+#             nest_pos=nest_pos,
+#             target_pos=target_pos,
+#         )
+#         sim._agents.add(agent)
+#         sim._all.add(agent)
+#     # Draw environment elements
+#     sim.spawn_obstacle("./images/rect_obst.png", 350, 100)
+#     sim.spawn_obstacle("./images/rect_obst (1).png", 100, 350)
+#     sim.spawn_site("./images/rect.png", target_x, target_y)
+#     sim.spawn_site("./images/nest.png", nest_x, nest_y)
+
+#     for agent in sim._agents:
+#         agent.bt.tick_once()
+    
+#     # Then run your simulation loop without ticking the BT further.
+#     while sim.running:
+#         sim.tick()
+#         if not sim.frame_queue.empty():
+#             frame = sim.frame_queue.get()
+#             # update_frame(frame) or display the frame as needed.
+#         time.sleep(1/30)

speech_processing.py

@@ -0,0 +1,40 @@
+from transformers import SeamlessM4Tv2Model, AutoProcessor
+import numpy as np
+import torch
+from pydub import AudioSegment
+
+# Load processor and model
+processor = AutoProcessor.from_pretrained("facebook/seamless-m4t-v2-large")
+model = SeamlessM4Tv2Model.from_pretrained("facebook/seamless-m4t-v2-large")
+
+def translate_audio(audio_file):
+    if audio_file is None:
+        return "No audio file detected. Please try again."
+    
+    try:
+        # Set the device (use GPU if available)
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        model.to(device)
+
+        # Reset audio file pointer and load audio
+        audio = AudioSegment.from_file(audio_file, format="wav")
+        audio = audio.set_frame_rate(16000).set_channels(1)
+
+        # Convert audio to float32 NumPy array
+        audio_array = np.array(audio.get_array_of_samples()).astype(np.float32) / 32768.0
+
+        # Process input
+        audio_inputs = processor(audios=audio_array, sampling_rate=16000, return_tensors="pt")
+        audio_inputs = {key: val.to(device) for key, val in audio_inputs.items()}  # Ensure tensors are on the correct device
+
+        # Generate translation
+        output_tokens = model.generate(**audio_inputs, tgt_lang="eng", generate_speech=False)
+
+        # Extract token IDs from the generated output
+        token_ids = output_tokens.sequences
+        # Decode token IDs to text
+        translated_text_from_audio = processor.batch_decode(token_ids, skip_special_tokens=True)[0]
+
+        return translated_text_from_audio
+    except Exception as e:
+        return f"Error during audio translation: {e}"

text_processing.py

@@ -0,0 +1,40 @@
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download
+
+# Download the single GGUF shard by its repo path:
+model_path = hf_hub_download(
+    repo_id="Inventors-Hub/SwarmChat-models",
+    repo_type="model",
+    filename="EuroLLM-9B-Instruct-Q4_K_M.gguf",
+)
+
+# llm = Llama(model_path=model_path, n_ctx=1024)#, verbose=True)
+llm = Llama(
+    model_path=model_path,
+    n_ctx=512,            # down from 4096
+    low_vram=True,         # llama.cpp low-vram mode
+    f16_kv=True,           # half-precision kv cache
+    use_mmap=True,         # mmap file
+    use_mlock=False,
+)
+# print("Llama backend initialized successfully!")
+
+
+
+# Function to process text using EuroLLM
+def translate_text(text):
+    input_prompt = f"""
+    <|im_start|>system
+    <|im_end|>
+    <|im_start|>user
+    Translate the following text to English:
+    Text: {text}
+    English: 
+    <|im_end|>
+    <|im_start|>assistant
+    """
+    output = llm(input_prompt, max_tokens=1024, temperature=0)
+
+    translated_text = output.get("choices", [{}])[0].get("text", "").strip()
+
+    return translated_text