Tambahkan fitur, apabila botnya berbicara, nah kita berbicara juga, maka langsung berhenti dan langsung merekam suara kita lho, perbaiki!

main.py
import logging
import threading
import urllib.request
import webbrowser
from fastapi import FastAPI, File, UploadFile, Form, Cookie
from fastapi.responses import HTMLResponse, JSONResponse
from starlette.middleware.cors import CORSMiddleware
from starlette.staticfiles import StaticFiles
from Chatbot import Chatbot
import cv2
import numpy as np
import uvicorn

from pyngrok import ngrok

logging.disable(logging.CRITICAL)


class ServerApp:
    def __init__(self):
        self.app = FastAPI()
        self.chatbot = Chatbot()
        self.ngrok_process = None
        self.public_url = None
        self.setup_routes_and_middlewares()

    def setup_routes_and_middlewares(self):
        self.app.add_middleware(
            CORSMiddleware,
            allow_origins=["*"],
            allow_credentials=True,
            allow_methods=["*"],
            allow_headers=["*"]
        )

        directories = {
            "/app": "app",
            "/assets": "app/assets",
            "/js": "app/js",
            "/temp": "temp",
            "/model/live2d": "model/live2d",
            "/CSS": "app/CSS"
        }
        for mount_point, directory in directories.items():
            self.app.mount(mount_point, StaticFiles(directory=directory), name=mount_point.strip("/"))

        self.app.get("/")(self.index)
        self.app.get("/api/audio_status")(self.get_audio_status)
        self.app.post("/api/reset_audio_status")(self.reset_audio_status)
        self.app.post("/api/upload_frame")(self.upload_frame)
        self.app.post("/api/upload_audio")(self.upload_audio)
        self.app.post("/api/start_ngrok")(self.start_ngrok)
        self.app.post("/api/stop_ngrok")(self.stop_ngrok)

    @staticmethod
    async def index(ngrok_api_key: str = Cookie(default=None)):
        with open('app/index.html', 'r') as f:
            html_content = f.read()
        headers = {"ngrok_api_key": ngrok_api_key} if ngrok_api_key else {}
        return HTMLResponse(content=html_content, headers=headers)

    async def get_audio_status(self):
        return {"audio_ready": self.chatbot.audio_ready}

    async def reset_audio_status(self):
        self.chatbot.audio_ready = False
        return {"audio_ready": self.chatbot.audio_ready}

    async def upload_frame(self, image: UploadFile = File(...)):
        try:
            image_bytes = await image.read()
            frame = cv2.imdecode(np.frombuffer(image_bytes, np.uint8), cv2.IMREAD_COLOR)
            self.chatbot.process_frame(frame)
            return JSONResponse(content={"message": "Frame processed successfully"}, status_code=200)
        except Exception as e:
            logging.error(f"Error processing frame: {e}")
            return JSONResponse(content={"message": "Failed to process frame"}, status_code=500)

    async def upload_audio(self, audio: UploadFile = File(...)):
        try:
            audio_bytes = await audio.read()
            self.chatbot.process_audio(audio_bytes)
            return JSONResponse(content={"message": "Audio processed successfully"}, status_code=200)
        except Exception as e:
            logging.error(f"Error processing audio: {e}")
            return JSONResponse(content={"message": "Failed to process audio"}, status_code=500)

    async def start_ngrok(self, api_key: str = Form(...)):
        if not self.check_internet_connection():
            return JSONResponse(content={"message": "Tidak ada koneksi internet."}, status_code=500)

        if self.ngrok_process:
            return JSONResponse(content={"message": "Ngrok sudah berjalan.", "public_url": self.public_url},
                                status_code=200)

        try:
            ngrok.set_auth_token(api_key)
            tunnel = ngrok.connect(8000)
            self.public_url = tunnel.public_url
            self.ngrok_process = ngrok.get_ngrok_process()
            threading.Thread(target=self.ngrok_process.proc.wait).start()

            return JSONResponse(content={"message": "Ngrok berhasil dimulai.", "public_url": self.public_url},
                                status_code=200)
        except Exception as e:
            logging.error(f"Error starting ngrok: {e}")
            return JSONResponse(content={"message": f"Error starting ngrok: {str(e)}"}, status_code=500)

    async def stop_ngrok(self):
        if self.ngrok_process:
            try:
                ngrok.kill()
                self.ngrok_process = None
                self.public_url = None
                return JSONResponse(content={"message": "Ngrok berhasil dihentikan."}, status_code=200)
            except Exception as e:
                logging.error(f"Error stopping ngrok: {e}")
                return JSONResponse(content={"message": f"Error stopping ngrok: {str(e)}"}, status_code=500)

        return JSONResponse(content={"message": "Ngrok tidak berjalan."}, status_code=400)

    @staticmethod
    def check_internet_connection():
        try:
            urllib.request.urlopen('https://google.com', timeout=5)
            return True
        except Exception:
            return False

    def run(self):
        threading.Thread(target=self.open_browser).start()
        uvicorn.run(self.app, host="localhost", port=8000)

    @staticmethod
    def open_browser():
        webbrowser.open_new("http://localhost:8000")


if __name__ == "__main__":
    server_app = ServerApp()
    server_app.run()
setup.js
import ModelController from './model.js';

(async () => {
    const app = new PIXI.Application({
        view: document.getElementById("canvas"),
        autoStart: true,
        resizeTo: window
    });
    const modelController = new ModelController(app, "../model/live2d/vixevia.model3.json");
    await modelController.loadModel();
    modelController.startBlinking();

    const statusDiv = document.getElementById('status');
    const audioLink = "/temp/response.wav";
    let audioPlaying = false;
    let isProcessing = false;
    let chunks = [];

    const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true });
    const video = document.createElement('video');
    video.srcObject = stream;
    video.autoplay = true;
    video.addEventListener('canplay', () => {
        const canvas = document.createElement('canvas');
        const context = canvas.getContext('2d');
        (function captureFrame() {
            if (video.readyState === 4) {
                canvas.width = video.videoWidth;
                canvas.height = video.videoHeight;
                context.drawImage(video, 0, 0, canvas.width, canvas.height);
                canvas.toBlob(blob => blob && fetch('/api/upload_frame', {
                    method: 'POST',
                    body: new FormData().append('image', blob, 'frame.jpg')
                }), 'image/jpeg');
            }
            setTimeout(captureFrame, 2500);
        })();
    });

    const mediaRecorder = new MediaRecorder(stream);
    mediaRecorder.addEventListener('dataavailable', e => chunks.push(e.data));
    mediaRecorder.addEventListener('stop', () => {
        const blob = new Blob(chunks, { type: 'audio/wav' });
        chunks = [];
        fetch('/api/upload_audio', {
            method: 'POST',
            body: new FormData().append('audio', blob, 'audio.wav')
        }).then(() => {
            statusDiv.textContent = "";
            isProcessing = false;
            (async function playAudioWhenReady() {
                while (!(await fetch('/api/audio_status').then(res => res.json()).then(data => data.audio_ready))) {
                    await new Promise(r => setTimeout(r, 500));
                }
                audioPlaying = true;
                const audio = new Audio(audioLink);
                audio.addEventListener('ended', () => {
                    fetch('/api/reset_audio_status', { method: 'POST' });
                    audioPlaying = false;
                });
                audio.play().then(() => {
                    const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
                    const analyser = audioCtx.createAnalyser();
                    const source = audioCtx.createMediaElementSource(audio);
                    source.connect(analyser);
                    analyser.connect(audioCtx.destination);
                    const dataArray = new Uint8Array(analyser.frequencyBinCount);
                    (function analyseVolume() {
                        analyser.getByteFrequencyData(dataArray);
                        modelController.setMouthOpenY(dataArray.reduce((a, b) => a + b, 0) / dataArray.length / 255);
                        if (!audio.paused) requestAnimationFrame(analyseVolume);
                    })();
                });
            })();
        });
    });

    vad.MicVAD.new({
        onSpeechStart: () => {
            if (!isProcessing && !audioPlaying) {
                mediaRecorder.start();
                statusDiv.textContent = "Listening...";
            }
        },
        onSpeechEnd: () => {
            if (!isProcessing && !audioPlaying) {
                mediaRecorder.stop();
                isProcessing = true;
                statusDiv.textContent = "Processing...";
            }
        }
    }).start();
})();
Chatbot.py
import json
import re
from datetime import datetime
from pathlib import Path
import base64

import cv2
import google.generativeai as genai
from google.generativeai.types import generation_types
from gtts import gTTS
from so_vits_svc_fork.inference.main import infer

from Config import Config


class Chatbot:
    def __init__(self):
        self.config = Config()
        self.api_keys = self._load_api_keys()
        if not self.api_keys:
            raise ValueError("API key list is empty. Please provide at least one API key.")
        self.api_key_index = 0
        self.system_prompt = self._load_system_prompt()
        self.generation_config, self.safety_settings = self._model_config()
        self.model = self._get_text_model()
        self.history = self._load_history()
        self.convo = self._start_conversation()
        self._ensure_system_prompt()
        self.frames = []
        self.audio_ready = False

    def _load_api_keys(self):
        with open(self.config.FILES.API_KEY, 'r') as f:
            keys = [line.strip() for line in f if line.strip()]
            print(f"Loaded {len(keys)} API keys.")
            return keys

    def _load_system_prompt(self):
        with open(self.config.FILES.SYSTEM_PROMPT, 'r') as f:
            return f.read().strip()

    def _model_config(self):
        generation_config = generation_types.GenerationConfig(
            temperature=self.config.TEMPERATURE,
            top_p=self.config.TOP_P,
            top_k=self.config.TOP_K,
            max_output_tokens=self.config.MAX_OUTPUT_TOKENS,
        )
        safety_settings = [{"category": cat, "threshold": self.config.BLOCK_NONE} for cat in
                           self.config.HARM_CATEGORIES]
        return generation_config, safety_settings

    @staticmethod
    def _mask_api_key(key):
        if len(key) > 6:
            return key[:-6] + '*' * 6
        return '*' * len(key)

    def _configure_genai(self):
        current_api_key = self.api_keys[self.api_key_index]
        genai.configure(api_key=current_api_key)
        masked_key = self._mask_api_key(current_api_key)
        print(f"Using API key index: {self.api_key_index} (Key: {masked_key})")

    def _get_model(self, model_name, config):
        self._configure_genai()
        return genai.GenerativeModel(
            model_name=model_name,
            generation_config=config,
            safety_settings=self.safety_settings,
            system_instruction=self.system_prompt
        )

    def _get_text_model(self):
        return self._get_model(self.config.MODEL_NAME, self.generation_config)

    def _load_history(self):
        session_path = Path(self.config.FILES.SESSION)
        if session_path.exists():
            try:
                with session_path.open('r', encoding='utf-8') as f:
                    history = json.load(f)
                    if isinstance(history, list):
                        print(f"Loaded history with {len(history)} messages.")
                        return history
            except json.JSONDecodeError:
                print("Session file corrupted or empty. Starting a new conversation.")
        return []

    def _save_history(self):
        session_path = Path(self.config.FILES.SESSION)
        try:
            with session_path.open('w', encoding='utf-8') as f:
                json.dump(self.history, f, ensure_ascii=False, indent=4)
            print(f"History saved with {len(self.history)} messages.")
        except Exception as e:
            print(f"Failed to save history: {e}")

    def _start_conversation(self):
        return self.model.start_chat(history=self.history)

    def _ensure_system_prompt(self):
        if not self.history:
            try:
                system_message = {"role": "user", "parts": [{"text": self.system_prompt}]}
                self.convo.send_message(system_message)
                self.history.append(system_message)
                self._save_history()
                print("System prompt initialized.")
            except Exception as e:
                print(f"Exception during system prompt initialization: {e}. Retrying...")
                self._rotate_api_key()
                self._ensure_system_prompt()

    def _rotate_api_key(self):
        previous_index = self.api_key_index
        self.api_key_index = (self.api_key_index + 1) % len(self.api_keys)
        if self.api_key_index == previous_index:
            print("All API keys have been exhausted.")
        else:
            print(f"Rotated API key to index {self.api_key_index}.")

    def _handle_response(self, user_input):
        max_attempts = len(self.api_keys)
        for attempt in range(max_attempts):
            try:
                response_chunks = self.convo.send_message(user_input)
                response = "".join(chunk.text for chunk in response_chunks)
                break
            except Exception as e:
                error_message = str(e).lower()
                print(f"Exception occurred: {e}.")
                if '429' in error_message or 'quota' in error_message or 'exhausted' in error_message:
                    print(f"Error 429 detected. Rotating API key. Attempt {attempt + 1}/{max_attempts}")
                    self._rotate_api_key()
                    self.model = self._get_text_model()
                else:
                    print("Non-rate limit error encountered. Retrying with the same API key.")
        else:
            print("All API keys have been exhausted. Failed to send message.")
            return

        response = re.sub(r'\(.*?\)', '', response).strip()
        if response:
            try:
                response_mp3_path = Path(self.config.FILES.RESPONSE_MP3)
                response_mp3_path.unlink(missing_ok=True)
                gTTS(text=response, lang='id').save(response_mp3_path)
                print(f"Saved response audio to {response_mp3_path}")

                infer(
                    input_path=response_mp3_path,
                    output_path=Path(self.config.FILES.RESPONSE_WAV),
                    model_path=Path(self.config.FILES.MODEL_PATH),
                    config_path=Path(self.config.FILES.CONFIG_PATH),
                    max_chunk_seconds=self.config.MAX_CHUNK_SECONDS,
                    device=self.config.DEVICE,
                    speaker="",
                    transpose=7,
                )
                self.audio_ready = True
                print("Audio inference completed.")
            except Exception as e:
                print(f"Failed to process audio: {e}")

        self.history.append(user_input)
        assistant_message = {"role": "model", "parts": [{"text": response}]}
        self.history.append(assistant_message)

        self._save_history()

    def process_frame(self, frame):
        self.frames.append(frame)
        print(f"Frame added. Total frames: {len(self.frames)}")

    def process_audio(self, audio_bytes):
        if not self.frames:
            print("No frames to process.")
            return

        try:
            encoded_frames = [
                {
                    "mime_type": "image/jpeg",
                    "data": base64.b64encode(cv2.imencode('.jpg', frame)[1]).decode('utf-8')
                }
                for frame in self.frames
            ]
            print(f"Encoded {len(encoded_frames)} frames.")
        except Exception as e:
            print(f"Failed to encode frames: {e}")
            encoded_frames = []

        try:
            encoded_audio = base64.b64encode(audio_bytes).decode('utf-8')
            print("Audio encoded successfully.")
        except Exception as e:
            print(f"Failed to encode audio: {e}")
            encoded_audio = ""

        user_input = {
            "role": "user",
            "parts": [
                {"text": "Waktu: " + datetime.now().strftime("%Y-%m-%d %H:%M:%S")},
                {"text": "Penglihatan Vixevia:"},
                *encoded_frames,
                {"text": "Pendengaran Vixevia:"},
                {"mime_type": "audio/wav", "data": encoded_audio},
            ],
        }
        self._handle_response(user_input)
        self.frames.clear()
        print("Frames cleared after processing audio.")
Tuliskan lengkap tanpa komentar. Tuliskan tanpa redudansi seperti senior programmer!