Project structure:

ArsenAI/
├── audio_files/
│   ├── audio.wav
│   ├── recording.wav
│   ├── recording_1723749763.wav
│   ├── recording_1723750110.wav
│   ├── recording_1724005566.wav
│   ├── recording_1724005614.wav
│   ├── recording_1724006017.wav
│   ├── recording_1724015867.wav
│   ├── recording_1724015888.wav
│   ├── recording_1724015903.wav
│   ├── recording_1724018743.wav
│   ├── recording_1724067849.wav
│   ├── recording_1724067866.wav
│   ├── recording_1724067927.wav
│   ├── recording_1724067932.wav
│   ├── recording_1724067963.wav
│   ├── recording_1724067966.wav
│   ├── recording_1724068241.wav
│   ├── recording_1724068253.wav
│   ├── recording_1724068312.wav
│   ├── recording_1724068316.wav
│   ├── recording_1724068423.wav
│   ├── recording_1724068431.wav
│   ├── recording_1724069008.wav
│   ├── recording_1724069016.wav
│   ├── recording_1724069337.wav
│   ├── recording_1724069342.wav
│   ├── recording_1724088967.wav
│   └── recording_1724089423.wav
├── processed_audio_chunks/
│   └── processed_chunks_1724089423.wav
├── static/
│   ├── css/
│   │   └── style.css
│   ├── js/
│   │   ├── script.js
│   │   └── worklet.js
├── templates/
│   └── index.html
├── test_audio_temp/
│   ├── chunk_1724069008.wav
│   ├── chunk_1724069010.wav
│   ├── chunk_1724069011.wav
│   ├── chunk_1724069012.wav
│   ├── chunk_1724069013.wav
│   ├── chunk_1724069014.wav
│   ├── chunk_1724069015.wav
│   ├── chunk_1724069016.wav
│   ├── chunk_1724069337.wav
│   ├── chunk_1724069338.wav
│   ├── chunk_1724069339.wav
│   ├── chunk_1724069340.wav
│   ├── chunk_1724069341.wav
│   ├── chunk_1724069342.wav
│   ├── vad_input_1724088931.wav
│   ├── vad_input_1724088932.wav
│   ├── vad_input_1724088933.wav
│   ├── vad_input_1724088959.wav
│   ├── vad_input_1724088962.wav
│   ├── vad_input_1724088963.wav
│   ├── vad_input_1724088964.wav
│   ├── vad_input_1724088965.wav
│   ├── vad_input_1724088966.wav
│   ├── vad_input_1724088967.wav
│   ├── vad_input_1724089417.wav
│   ├── vad_input_1724089420.wav
│   ├── vad_input_1724089421.wav
│   ├── vad_input_1724089422.wav
│   ├── vad_input_1724089423.wav
│   └── vad_input_1729341948.wav
├── app.py
├── audio_handler.py
└── main.py


Project files:

app.py
```py
import os
import numpy as np
import torch
import wave
from flask import Flask, render_template
from flask_socketio import SocketIO, emit
from collections import deque
import librosa
import time
import threading

# Constants
INPUT_SAMPLE_RATE = 48000
VAD_SAMPLE_RATE = 16000
CHANNELS = 1
CHUNK_SIZE = 512  # 60 ms at 16 kHz
UPLOAD_FOLDER = 'audio_files'
TEMP_AUDIO_FOLDER = 'test_audio_temp'
PROCESSED_AUDIO_FOLDER = 'processed_audio_chunks'
PROCESSING_INTERVAL = 0.1  # 100 ms

app = Flask(__name__)
app.config['SECRET_KEY'] = 'secret!'
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['TEMP_AUDIO_FOLDER'] = TEMP_AUDIO_FOLDER
app.config['PROCESSED_AUDIO_FOLDER'] = PROCESSED_AUDIO_FOLDER
socketio = SocketIO(app, async_mode='threading')

# Device selection (CPU or GPU)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Initialize Silero VAD
vad_model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad',
                                  model='silero_vad',
                                  force_reload=True)
vad_model = vad_model.to(device)
vad_model.eval()

# Global variables
audio_buffer = deque(maxlen=CHUNK_SIZE)
speech_detected = False
previous_speech_detected = False
last_processed_time = 0
processing_lock = threading.Lock()

# Buffer to store all audio chunks for testing
processed_audio_chunks = []

def ensure_folder_exists(folder):
    if not os.path.exists(folder):
        os.makedirs(folder)

def int2float(sound):
    abs_max = np.abs(sound).max()
    sound = sound.astype('float32')
    if abs_max > 0:
        sound *= 1 / 32768
    sound = sound.squeeze()
    return sound

def save_audio_chunk_to_file(audio_chunk, folder, filename):
    file_path = os.path.join(folder, filename)
    int_data = np.int16(audio_chunk * 32767)
    with wave.open(file_path, 'wb') as wf:
        wf.setnchannels(CHANNELS)
        wf.setsampwidth(2)
        wf.setframerate(VAD_SAMPLE_RATE)
        wf.writeframes(int_data.tobytes())
    print(f"Saved chunk to {file_path}")

def save_processed_audio_chunks():
    print(f"---save_processed_audio_chunks")
    global processed_audio_chunks

    if processed_audio_chunks:
        filename = f'processed_chunks_{int(time.time())}.wav'
        file_path = os.path.join(app.config['PROCESSED_AUDIO_FOLDER'], filename)

        # Concatenate all chunks
        all_chunks = np.concatenate(processed_audio_chunks)
        int_data = np.int16(all_chunks * 32767)

        with wave.open(file_path, 'wb') as wf:
            wf.setnchannels(CHANNELS)
            wf.setsampwidth(2)
            wf.setframerate(VAD_SAMPLE_RATE)
            wf.writeframes(int_data.tobytes())

        print(f"Processed audio saved to {file_path}")
        processed_audio_chunks.clear()
    else:
        print("No processed audio chunks to save.")

def detect_speech(audio_chunk):
    global speech_detected, vad_model, device, processed_audio_chunks

    # Convert to float
    audio_float32 = int2float(audio_chunk)

    # Save the chunk for debugging
    save_audio_chunk_to_file(audio_float32, app.config['TEMP_AUDIO_FOLDER'], f'vad_input_{int(time.time())}.wav')

    # Store the chunk in the processed audio buffer
    processed_audio_chunks.append(audio_float32)

    try:
        with torch.no_grad():
            tensor = torch.from_numpy(audio_float32).to(device).unsqueeze(0)
            speech_prob = vad_model(tensor, VAD_SAMPLE_RATE).item()
            print(f"speech_prob = {speech_prob}")
    except RuntimeError as e:
        if "out of memory" in str(e):
            print("CUDA out of memory. Falling back to CPU.")
            torch.cuda.empty_cache()
            device = torch.device('cpu')
            vad_model = vad_model.to(device)
            tensor = torch.from_numpy(audio_float32).to(device).unsqueeze(0)
            speech_prob = vad_model(tensor, VAD_SAMPLE_RATE).item()
        else:
            raise e

    speech_detected = speech_prob > 0.5
    return speech_detected

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/save_processed_audio', methods=['POST'])
def save_processed_audio():
    save_processed_audio_chunks()
    return "Processed audio saved."

def process_audio_buffer():
    global audio_buffer, speech_detected, previous_speech_detected, last_processed_time

    with processing_lock:
        current_time = time.time()
        if current_time - last_processed_time < PROCESSING_INTERVAL:
            return

        if len(audio_buffer) == CHUNK_SIZE:
            audio_data = np.array(list(audio_buffer))
            is_speech = detect_speech(audio_data)

            if is_speech != previous_speech_detected:
                print(f"SWITCH TO: {is_speech}")
                socketio.emit('speech_status', {'is_speech': is_speech})
                previous_speech_detected = is_speech

            # Clear the buffer after processing
            audio_buffer.clear()

        last_processed_time = current_time

@socketio.on('audio_data')
def handle_audio_data(data):
    global audio_buffer

    audio_chunk = np.frombuffer(data, dtype=np.float32)

    if audio_chunk.ndim > 1:
        audio_chunk = np.mean(audio_chunk, axis=1)

    # Resample to 16 kHz
    resampled_chunk = librosa.resample(audio_chunk, orig_sr=INPUT_SAMPLE_RATE, target_sr=VAD_SAMPLE_RATE)

    # Add resampled data to buffer
    audio_buffer.extend(resampled_chunk)

    # Process audio in a separate thread to avoid blocking
    threading.Thread(target=process_audio_buffer).start()

@socketio.on('stop_recording')
def handle_stop_recording():
    save_processed_audio_chunks()
    global audio_buffer
    if audio_buffer:
        save_audio_to_file(list(audio_buffer))
        audio_buffer.clear()

def save_audio_to_file(audio_data):
    filename = f'recording_{int(time.time())}.wav'
    file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)

    int_data = np.int16(np.array(audio_data) * 32767)

    with wave.open(file_path, 'wb') as wf:
        wf.setnchannels(CHANNELS)
        wf.setsampwidth(2)
        wf.setframerate(VAD_SAMPLE_RATE)
        wf.writeframes(int_data.tobytes())

    print(f"Audio saved to {file_path}")

if __name__ == '__main__':
    ensure_folder_exists(app.config['UPLOAD_FOLDER'])
    ensure_folder_exists(app.config['TEMP_AUDIO_FOLDER'])
    ensure_folder_exists(app.config['PROCESSED_AUDIO_FOLDER'])
    socketio.run(app, debug=True, host='0.0.0.0', port=8081, allow_unsafe_werkzeug=True)


```


audio_handler.py
```py
import wave
import os
import numpy as np
from pydub import AudioSegment

audio_data = []

def save_audio(data):
    global audio_data
    audio_data.append(np.frombuffer(data, dtype=np.int16))

def finalize_audio():
    global audio_data
    if audio_data:
        audio_array = np.concatenate(audio_data)
        audio_segment = AudioSegment(
            audio_array.tobytes(),
            frame_rate=44100,
            sample_width=2,
            channels=1
        )
        output_path = os.path.join('static', 'output.wav')
        audio_segment.export(output_path, format="wav")
        audio_data = []
        return output_path
    return None

```


main.py
```py
# This is a sample Python script.

# Press Shift+F10 to execute it or replace it with your code.
# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.


def print_hi(name):
    # Use a breakpoint in the code line below to debug your script.
    print(f'Hi, {name}')  # Press Ctrl+F8 to toggle the breakpoint.


# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    print_hi('PyCharm')

# See PyCharm help at https://www.jetbrains.com/help/pycharm/

```


static/css/style.css
```css
body {
    display: flex;
    justify-content: center;
    align-items: center;
    height: 100vh;
    margin: 0;
    background-color: #f0f0f0;
}

.container {
    display: flex;
    justify-content: center;
    align-items: center;
}

.voice-button {
    /* display: flex; */
    width: 100px;
    height: 100px;
    border-radius: 50%;
    background-color: #000;
    cursor: pointer;
    transition: all 0.3s ease;
    position: relative;
    /* align-items: center; */
    /* justify-content: center; */
}

.listening {
    animation: pulse 1s infinite alternate;
    transition: background-color 0.1s ease;
}

@keyframes pulse {
    0% {
        background-color: #492516;
    }
    100% {
        background-color: #7e391d;
    }
}

.voice-detect-point {
    background-color: white;
    width: 25px;
    height: 25px;
    border-radius: 100%;
    position: absolute;
    left: 50%;
    top: 50%;
    filter: blur(10px);
    transform: translate(-50%, -50%);

    opacity: 0;
    transition: opacity 0.2s;
}
```


static/js/script.js
```js
let isRecording = false;
let audioContext;
let socket;
let analyser;
let dataArray;

const voiceButton = document.getElementById('voice-button');
const voiceDetectPoint = document.querySelector('.voice-detect-point');
voiceButton.addEventListener('click', toggleRecording);

function initSocket() {
    socket = io.connect(location.protocol + '//' + document.domain + ':' + location.port);

    // Добавляем обработчик для получения статуса речи
    socket.on('speech_status', function(data) {
        if (data.is_speech) {
            voiceDetectPoint.style.opacity = '1';let isRecording = false;
let audioContext;
let socket;
let analyser;
let dataArray;

const voiceButton = document.getElementById('voice-button');
const voiceDetectPoint = document.querySelector('.voice-detect-point');
voiceButton.addEventListener('click', toggleRecording);

function initSocket() {
    socket = io.connect(location.protocol + '//' + document.domain + ':' + location.port);

    // Добавляем обработчик для получения статуса речи
    socket.on('speech_status', function(data) {
        if (data.is_speech) {
            voiceDetectPoint.style.opacity = '1';
        } else {
            voiceDetectPoint.style.opacity = '0';
        }
    });
}

async function toggleRecording() {
    if (!isRecording) {
        try {
            const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
            startRecording(stream);
        } catch (err) {
            console.error('Error accessing the microphone:', err);
        }
    } else {
        stopRecording();
    }
}

async function startRecording(stream) {
    audioContext = new (window.AudioContext || window.webkitAudioContext)();
    await audioContext.audioWorklet.addModule('/static/js/worklet.js');
    const source = audioContext.createMediaStreamSource(stream);

    // Create an analyser node
    analyser = audioContext.createAnalyser();
    analyser.fftSize = 256;
    const bufferLength = analyser.frequencyBinCount;
    dataArray = new Uint8Array(bufferLength);

    const audioWorkletNode = new AudioWorkletNode(audioContext, 'audio-processor');
    source.connect(audioWorkletNode);
    source.connect(analyser);

    audioWorkletNode.port.onmessage = (event) => {
        if (!isRecording) return;
        const buf = new Float32Array(event.data);
        socket.emit('audio_data', buf.buffer);
    };

    isRecording = true;
    voiceButton.classList.add('listening');
    // Disable transition during recording
    voiceButton.style.transition = 'none';
    updateButtonSize();
}

function stopRecording() {
    isRecording = false;
    voiceButton.classList.remove('listening');
    if (audioContext) {
        audioContext.close();
    }
    socket.emit('stop_recording');
}

function updateButtonSize() {
    if (!isRecording) return;

    analyser.getByteFrequencyData(dataArray);
    const average = dataArray.reduce((acc, val) => acc + val, 0) / dataArray.length;
    const volume = average / 255; // Normalize to 0-1 range

    // Scale button size based on volume (min 100px, max 200px)
    const size = 130 + ((volume * 100)*2.5);
    voiceButton.style.width = `${size}px`;
    voiceButton.style.height = `${size}px`;

    requestAnimationFrame(updateButtonSize);
}

document.addEventListener('DOMContentLoaded', initSocket);

        } else {
            voiceDetectPoint.style.opacity = '0';
        }
    });
}

async function toggleRecording() {
    if (!isRecording) {
        try {
            const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
            startRecording(stream);
        } catch (err) {
            console.error('Error accessing the microphone:', err);
        }
    } else {
        stopRecording();
    }
}

async function startRecording(stream) {
    audioContext = new (window.AudioContext || window.webkitAudioContext)();
    await audioContext.audioWorklet.addModule('/static/js/worklet.js');
    const source = audioContext.createMediaStreamSource(stream);

    // Create an analyser node
    analyser = audioContext.createAnalyser();
    analyser.fftSize = 256;
    const bufferLength = analyser.frequencyBinCount;
    dataArray = new Uint8Array(bufferLength);

    const audioWorkletNode = new AudioWorkletNode(audioContext, 'audio-processor');
    source.connect(audioWorkletNode);
    source.connect(analyser);

    audioWorkletNode.port.onmessage = (event) => {
        if (!isRecording) return;
        const buf = new Float32Array(event.data);
        socket.emit('audio_data', buf.buffer);
    };

    isRecording = true;
    voiceButton.classList.add('listening');
    // Disable transition during recording
    voiceButton.style.transition = 'none';
    updateButtonSize();
}

function stopRecording() {
    isRecording = false;
    voiceButton.classList.remove('listening');
    if (audioContext) {
        audioContext.close();
    }
    socket.emit('stop_recording');
}

function updateButtonSize() {
    if (!isRecording) return;

    analyser.getByteFrequencyData(dataArray);
    const average = dataArray.reduce((acc, val) => acc + val, 0) / dataArray.length;
    const volume = average / 255; // Normalize to 0-1 range

    // Scale button size based on volume (min 100px, max 200px)
    const size = 130 + ((volume * 100)*2.5);
    voiceButton.style.width = `${size}px`;
    voiceButton.style.height = `${size}px`;

    requestAnimationFrame(updateButtonSize);
}

document.addEventListener('DOMContentLoaded', initSocket);

```


static/js/worklet.js
```js
class AudioProcessor extends AudioWorkletProcessor {
  process(inputs, outputs, parameters) {
    const input = inputs[0];
    const output = outputs[0];
    for (let channel = 0; channel < input.length; ++channel) {
      const inputChannel = input[channel];
      const outputChannel = output[channel];
      for (let i = 0; i < inputChannel.length; ++i) {
        outputChannel[i] = inputChannel[i];
      }
    }

    // Отправляем данные на сервер
    this.port.postMessage(input[0]);

    return true;
  }
}

registerProcessor('audio-processor', AudioProcessor);
```


templates/index.html
```html
<!DOCTYPE html>
<html lang="ru">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Голосовой чат-бот</title>
    <link rel="stylesheet" href="../static/css/style.css">
    <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.0.1/socket.io.js"></script>
</head>
<body>
    <div class="container">
        <div id="voice-button" class="voice-button">
            <div class="voice-detect-point"></div>
        </div>
    </div>
    <script src="../static/js/script.js"></script>
</body>
</html>
```



Это мой проект для создания ConversationalAI.
Сейчас проблема с VAD. В момент речи очень большая задержка на определении VAD и сохранение.
Можешь полностью пересмотреть способ доставки чанков на сервер и определение там VAD.

Главное: VAD работает только с фиксированной длиной чанка, а также с фиксированной частотой. Это не меняй.