Project structure:

ArsenAI/
├── audio_files/
│   ├── audio.wav
│   ├── recording.wav
│   ├── recording_1723749763.wav
│   ├── recording_1723750110.wav
│   ├── recording_1724005566.wav
│   ├── recording_1724005614.wav
│   ├── recording_1724006017.wav
│   ├── recording_1724015867.wav
│   ├── recording_1724015888.wav
│   ├── recording_1724015903.wav
│   ├── recording_1724018743.wav
│   ├── recording_1724067849.wav
│   ├── recording_1724067866.wav
│   ├── recording_1724067927.wav
│   ├── recording_1724067932.wav
│   ├── recording_1724067963.wav
│   ├── recording_1724067966.wav
│   ├── recording_1724068241.wav
│   ├── recording_1724068253.wav
│   ├── recording_1724068312.wav
│   ├── recording_1724068316.wav
│   ├── recording_1724068423.wav
│   ├── recording_1724068431.wav
│   ├── recording_1724069008.wav
│   ├── recording_1724069016.wav
│   ├── recording_1724069337.wav
│   ├── recording_1724069342.wav
│   ├── recording_1724088967.wav
│   ├── recording_1724089423.wav
│   └── recording_1729344160.wav
├── processed_audio_chunks/
│   ├── processed_chunks_1724089423.wav
│   ├── processed_chunks_1729344160.wav
│   ├── processed_chunks_1729346895.wav
│   ├── processed_chunks_1729346910.wav
│   ├── processed_chunks_1729346920.wav
│   ├── processed_chunks_1729347764.wav
│   ├── processed_chunks_1729347850.wav
│   ├── processed_chunks_1729347933.wav
│   ├── processed_chunks_1729347965.wav
│   ├── processed_chunks_1729348238.wav
│   ├── processed_chunks_1729348243.wav
│   ├── processed_chunks_1729348250.wav
│   ├── processed_chunks_1729348362.wav
│   ├── processed_chunks_1729348667.wav
│   ├── processed_chunks_1729348863.wav
│   ├── processed_chunks_1729348903.wav
│   ├── processed_chunks_1729348969.wav
│   ├── processed_chunks_1729349216.wav
│   ├── processed_chunks_1729349371.wav
│   ├── processed_chunks_1729349380.wav
│   ├── processed_chunks_1729349610.wav
│   ├── processed_chunks_1729349659.wav
│   ├── processed_chunks_1729349984.wav
│   ├── processed_chunks_1729350005.wav
│   ├── processed_chunks_1729350053.wav
│   ├── processed_chunks_1729350115.wav
│   ├── processed_chunks_1729350231.wav
│   └── processed_chunks_1729350314.wav
├── static/
│   ├── css/
│   │   └── style.css
│   ├── js/
│   │   ├── script.js
│   │   └── worklet.js
├── templates/
│   └── index.html
├── test_audio_temp/
│   ├── chunk_1724069008.wav
│   ├── chunk_1724069010.wav
│   ├── chunk_1724069011.wav
│   ├── chunk_1724069012.wav
│   ├── chunk_1724069013.wav
│   ├── chunk_1724069014.wav
│   ├── chunk_1724069015.wav
│   ├── chunk_1724069016.wav
│   ├── chunk_1724069337.wav
│   ├── chunk_1724069338.wav
│   ├── chunk_1724069339.wav
│   ├── chunk_1724069340.wav
│   ├── chunk_1724069341.wav
│   ├── chunk_1724069342.wav
│   ├── vad_input_1724088931.wav
│   ├── vad_input_1724088932.wav
│   ├── vad_input_1724088933.wav
│   ├── vad_input_1724088959.wav
│   ├── vad_input_1724088962.wav
│   ├── vad_input_1724088963.wav
│   ├── vad_input_1724088964.wav
│   ├── vad_input_1724088965.wav
│   ├── vad_input_1724088966.wav
│   ├── vad_input_1724088967.wav
│   ├── vad_input_1724089417.wav
│   ├── vad_input_1724089420.wav
│   ├── vad_input_1724089421.wav
│   ├── vad_input_1724089422.wav
│   ├── vad_input_1724089423.wav
│   ├── vad_input_1729341948.wav
│   ├── vad_input_1729344151.wav
│   ├── vad_input_1729344152.wav
│   ├── vad_input_1729347848.wav
│   ├── vad_input_1729347849.wav
│   └── vad_input_1729347850.wav
├── app.py
├── audio_handler.py
└── main.py


Project files:

app.py
```py
import os
import numpy as np
import torch
import wave
from flask import Flask, render_template
from flask_socketio import SocketIO, emit
from collections import deque
import time
import threading
from scipy.signal import resample_poly
from queue import Queue

# Константы
INPUT_SAMPLE_RATE = 48000
VAD_SAMPLE_RATE = 16000
CHANNELS = 1
CHUNK_SIZE = 512  # 32 мс при 16 кГц
UPLOAD_FOLDER = 'audio_files'
TEMP_AUDIO_FOLDER = 'test_audio_temp'
PROCESSED_AUDIO_FOLDER = 'processed_audio_chunks'
PROCESSING_INTERVAL = 0.05  # 50 мс для повышения отзывчивости

app = Flask(__name__)
app.config['SECRET_KEY'] = 'secret!'
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['TEMP_AUDIO_FOLDER'] = TEMP_AUDIO_FOLDER
app.config['PROCESSED_AUDIO_FOLDER'] = PROCESSED_AUDIO_FOLDER

# Используем eventlet для лучшей конкурентности
socketio = SocketIO(app, async_mode='eventlet', cors_allowed_origins="*")

# Выбор устройства (CPU или GPU)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Инициализация Silero VAD
vad_model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad',
                                 model='silero_vad',
                                 force_reload=True)
vad_model = vad_model.to(device)
vad_model.eval()

# Глобальные переменные
audio_buffer = deque()
processing_queue = Queue()
speech_detected = False
previous_speech_detected = False
last_processed_time = 0
processing_lock = threading.Lock()

# Буфер для VAD
vad_buffer = np.array([])

# Буфер для тестового сохранения (можно отключить для повышения производительности)
processed_audio_chunks = []

def ensure_folder_exists(folder):
    if not os.path.exists(folder):
        os.makedirs(folder)

def int2float(sound):
    if sound.dtype.kind in 'iu':  # Проверка, является ли тип данных целым числом
        sound = sound.astype('float32') / 32768
    else:
        sound = sound.astype('float32')
    sound = sound.squeeze()
    return sound

def save_audio_chunk_to_file(audio_chunk, folder, filename):
    file_path = os.path.join(folder, filename)
    int_data = np.int16(audio_chunk * 32767)
    os.makedirs(folder, exist_ok=True)
    with wave.open(file_path, 'wb') as wf:
        wf.setnchannels(CHANNELS)
        wf.setsampwidth(2)
        wf.setframerate(VAD_SAMPLE_RATE)
        wf.writeframes(int_data.tobytes())
    print(f"Saved chunk to {file_path}")

def save_processed_audio_chunks():
    global processed_audio_chunks

    if processed_audio_chunks:
        filename = f'processed_chunks_{int(time.time())}.wav'
        file_path = os.path.join(app.config['PROCESSED_AUDIO_FOLDER'], filename)

        # Соединяем все чанки
        all_chunks = np.concatenate(processed_audio_chunks)
        int_data = np.int16(all_chunks * 32767)

        os.makedirs(app.config['PROCESSED_AUDIO_FOLDER'], exist_ok=True)
        with wave.open(file_path, 'wb') as wf:
            wf.setnchannels(CHANNELS)
            wf.setsampwidth(2)
            wf.setframerate(VAD_SAMPLE_RATE)
            wf.writeframes(int_data.tobytes())

        print(f"Processed audio saved to {file_path}")
        processed_audio_chunks.clear()
    else:
        print("No processed audio chunks to save.")

def detect_speech(audio_chunk):
    global speech_detected, vad_model, device, processed_audio_chunks

    # Преобразуем в float без дополнительного масштабирования
    audio_float32 = int2float(audio_chunk)

    # Опционально сохраняем чанк для отладки
    # save_audio_chunk_to_file(audio_float32, app.config['TEMP_AUDIO_FOLDER'], f'vad_input_{int(time.time())}.wav')

    # Сохраняем чанк в буфере для тестирования
    processed_audio_chunks.append(audio_float32)

    try:
        with torch.no_grad():
            tensor = torch.from_numpy(audio_float32).to(device).unsqueeze(0)
            speech_prob = vad_model(tensor, VAD_SAMPLE_RATE).item()
            print(f"speech_prob = {speech_prob}")
    except RuntimeError as e:
        if "out of memory" in str(e):
            print("CUDA out of memory. Falling back to CPU.")
            torch.cuda.empty_cache()
            device = torch.device('cpu')
            vad_model = vad_model.to(device)
            tensor = torch.from_numpy(audio_float32).to(device).unsqueeze(0)
            speech_prob = vad_model(tensor, VAD_SAMPLE_RATE).item()
        else:
            raise e

    speech_detected = speech_prob > 0.5
    return speech_detected

def audio_processing_worker():
    global vad_buffer, speech_detected, previous_speech_detected, last_processed_time

    while True:
        audio_chunk = processing_queue.get()
        if audio_chunk is None:
            break

        # Накопление в буфере для VAD
        vad_buffer = np.concatenate((vad_buffer, audio_chunk))

        while len(vad_buffer) >= CHUNK_SIZE:
            current_chunk = vad_buffer[:CHUNK_SIZE]
            is_speech = detect_speech(current_chunk)

            if is_speech != previous_speech_detected:
                print(f"SWITCH TO: {is_speech}")
                socketio.emit('speech_status', {'is_speech': is_speech})
                previous_speech_detected = is_speech

            # Удаляем обработанный чанк из буфера
            vad_buffer = vad_buffer[CHUNK_SIZE:]

            last_processed_time = time.time()

        processing_queue.task_done()

# Запускаем рабочий поток для обработки аудио
processing_thread = threading.Thread(target=audio_processing_worker, daemon=True)
processing_thread.start()

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/save_processed_audio', methods=['POST'])
def save_processed_audio():
    save_processed_audio_chunks()
    return "Processed audio saved."

@socketio.on('audio_data')
def handle_audio_data(data):
    # Преобразуем полученные данные в массив float32
    audio_chunk = np.frombuffer(data, dtype=np.float32)

    if audio_chunk.ndim > 1:
        audio_chunk = np.mean(audio_chunk, axis=1)

    # Логирование диапазона значений
    print(f"Received audio chunk - min: {audio_chunk.min()}, max: {audio_chunk.max()}")

    # Ресемплирование с 48 кГц до 16 кГц
    resampled_chunk = resample_poly(audio_chunk, up=1, down=3)

    print(f"Resampled audio chunk - min: {resampled_chunk.min()}, max: {resampled_chunk.max()}")

    processing_queue.put(resampled_chunk)

@socketio.on('stop_recording')
def handle_stop_recording():
    save_processed_audio_chunks()
    # Нет необходимости сохранять буфер здесь, так как обрабатывается через очередь

def save_audio_to_file(audio_data):
    filename = f'recording_{int(time.time())}.wav'
    file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)

    int_data = np.int16(np.array(audio_data) * 32767)

    os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True)
    with wave.open(file_path, 'wb') as wf:
        wf.setnchannels(CHANNELS)
        wf.setsampwidth(2)
        wf.setframerate(VAD_SAMPLE_RATE)
        wf.writeframes(int_data.tobytes())

    print(f"Audio saved to {file_path}")

if __name__ == '__main__':
    ensure_folder_exists(app.config['UPLOAD_FOLDER'])
    ensure_folder_exists(app.config['TEMP_AUDIO_FOLDER'])
    ensure_folder_exists(app.config['PROCESSED_AUDIO_FOLDER'])
    socketio.run(app, debug=True, host='0.0.0.0', port=8081)
```


audio_handler.py
```py
import wave
import os
import numpy as np
from pydub import AudioSegment

audio_data = []

def save_audio(data):
    global audio_data
    audio_data.append(np.frombuffer(data, dtype=np.int16))

def finalize_audio():
    global audio_data
    if audio_data:
        audio_array = np.concatenate(audio_data)
        audio_segment = AudioSegment(
            audio_array.tobytes(),
            frame_rate=44100,
            sample_width=2,
            channels=1
        )
        output_path = os.path.join('static', 'output.wav')
        audio_segment.export(output_path, format="wav")
        audio_data = []
        return output_path
    return None

```


main.py
```py
# This is a sample Python script.

# Press Shift+F10 to execute it or replace it with your code.
# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.


def print_hi(name):
    # Use a breakpoint in the code line below to debug your script.
    print(f'Hi, {name}')  # Press Ctrl+F8 to toggle the breakpoint.


# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    print_hi('PyCharm')

# See PyCharm help at https://www.jetbrains.com/help/pycharm/

```


static/css/style.css
```css
body {
    display: flex;
    justify-content: center;
    align-items: center;
    height: 100vh;
    margin: 0;
    background-color: #f0f0f0;
}

.container {
    display: flex;
    justify-content: center;
    align-items: center;
}

.voice-button {
    /* display: flex; */
    width: 100px;
    height: 100px;
    border-radius: 50%;
    background-color: #000;
    cursor: pointer;
    transition: all 0.3s ease;
    position: relative;
    /* align-items: center; */
    /* justify-content: center; */
}

.listening {
    animation: pulse 1s infinite alternate;
    transition: background-color 0.1s ease;
}

@keyframes pulse {
    0% {
        background-color: #492516;
    }
    100% {
        background-color: #7e391d;
    }
}

.voice-detect-point {
    background-color: white;
    width: 25px;
    height: 25px;
    border-radius: 100%;
    position: absolute;
    left: 50%;
    top: 50%;
    filter: blur(10px);
    transform: translate(-50%, -50%);

    opacity: 0;
    transition: opacity 0.2s;
}
```


static/js/script.js
```js
let isRecording = false;
let audioContext;
let socket;
let analyser;
let dataArray;

const voiceButton = document.getElementById('voice-button');
const voiceDetectPoint = document.querySelector('.voice-detect-point');
voiceButton.addEventListener('click', toggleRecording);

function initSocket() {
    socket = io.connect(location.protocol + '//' + document.domain + ':' + location.port);

    // Обработчик статуса речи
    socket.on('speech_status', function(data) {
        if (data.is_speech) {
            voiceDetectPoint.style.opacity = '1';
        } else {
            voiceDetectPoint.style.opacity = '0';
        }
    });
}

async function toggleRecording() {
    if (!isRecording) {
        try {
            const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
            startRecording(stream);
        } catch (err) {
            console.error('Ошибка доступа к микрофону:', err);
        }
    } else {
        stopRecording();
    }
}

async function startRecording(stream) {
    audioContext = new (window.AudioContext || window.webkitAudioContext)();
    console.log(`Клиентская частота дискретизации: ${audioContext.sampleRate} Гц`);
    await audioContext.audioWorklet.addModule('/static/js/worklet.js');
    const source = audioContext.createMediaStreamSource(stream);

    // Создаём GainNode для усиления сигнала
    const gainNode = audioContext.createGain();
    gainNode.gain.value = 3.0; // Устанавливаем коэффициент усиления (можно настроить)

    // Создаём AnalyserNode для визуализации громкости
    analyser = audioContext.createAnalyser();
    analyser.fftSize = 256;
    const bufferLength = analyser.frequencyBinCount;
    dataArray = new Uint8Array(bufferLength);

    const audioWorkletNode = new AudioWorkletNode(audioContext, 'audio-processor');

    // Подключаем цепочку: источник -> GainNode -> AudioWorkletNode
    source.connect(gainNode).connect(audioWorkletNode);
    // Подключаем GainNode к AnalyserNode для визуализации
    source.connect(gainNode).connect(analyser);

    audioWorkletNode.port.onmessage = (event) => {
        if (!isRecording) return;
        const buf = new Float32Array(event.data);

        // Логирование амплитуд перед отправкой
        const min = Math.min(...buf);
        const max = Math.max(...buf);
        console.log(`Отправка аудиочанка - min: ${min}, max: ${max}`);
        socket.emit('audio_data', buf.buffer);
    };

        // Добавляем аудио для локального прослушивания
    const destination = audioContext.createMediaStreamDestination();
    gainNode.connect(destination);
    const localAudio = new Audio();
    localAudio.srcObject = destination.stream;
    localAudio.play();

    isRecording = true;
    voiceButton.classList.add('listening');
    // Отключаем переход для более плавной анимации
    voiceButton.style.transition = 'none';
    updateButtonSize();
}

function stopRecording() {
    isRecording = false;
    voiceButton.classList.remove('listening');
    if (audioContext) {
        audioContext.close();
    }
    socket.emit('stop_recording');
}

function updateButtonSize() {
    if (!isRecording) return;

    analyser.getByteFrequencyData(dataArray);
    const average = dataArray.reduce((acc, val) => acc + val, 0) / dataArray.length;
    const volume = average / 255; // Нормализуем в диапазон 0-1

    // Масштабируем размер кнопки на основе громкости (мин 100px, макс 200px)
    const size = 130 + ((volume * 100) * 2.5);
    voiceButton.style.width = `${size}px`;
    voiceButton.style.height = `${size}px`;

    requestAnimationFrame(updateButtonSize);
}

document.addEventListener('DOMContentLoaded', initSocket);
```


static/js/worklet.js
```js
class AudioProcessor extends AudioWorkletProcessor {
  process(inputs, outputs, parameters) {
    const input = inputs[0];
    const output = outputs[0];
    for (let channel = 0; channel < input.length; ++channel) {
      const inputChannel = input[channel];
      const outputChannel = output[channel];
      for (let i = 0; i < inputChannel.length; ++i) {
        outputChannel[i] = inputChannel[i];
      }
    }

    // Создаём копию данных для отправки
    const copy = new Float32Array(input[0].length);
    copy.set(input[0]);
    this.port.postMessage(copy.buffer);

    return true;
  }
}

registerProcessor('audio-processor', AudioProcessor);
```


templates/index.html
```html
<!DOCTYPE html>
<html lang="ru">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Голосовой чат-бот</title>
    <link rel="stylesheet" href="../static/css/style.css">
    <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.0.1/socket.io.js"></script>
</head>
<body>
    <div class="container">
        <div id="voice-button" class="voice-button">
            <div class="voice-detect-point"></div>
        </div>
    </div>
    <script src="../static/js/script.js"></script>
</body>
</html>
```



Сохраненное аудио на сервере очень забасьбусченное, там ничего не разобрать. Где-то проблема есть