update

data/call_monitor/id-ID/voice/000a3f9a-b2bf-46fd-9c69-477fc62cda51_id-ID_1671935534167 - 副本.wav

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cf9e6ef0ee87be308c8a59a1459836dc9229c83be37c5e7204586c385d8d7a84
-size 32044

main.py

@@ -41,6 +41,8 @@ def click_ring_vad_button(audio: Tuple[int, np.ndarray],
                           max_silence_length_ms: int = 300,
                           start_ring_rate: float = 0.9,
                           end_ring_rate: float = 0.1,
+                          max_speech_length_s: float = 2.0,
+                          min_speech_length_s: float = 0.3,
                           ):
     global vad
 
@@ -64,8 +66,11 @@ def click_ring_vad_button(audio: Tuple[int, np.ndarray],
               start_ring_rate=start_ring_rate,
               end_ring_rate=end_ring_rate,
               frame_length_ms=frame_length_ms,
+              frame_step_ms=frame_step_ms,
               padding_length_ms=padding_length_ms,
               max_silence_length_ms=max_silence_length_ms,
+              max_speech_length_s=max_speech_length_s,
+              min_speech_length_s=min_speech_length_s,
               sample_rate=sample_rate,
               )
 
@@ -88,7 +93,7 @@ def click_ring_vad_button(audio: Tuple[int, np.ndarray],
     time = np.arange(0, len(signal)) / sample_rate
     plt.figure(figsize=(12, 5))
     plt.plot(time, signal / 32768, color="b")
-    plt.plot(time, speech_probs * 2, color="gray")
+    plt.plot(time, speech_probs, color="gray")
 
     for start, end in vad_segments:
         plt.axvline(x=start, ymin=0.15, ymax=0.85, color="g", linestyle="--", label="开始端点")
@@ -143,6 +148,10 @@ def main():
                                     ring_start_ring_rate = gr.Slider(minimum=0, maximum=1, value=0.9, step=0.05, label="start_ring_rate")
                                     ring_end_ring_rate = gr.Slider(minimum=0, maximum=1, value=0.1, step=0.05, label="end_ring_rate")
 
+                                with gr.Row():
+                                    ring_max_speech_length_s = gr.Slider(minimum=0.0, maximum=10.0, value=2.0, step=0.05, label="max_speech_length_s")
+                                    ring_min_speech_length_s = gr.Slider(minimum=0.0, maximum=2.0, value=0.3, step=0.05, label="min_speech_length_s")
+
                                 ring_button = gr.Button("retrieval", variant="primary")
 
                             with gr.Column(scale=1):
@@ -156,7 +165,8 @@ def main():
                                 ring_model_name, ring_agg,
                                 ring_frame_length_ms, ring_frame_step_ms,
                                 ring_padding_length_ms, ring_max_silence_length_ms,
-                                ring_start_ring_rate, ring_end_ring_rate
+                                ring_start_ring_rate, ring_end_ring_rate,
+                                ring_max_speech_length_s, ring_min_speech_length_s
                             ],
                             outputs=[ring_image, ring_end_points],
                             fn=click_ring_vad_button
@@ -170,7 +180,8 @@ def main():
                                 ring_model_name, ring_agg,
                                 ring_frame_length_ms, ring_frame_step_ms,
                                 ring_padding_length_ms, ring_max_silence_length_ms,
-                                ring_start_ring_rate, ring_end_ring_rate
+                                ring_start_ring_rate, ring_end_ring_rate,
+                                ring_max_speech_length_s, ring_min_speech_length_s
                             ],
                             outputs=[ring_image, ring_end_points],
                         )

ring_vad_examples.json

@@ -1,38 +1,66 @@
 [
     [
         "data/early_media/3300999628164249998.wav",
-        "webrtcvad", 3, 30, 300, 300, 300, 0.9, 0.1
+        "webrtcvad", 3, 30, 30, 300, 300, 0.9, 0.1, 2.0, 0.3
     ],
     [
         "data/early_media/3300999628164852605.wav",
-        "webrtcvad", 3, 30, 300, 300, 300, 0.9, 0.1
+        "webrtcvad", 3, 30, 30, 300, 300, 0.9, 0.1, 2.0, 0.3
     ],
     [
         "data/early_media/3300999628164249998.wav",
-        "silerovad", 3, 35, 350, 350, 350, 0.7, 0.3
+        "silerovad", 3, 35, 35, 350, 350, 0.7, 0.3, 2.0, 0.3
     ],
     [
         "data/early_media/3300999628164852605.wav",
-        "silerovad", 3, 35, 350, 350, 350, 0.5, 0.5
+        "silerovad", 3, 35, 35, 350, 350, 0.5, 0.5, 2.0, 0.3
+    ],
+    [
+        "data/call_monitor/id-ID/noise_mute/000d7fba-80ce-4bd7-84fe-e9c43de30f4a_id-ID_1678495379262.wav",
+        "silerovad", 3, 35, 35, 350, 350, 0.7, 0.3, 2.0, 0.3
+    ],
+    [
+        "data/call_monitor/id-ID/noise/00a0a2a3-14ff-4a84-8aee-b18b2fb65355_id-ID_1680237229413.wav",
+        "silerovad", 3, 35, 35, 350, 350, 0.7, 0.3, 2.0, 0.3
     ],
     [
         "data/early_media/3300999628164852605.wav",
-        "call_voice", 3, 300, 30, 300, 300, 0.2, 0.1
+        "call_voice", 3, 300, 30, 300, 300, 0.2, 0.1, 2.0, 0.3
     ],
     [
         "data/early_media/62/3300999628999191096.wav",
-        "call_voice", 3, 300, 30, 300, 300, 0.2, 0.1
+        "call_voice", 3, 300, 30, 300, 300, 0.2, 0.1, 2.0, 0.3
     ],
     [
         "data/early_media/62/33009996287818451333.wav",
-        "call_voice", 3, 300, 30, 300, 300, 0.2, 0.1
+        "call_voice", 3, 300, 30, 300, 300, 0.2, 0.1, 2.0, 0.3
     ],
     [
-        "data/call_monitor/id-ID/noise_mute/000d7fba-80ce-4bd7-84fe-e9c43de30f4a_id-ID_1678495379262.wav",
-        "silerovad", 3, 35, 350, 350, 350, 0.7, 0.3
+        "data/call_monitor/id-ID/noise/00a0a2a3-14ff-4a84-8aee-b18b2fb65355_id-ID_1680237229413.wav",
+        "call_voice", 3, 300, 30, 300, 300, 0.2, 0.1, 2.0, 0.3
     ],
     [
-        "data/call_monitor/id-ID/noise/00a0a2a3-14ff-4a84-8aee-b18b2fb65355_id-ID_1680237229413.wav",
-        "silerovad", 3, 35, 350, 350, 350, 0.7, 0.3
+        "data/call_monitor/id-ID/noise/000ad44a-fbad-4a22-ba5a-c6dc855779b2_id-ID_1672040947119.wav",
+        "call_voice", 3, 300, 30, 300, 300, 0.2, 0.1, 2.0, 0.3
+    ],
+    [
+        "data/call_monitor/id-ID/noise/000da369-6652-4601-b241-33ffbd52a224_id-ID_1676000326981.wav",
+        "call_voice", 3, 300, 30, 300, 300, 0.2, 0.1, 2.0, 0.3
+    ],
+    [
+        "data/call_monitor/id-ID/voicemail/00a20d31-e1cb-4c70-821b-6fd151b260ae_id-ID_1671762897272.wav",
+        "call_voice", 3, 300, 30, 300, 300, 0.2, 0.1, 2.0, 0.3
+    ],
+    [
+        "data/call_monitor/id-ID/voicemail/000b03b3-172e-4784-8510-24cf37e205ba_id-ID_1672193551438.wav",
+        "call_voice", 3, 300, 30, 300, 300, 0.2, 0.1, 2.0, 0.3
+    ],
+    [
+        "data/call_monitor/id-ID/voice/000a3f9a-b2bf-46fd-9c69-477fc62cda51_id-ID_1671935534167.wav",
+        "call_voice", 3, 300, 30, 120, 300, 0.4, 0.1, 2.0, 0.3
+    ],
+    [
+        "data/call_monitor/id-ID/voice/000cb369-a0ee-44aa-a213-18b036f1baf7_id-ID_1678762306513.wav",
+        "call_voice", 3, 300, 30, 120, 300, 0.4, 0.1, 2.0, 0.3
     ]
 ]

toolbox/vad/vad.py

@@ -107,6 +107,8 @@ class Vad(object):
                  frame_step_ms: int = 30,
                  padding_length_ms: int = 300,
                  max_silence_length_ms: int = 300,
+                 max_speech_length_s: float = 2.0,
+                 min_speech_length_s: float = 0.3,
                  sample_rate: int = 8000
                  ):
         self.model = model
@@ -115,13 +117,16 @@ class Vad(object):
         self.frame_length_ms = frame_length_ms
         self.padding_length_ms = padding_length_ms
         self.max_silence_length_ms = max_silence_length_ms
+        self.max_speech_length_s = max_speech_length_s
+        self.min_speech_length_s = min_speech_length_s
         self.sample_rate = sample_rate
 
         # frames
         self.frame_length = int(sample_rate * (frame_length_ms / 1000.0))
         self.frame_step = int(sample_rate * (frame_step_ms / 1000.0))
         self.frame_timestamp_s = 0.0
-        self.signal_cache = np.zeros(shape=(self.frame_length,), dtype=np.int16)
+        # self.signal_cache = np.zeros(shape=(self.frame_length,), dtype=np.int16)
+        self.signal_cache = None
 
         # segments
         self.num_padding_frames = int(padding_length_ms / frame_step_ms)
@@ -184,22 +189,23 @@ class Vad(object):
 
                     for f, _ in self.ring_buffer:
                         self.voiced_frames.append(f)
-                    self.ring_buffer.clear()
-            else:
-                self.voiced_frames.append(frame)
-                self.ring_buffer.append((frame, speech_prob))
-                num_voiced = sum([p for _, p in self.ring_buffer])
+                continue
 
-                if num_voiced < self.end_ring_rate * self.ring_buffer.maxlen:
-                    self.triggered = False
-                    segment = [
-                        np.concatenate([f.signal for f in self.voiced_frames]),
-                        self.voiced_frames[0].timestamp_s,
-                        self.voiced_frames[-1].timestamp_s,
-                    ]
-                    yield segment
-                    self.ring_buffer.clear()
-                    self.voiced_frames = []
+            self.voiced_frames.append(frame)
+            self.ring_buffer.append((frame, speech_prob))
+            num_voiced = sum([p for _, p in self.ring_buffer])
+
+            if num_voiced < self.end_ring_rate * self.ring_buffer.maxlen:
+                segment = [
+                    np.concatenate([f.signal for f in self.voiced_frames]),
+                    self.voiced_frames[0].timestamp_s,
+                    self.voiced_frames[-1].timestamp_s,
+                ]
+                yield segment
+                self.triggered = False
+                self.ring_buffer.clear()
+                self.voiced_frames = []
+                continue
 
     def vad_segments_generator(self, segments_generator):
         segments = list(segments_generator)
@@ -208,22 +214,31 @@ class Vad(object):
             start = round(segment[1], 4)
             end = round(segment[2], 4)
 
-            if self.is_first_segment:
+            if self.timestamp_start_s is None and self.timestamp_end_s is None:
                 self.timestamp_start_s = start
                 self.timestamp_end_s = end
-                self.is_first_segment = False
                 continue
 
-            if self.timestamp_start_s:
-                silence_length_s = (start - self.timestamp_end_s) * 1000
-                if silence_length_s > self.max_silence_length_ms:
-                    vad_segment = [self.timestamp_start_s, self.timestamp_end_s]
-                    yield vad_segment
+            if self.timestamp_end_s - self.timestamp_start_s > self.max_speech_length_s:
+                end_ = self.timestamp_start_s + self.max_speech_length_s
+                vad_segment = [self.timestamp_start_s, end_]
+                yield vad_segment
+                self.timestamp_start_s = end_
+
+            silence_length_ms = (start - self.timestamp_end_s) * 1000
+            if silence_length_ms < self.max_silence_length_ms:
+                self.timestamp_end_s = end
+                continue
+
+            if self.timestamp_end_s - self.timestamp_start_s < self.min_speech_length_s:
+                self.timestamp_start_s = start
+                self.timestamp_end_s = end
+                continue
 
-                    self.timestamp_start_s = start
-                    self.timestamp_end_s = end
-                else:
-                    self.timestamp_end_s = end
+            vad_segment = [self.timestamp_start_s, self.timestamp_end_s]
+            yield vad_segment
+            self.timestamp_start_s = start
+            self.timestamp_end_s = end
 
     def vad(self, signal: np.ndarray) -> List[list]:
         segments = self.segments_generator(signal)