Repository: castorini/howl (language: python)

Issue:
printing out how many instances of vocab is captured from dataset generation
I have tried to generate datasets for `love you`

However most of the samples are captured for `you` and samples for `love` was only a handful.

It would be beneficial to see how many instances are captured for each vocab and keep the amount simliar



Discussion / hints:
Yes it might be useful. I try to generate custom dataset and do the training. But the performance is not good because there is some bias in the generated dataset.
@Alex-Beh it seems like you have been trying to datasets for a custom wakeword. you might find scripts in #55 useful.
I still need to test and fix couple things but it should be a good reference.
@ljj7975 I try to run the generate_dataset.sh and the train_model.sh . Do we need to set the [VOCAB ](echo ">>> training a model for ${VOCAB}; model will be stored at ${WORKSPACE_PATH}")variable before we start training? Because it still show  `labeler vocab:  [' hey', 'fire', 'fox']` ....

As I described in [this issue](https://github.com/castorini/howl/issues/37#issuecomment-758475061), the confusion matrix is `ConfusionMatrix(tp=0, fp=0, tn=0, fn=0)` most of the time and and only the few episode is `ConfusionMatrix(tp=0, fp=0, tn=806, fn=0)`. Is that something wrong with the training process?
let me try it myself and get you working version, can you open a separate ticket just for this issue?
reproducing hey firefox detection from scratch

Fix patch:
diff --git a/.gitignore b/.gitignore
index 0133fb7b..5a969a7e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,7 @@
-data
-workspaces
-exp_results
-montreal-forced-aligner
+/data
+/workspaces
+/exp_results
+/montreal-forced-aligner
 
 .vscode/
 
diff --git a/howl/data/dataset/__init__.py b/howl/data/dataset/__init__.py
index 31ea1e9b..af5e7c2d 100644
--- a/howl/data/dataset/__init__.py
+++ b/howl/data/dataset/__init__.py
@@ -1,4 +1,5 @@
 from .base import *
 from .dataset import *
+from .labeller import *
 from .phone import *
 from .serialize import *
diff --git a/howl/data/dataset/base.py b/howl/data/dataset/base.py
index ddbbdf4b..ef9a21b0 100644
--- a/howl/data/dataset/base.py
+++ b/howl/data/dataset/base.py
@@ -18,11 +18,8 @@
            'DatasetType',
            'EmplacableExample',
            'WakeWordClipExample',
-           'FrameLabeler',
-           'WordFrameLabeler',
-           'FrameLabelData',
            'SequenceBatch',
-           'PhoneticFrameLabeler',
+           'FrameLabelData',
            'UNKNOWN_TRANSCRIPTION',
            'NEGATIVE_CLASS']
 
@@ -31,6 +28,11 @@
 NEGATIVE_CLASS = '[NEGATIVE]'
 
 
+@dataclass
+class FrameLabelData:
+    timestamp_label_map: Mapping[float, int]
+
+
 @dataclass
 class AudioDatasetStatistics:
     num_examples: int
@@ -56,59 +58,6 @@ def phone_phrase(self) -> Optional[PhonePhrase]:
         return PhonePhrase([Phone(x) for x in self.phone_strings]) if self.phone_strings else None
 
 
-@dataclass
-class FrameLabelData:
-    timestamp_label_map: Mapping[float, int]
-
-
-class FrameLabeler:
-    def compute_frame_labels(self, metadata: AudioClipMetadata) -> FrameLabelData:
-        raise NotImplementedError
-
-
-class PhoneticFrameLabeler(FrameLabeler):
-    def __init__(self, phrases: List[PhonePhrase]):
-        self.phrases = phrases
-
-    def compute_frame_labels(self, metadata: AudioClipMetadata) -> FrameLabelData:
-        frame_labels = dict()
-        start = 0
-        pp = PhonePhrase.from_string(metadata.transcription)
-        for idx, phrase in enumerate(self.phrases):
-            while True:
-                try:
-                    start = pp.audible_index(phrase, start)
-                except ValueError:
-                    break
-                # TODO: make alignment to token instead of character
-                start = pp.all_idx_to_transcript_idx(pp.audible_idx_to_all_idx(start))
-                frame_labels[metadata.end_timestamps[start + len(str(phrase)) - 1]] = idx
-                start += 1
-        return FrameLabelData(frame_labels)
-
-
-class WordFrameLabeler(FrameLabeler):
-    def __init__(self, words: List[str], ceil_word_boundary: bool = False):
-        self.words = words
-        self.ceil_word_boundary = ceil_word_boundary
-
-    def compute_frame_labels(self, metadata: AudioClipMetadata) -> FrameLabelData:
-        frame_labels = dict()
-        t = f' {metadata.transcription} '
-        start = 0
-        for idx, word in enumerate(self.words):
-            while True:
-                try:
-                    start = t.index(word, start)
-                except ValueError:
-                    break
-                while self.ceil_word_boundary and start + len(word) < len(t) - 1 and t[start + len(word)] != ' ':
-                    start += 1
-                frame_labels[metadata.end_timestamps[start + len(word.rstrip()) - 2]] = idx
-                start += 1
-        return FrameLabelData(frame_labels)
-
-
 class EmplacableExample:
     audio_data: torch.Tensor
 
diff --git a/howl/data/dataset/dataset.py b/howl/data/dataset/dataset.py
index 9eb8f49c..6b6b41a2 100644
--- a/howl/data/dataset/dataset.py
+++ b/howl/data/dataset/dataset.py
@@ -12,7 +12,8 @@
 
 from .base import (NEGATIVE_CLASS, AudioClipExample, AudioClipMetadata,
                    AudioDatasetStatistics, ClassificationClipExample,
-                   DatasetType, FrameLabeler, WakeWordClipExample)
+                   DatasetType, WakeWordClipExample)
+from .labeller import FrameLabeler
 from .phone import PronunciationDictionary
 
 __all__ = ['AudioDataset',
diff --git a/howl/data/dataset/labeller.py b/howl/data/dataset/labeller.py
new file mode 100644
index 00000000..f2b714b7
--- /dev/null
+++ b/howl/data/dataset/labeller.py
@@ -0,0 +1,59 @@
+
+from dataclasses import dataclass
+from typing import List
+
+from howl.data.dataset.phone import PhonePhrase
+
+from .base import AudioClipMetadata, FrameLabelData
+
+__all__ = ['FrameLabeler',
+           'WordFrameLabeler',
+           'PhoneticFrameLabeler']
+
+
+class FrameLabeler:
+    def compute_frame_labels(self, metadata: AudioClipMetadata) -> FrameLabelData:
+        raise NotImplementedError
+
+
+class PhoneticFrameLabeler(FrameLabeler):
+    def __init__(self, phrases: List[PhonePhrase]):
+        self.phrases = phrases
+
+    def compute_frame_labels(self, metadata: AudioClipMetadata) -> FrameLabelData:
+        frame_labels = dict()
+        start = 0
+        pp = PhonePhrase.from_string(metadata.transcription)
+        for idx, phrase in enumerate(self.phrases):
+            while True:
+                try:
+                    start = pp.audible_index(phrase, start)
+                except ValueError:
+                    break
+                # TODO: make alignment to token instead of character
+                start = pp.all_idx_to_transcript_idx(pp.audible_idx_to_all_idx(start))
+                frame_labels[metadata.end_timestamps[start + len(str(phrase)) - 1]] = idx
+                start += 1
+        return FrameLabelData(frame_labels)
+
+
+class WordFrameLabeler(FrameLabeler):
+    def __init__(self, words: List[str], ceil_word_boundary: bool = False):
+        self.words = words
+        self.ceil_word_boundary = ceil_word_boundary
+
+    def compute_frame_labels(self, metadata: AudioClipMetadata) -> FrameLabelData:
+        frame_labels = dict()
+        t = f' {metadata.transcription} '
+        start = 0
+        for idx, word in enumerate(self.words):
+            while True:
+                try:
+                    start = t.index(word, start)
+                except ValueError:
+                    break
+                while self.ceil_word_boundary and start + len(word) < len(t) - 1 and t[start + len(word)] != ' ':
+                    start += 1
+                frame_labels[metadata.end_timestamps[start + len(word.rstrip()) - 2]] = idx
+                start += 1
+        return FrameLabelData(frame_labels)
diff --git a/howl/data/dataset/phone.py b/howl/data/dataset/phone.py
index 503c4b87..67942818 100644
--- a/howl/data/dataset/phone.py
+++ b/howl/data/dataset/phone.py
@@ -1,9 +1,8 @@
+import enum
 from collections import defaultdict
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Mapping, List
-import enum
-
+from typing import List, Mapping
 
 __all__ = ['PhoneEnum',
            'PronunciationDictionary',
diff --git a/howl/data/searcher.py b/howl/data/searcher.py
new file mode 100644
index 00000000..a815e7ad
--- /dev/null
+++ b/howl/data/searcher.py
@@ -0,0 +1,130 @@
+import logging
+import re
+from collections import defaultdict
+from typing import List
+
+import numpy as np
+from howl.data.dataset.phone import PhonePhrase
+from howl.data.tokenize import Vocab, WakeWordTokenizer
+from howl.settings import SETTINGS
+
+__all__ = ['LabelColoring',
+           'PhoneticTranscriptSearcher',
+           'TranscriptSearcher',
+           'WordTranscriptSearcher']
+
+
+class LabelColoring:
+    def __init__(self):
+        self.color_map = {}
+        self.color_counter = 0
+        self.label_counter = 0
+
+    def append_label(self, label: int, color: int = None):
+        color = self._inc_color_counter(color)
+        self.color_map[label] = color
+
+    def _inc_color_counter(self, color: int = None):
+        if color is None:
+            color = self.color_counter
+        else:
+            self.color_counter = max(self.color_counter, color + 1)
+        self.color_counter += 1
+        return color
+
+    def extend_sequence(self, size: int, color: int = None):
+        color = self._inc_color_counter(color)
+        for label in range(self.label_counter, self.label_counter + size):
+            self.color_map[label] = color
+        self.label_counter += size
+
+    @classmethod
+    def sequential_coloring(cls, num_labels: int):
+        coloring = cls()
+        for label_idx in range(num_labels):
+            coloring.append_label(label_idx)
+        return coloring
+
+
+class TranscriptSearcher:
+    def __init__(self):
+        self.settings = SETTINGS.inference_engine
+
+    def search(self, item: str) -> bool:
+        raise NotImplementedError
+
+    def contains_any(self, item: str) -> bool:
+        raise NotImplementedError
+
+
+class WordTranscriptSearcher(TranscriptSearcher):
+    # Utilize Vocab class
+    def __init__(self, vocab: Vocab, **kwargs):
+        super().__init__(**kwargs)
+        self.vocab = vocab
+        self.tokenizer = WakeWordTokenizer(self.vocab, False)
+        self.inference_sequence_str = ''.join(map(str, self.settings.inference_sequence))
+        # self.wakeword = self.vocab.decode(self.settings.inference_sequence)
+
+    def search(self, item: str) -> bool:
+        """return true if wakeword is in the item
+        Args:
+            item (str): string to search from
+        Returns:
+            bool: true if wakeword is in the item
+        """
+        encoded_output = self.tokenizer.encode(item)
+        encoded_str = ''.join(map(str, encoded_output))
+        return self.inference_sequence_str in encoded_str
+
+    def contains_any(self, item: str) -> bool:
+        """retrun true if at least one vocab is in the item
+        Args:
+            item (str): string to search from
+        Returns:
+            bool: true if at least one vocab is in the item
+        """
+        encoded_output = self.tokenizer.encode(item)
+        return any((encoding != self.vocab.oov_token_id) for encoding in encoded_output)
+
+    def count_vocab(self, item: str, ignore_oov: bool = True) -> dict:
+        """generate counter per vocab for the item
+        Args:
+            item (str): string to analyze
+            ignore_oov (bool, optional): set to true to ignore oov word. Defaults to True.
+        Returns:
+            dict[str, int]: number of occurance in the item for each vocab
+        """
+        encoded_output = self.tokenizer.encode(item)
+        counter = dict((self.vocab[i], 0) for i in range(len(self.vocab)))
+        for encoding in encoded_output:
+            if ignore_oov and encoding == self.vocab.oov_token_id:
+                continue
+            counter[self.vocab[encoding]] += 1
+
+        return counter
+
+
+class PhoneticTranscriptSearcher(TranscriptSearcher):
+    def __init__(self, phrases: List[PhonePhrase], coloring: LabelColoring, **kwargs):
+        super().__init__(**kwargs)
+        self.phrases = phrases
+        label_map = [(phrase.audible_transcript, coloring.color_map[idx]) for idx, phrase in enumerate(phrases)]
+        buckets = defaultdict(list)
+        for transcript, color in label_map:
+            buckets[color].append(transcript)
+        pattern_strings = []
+        for _, transcripts in sorted(buckets.items(), key=lambda x: x[0]):
+            pattern_strings.append('(' + '|'.join(f'({x})' for x in transcripts) + ')')
+        pattern_strings = np.array(pattern_strings)[self.settings.inference_sequence]
+        pattern_str = '^.*' + ' '.join(pattern_strings) + '.*$'
+        logging.info(f'Using search pattern {pattern_str}')
+        self.pattern = re.compile(pattern_str)
+
+    def search(self, item: str) -> bool:
+        transcript = PhonePhrase.from_string(item).audible_transcript
+        return self.pattern.match(transcript) is not None
+
+    def contains_any(self, item: str) -> bool:
+        transcript = PhonePhrase.from_string(item).audible_transcript
+        return any(word.audible_transcript in transcript for word in self.phrases)


Test patch:
diff --git a/test/data/dataset.py b/test/data/dataset.py
new file mode 100644
index 00000000..a01e45e0
--- /dev/null
+++ b/test/data/dataset.py
@@ -0,0 +1,74 @@
+import unittest
+
+import torch
+from howl.data.dataset import (AudioClipExample, AudioClipMetadata,
+                               AudioDataset, DatasetType)
+from howl.data.searcher import WordTranscriptSearcher
+from howl.data.tokenize import Vocab
+from howl.settings import SETTINGS
+
+
+class TestDataset(AudioDataset[AudioClipMetadata]):
+    """Sample dataset for testing"""
+
+    def __init__(self,
+                 **kwargs):
+        super().__init__(**kwargs)
+
+        self.sample_rate = SETTINGS.audio.sample_rate
+
+        audio_data = torch.zeros(self.sample_rate)
+
+        metadata1 = AudioClipMetadata(transcription='hello world')
+        sample1 = AudioClipExample(metadata=metadata1, audio_data=audio_data, sample_rate=self.sample_rate)
+
+        metadata2 = AudioClipMetadata(transcription='happy new year')
+        sample2 = AudioClipExample(metadata=metadata2, audio_data=audio_data, sample_rate=self.sample_rate)
+
+        metadata3 = AudioClipMetadata(transcription='what a beautiful world')
+        sample3 = AudioClipExample(metadata=metadata3, audio_data=audio_data, sample_rate=self.sample_rate)
+
+        self.samples = [sample1, sample2, sample3]
+
+    def __len__(self):
+        return len(self.samples)
+
+    def __getitem__(self, idx) -> AudioClipExample:
+        return self.samples[idx]
+
+
+class TestAudioDataset(unittest.TestCase):
+
+    def test_compute_statistics(self):
+        """test compute statistics
+        """
+        SETTINGS.training.vocab = ["hello", "world"]
+        SETTINGS.training.token_type = "word"
+        SETTINGS.inference_engine.inference_sequence = [0, 1]
+        SETTINGS.inference_engine.inference_sequence = [0, 1]
+
+        vocab = Vocab({"Hello": 0, "World": 1}, oov_token_id=2, oov_word_repr='<OOV>')
+
+        searcher = WordTranscriptSearcher(vocab)
+
+        audio_dataset = TestDataset(metadata_list=[], set_type=DatasetType.TRAINING)
+        num_samples = len(audio_dataset)
+        total_audio_length = len(audio_dataset)
+
+        # without word_searcher, compute_length=True
+        stat = audio_dataset.compute_statistics()
+        self.assertEqual(stat.num_examples, num_samples)
+        self.assertEqual(stat.audio_length_seconds, total_audio_length)
+        self.assertFalse(stat.vocab_counts)
+
+        # with word_searcher, compute_length=True
+        stat = audio_dataset.compute_statistics(word_searcher=searcher)
+        self.assertEqual(stat.num_examples, num_samples)
+        self.assertEqual(stat.audio_length_seconds, total_audio_length)
+        self.assertEqual(len(stat.vocab_counts), len(vocab))
+        self.assertEqual(stat.vocab_counts["Hello"], 1)
+        self.assertEqual(stat.vocab_counts["World"], 2)
+
+
+if __name__ == '__main__':
+    unittest.main()