queries only sentences at least 10 speakers

README.md

@@ -1,5 +1,5 @@
 ---
-title: Pce
+title: Prosody clustering and evaluation
 emoji: ⚡
 colorFrom: pink
 colorTo: pink

scripts/ctcalign.py

@@ -0,0 +1,274 @@
+from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
+import torch
+import numpy as np
+import soundfile as sf
+from dataclasses import dataclass
+
+
+# read wav audio, make mono and 16khz if necessary
+def wav16m(sound_path):
+    aud, sr = sf.read(sound_path, dtype=np.float32)
+    if len(aud.shape) == 2:
+        aud = aud.mean(1)
+    if sr != 16000:
+        alen = int(aud.shape[0] / sr * 16000)
+        aud = signal.resample(aud, alen)
+    return aud
+
+
+def aligner(model_path,model_word_separator = '|', model_blank_token = '[PAD]'):
+
+    # build labels dict from a processor where it is not directly accessible
+    def get_processor_labels(processor,word_sep,max_labels=100):
+        ixs = sorted(list(range(max_labels)),reverse=True)
+        return {processor.tokenizer.decode(n) or word_sep:n for n in ixs}
+
+    #------------------------------------------
+    # setup wav2vec2
+    #------------------------------------------
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    torch.random.manual_seed(0)
+    max_labels = 100 # any reasonable number higher than vocab + extra + special tokens in any language used
+
+
+    model = Wav2Vec2ForCTC.from_pretrained(model_path).to(device)
+    processor = Wav2Vec2Processor.from_pretrained(model_path)
+    labels_dict = get_processor_labels(processor,model_word_separator)
+    blank_id = labels_dict[model_blank_token]
+
+
+    #convert frame-numbers to timestamps in seconds
+    # w2v2 step size is about 20ms, or 50 frames per second
+    def f2s(fr):
+        return fr/50
+
+
+
+    #------------------------------------------
+    # forced alignment with ctc decoder
+    #   based on implementation of
+    #   https://pytorch.org/audio/main/tutorials/forced_alignment_tutorial.html
+    #------------------------------------------
+
+
+    # return the label class probability of each audio frame
+    # wav is the wav data already read in, NOT the file path.
+    def get_frame_probs(wav):
+        with torch.inference_mode(): # similar to with torch.no_grad():
+            input_values = processor(wav,sampling_rate=16000).input_values[0]
+            input_values = torch.tensor(input_values, device=device).unsqueeze(0)
+            emits = model(input_values).logits
+            emits = torch.log_softmax(emits, dim=-1)
+        return emits[0].cpu().detach()
+
+
+    def get_trellis(emission, tokens, blank_id):
+    
+        num_frame = emission.size(0)
+        num_tokens = len(tokens)
+        trellis = torch.empty((num_frame + 1, num_tokens + 1))
+        trellis[0, 0] = 0
+        trellis[1:, 0] = torch.cumsum(emission[:, 0], 0) # len of this slice of trellis is len of audio frames)
+        trellis[0, -num_tokens:] = -float("inf") # len of this slice of trellis is len of transcript tokens
+        trellis[-num_tokens:, 0] = float("inf")
+        for t in range(num_frame):
+            trellis[t + 1, 1:] = torch.maximum(
+                # Score for staying at the same token
+                trellis[t, 1:] + emission[t, blank_id],
+                # Score for changing to the next token
+                trellis[t, :-1] + emission[t, tokens],
+            )
+        return trellis
+
+
+
+    @dataclass
+    class Point:
+        token_index: int
+        time_index: int
+        score: float
+    
+    @dataclass
+    class Segment:
+        label: str
+        start: int
+        end: int
+        score: float
+
+        @property
+        def mfaform(self):
+            return f"{f2s(self.start)},{f2s(self.end)},{self.label}"
+
+        @property
+        def length(self):
+            return self.end - self.start
+    
+    
+    
+    def backtrack(trellis, emission, tokens, blank_id):
+    # Note:
+    # j and t are indices for trellis, which has extra dimensions
+    # for time and tokens at the beginning.
+    # When referring to time frame index `T` in trellis,
+    # the corresponding index in emission is `T-1`.
+    # Similarly, when referring to token index `J` in trellis,
+    # the corresponding index in transcript is `J-1`.
+        j = trellis.size(1) - 1
+        t_start = torch.argmax(trellis[:, j]).item()
+    
+        path = []
+        for t in range(t_start, 0, -1):
+        # 1. Figure out if the current position was stay or change
+        # `emission[J-1]` is the emission at time frame `J` of trellis dimension.
+        # Score for token staying the same from time frame J-1 to T.
+            stayed = trellis[t - 1, j] + emission[t - 1, blank_id]
+        # Score for token changing from C-1 at T-1 to J at T.
+            changed = trellis[t - 1, j - 1] + emission[t - 1, tokens[j - 1]]
+
+        # 2. Store the path with frame-wise probability.
+            prob = emission[t - 1, tokens[j - 1] if changed > stayed else 0].exp().item()
+        # Return token index and time index in non-trellis coordinate.
+            path.append(Point(j - 1, t - 1, prob))
+            
+            # 3. Update the token
+            if changed > stayed:
+                j -= 1
+                if j == 0:
+                    break
+        else:
+            raise ValueError("Failed to align")
+        return path[::-1]
+
+
+    def merge_repeats(path,transcript):
+        i1, i2 = 0, 0
+        segments = []
+        while i1 < len(path):
+            while i2 < len(path) and path[i1].token_index == path[i2].token_index: # while both path steps point to the same token index
+                i2 += 1
+            score = sum(path[k].score for k in range(i1, i2)) / (i2 - i1)
+            segments.append( # when i2 finally switches to a different token,
+                Segment(
+                    transcript[path[i1].token_index],# to the list of segments, append the token from i1
+                    path[i1].time_index, # time of the first path-point of that token
+                    path[i2 - 1].time_index + 1, # time of the final path-point for that token.
+                    score,
+                )
+            )
+            i1 = i2
+        return segments
+
+
+
+    def merge_words(segments, separator):
+        words = []
+        i1, i2 = 0, 0
+        while i1 < len(segments):
+            if i2 >= len(segments) or segments[i2].label == separator:
+                if i1 != i2:
+                    segs = segments[i1:i2]
+                    word = "".join([seg.label for seg in segs])
+                    score = sum(seg.score * seg.length for seg in segs) / sum(seg.length for seg in segs)
+                    words.append(Segment(word, segments[i1].start, segments[i2 - 1].end, score))
+                i1 = i2 + 1
+                i2 = i1
+            else:
+                i2 += 1
+        return words
+
+
+
+
+    #------------------------------------------
+    # handle, i/o, etc.
+    #------------------------------------------
+
+
+    # generate mfa format for character (phone) and word alignments
+    # skip the word separator as it is not a phone
+    def mfalike(chars,wds,wsep):
+        hed = ['Begin,End,Label,Type,Speaker\n']
+        wlines = [f'{w.mfaform},words,000\n' for w in wds]
+        slines = [f'{ch.mfaform},phones,000\n' for ch in chars if ch.label != wsep]
+        return (''.join(hed+wlines+slines))
+
+    # generate basic exportable list format for character OR word alignments
+    # skip the word separator as it is not a phone
+    def basic(segs,wsep="|"):
+        return [[s.label,f2s(s.start),f2s(s.end)] for s in segs if s.label != wsep]
+        
+        
+    # generate numbered dicts to use in dtw
+    # alignment is given in numbered frames, not converted to timestamps
+    def fordtw(words,segments):
+
+        # index i, and word/seg, startframe, endframe
+        # preppend the index i to the word or seg
+        def _ix(i,elem):
+            return [f'{i:03d}__{elem.label}', elem.start, elem.end]
+    
+        w_al = [_ix(i,wse) for i,wse in enumerate(words)] # from tuple to list
+
+        wsegdict = {}
+        for w,s,e in w_al:
+            nlett = len(w.split('__')[1])
+            wsegs = segments[:nlett]
+            wstart = s
+            wsegs = [_ix(i,cse) for i,cse in enumerate(wsegs)]
+            wsegs = [[seg, ss-s, se-s] for seg,ss,se in wsegs]
+            wsegdict[w] = wsegs
+            segments = segments[nlett:]
+        
+        return w_al, wsegdict
+
+
+    # basic cleaning
+    # skip with is_normed=True 
+    #  if transcript was already normalised externally
+    def normalise_transcript(xcp):
+        xcp = xcp.lower()
+        xcp = xcp.replace('-','')
+        while '  ' in xcp:
+            xcp = xcp.replace('  ', ' ')
+        return xcp
+
+
+    # needs pad labels added to correctly time first segment
+    # and therefore add word sep character as placeholder in transcript
+    def prep_transcript(xcp,is_normed):
+        if not is_normed:
+            xcp = normalise_transcript(xcp)
+        xcp = xcp.replace(' ',model_word_separator)
+        label_ids = [labels_dict[c] for c in xcp]
+        label_ids = [blank_id] + label_ids + [blank_id]
+        xcp = f'{model_word_separator}{xcp}{model_word_separator}'
+        return xcp,label_ids
+    
+    
+    def _align(wav_data,transcript,is_normed=False):
+
+        norm_transcript,rec_label_ids = prep_transcript(transcript,is_normed)
+        emit = get_frame_probs(wav_data)
+        trellis = get_trellis(emit, rec_label_ids, blank_id)
+        path = backtrack(trellis, emit, rec_label_ids, blank_id)
+        
+        segments = merge_repeats(path,norm_transcript)
+        words = merge_words(segments, model_word_separator)
+
+        #return fordtw(words,model_word_separator), basic(segments,model_word_separator)
+        return basic(words,model_word_separator)
+
+    return _align
+
+
+# usage:
+# from ctcalign import aligner, wav16m
+# model_path ="/home/caitlinr/work/models/LVL/wav2vec2-large-xlsr-53-icelandic-ep10-1000h"
+# model_word_sep = '|'
+# model_blank_tk = '[PAD]'
+# caligner = aligner(model_path,model_word_sep,model_blank_tk)
+# word_aln, seg_aln = caligner(wav16m(wav_path),transcript_string)
+
+
+

scripts/runSQ.py

@@ -0,0 +1,221 @@
+import os
+from ctcalign import aligner, wav16m
+from tapi import tiro
+
+# given a Sentence string,
+# using a metadata file of SQ, // SQL1adult_metadata.tsv
+# get every file from SQ of a L1 adult with that sentence
+#  report how many, or if 0.
+
+
+def run():
+    sentence = 'hvaða sjúkdómar geta fylgt óbeinum reykingum'
+    voices = ['Alfur','Dilja','Karl', 'Dora']
+    # On tts.tiro.is speech marks are only available 
+    # for the voices: Alfur, Dilja, Karl and Dora.
+
+    corpus_meta = 'human_data/SQL1adult_metadata.tsv'
+    speech_dir = 'human_data/audio/squeries/'
+    speech_aligns = 'human_data/aligns/squeries/'
+    speech_f0 = 'human_data/f0/squeries/'
+    align_model_path ="carlosdanielhernandezmena/wav2vec2-large-xlsr-53-icelandic-ep10-1000h"
+
+    tts_dir = 'tts_data/'
+
+
+    meta = get_recordings(sentence, corpus_meta)
+    if meta:
+        align_human(meta,speech_aligns,speech_dir,align_model_path)
+        f0_human(meta, speech_f0, speech_dir, 'TODO path to reaper')
+    if voices:
+        get_tts(sentence,voices,tts_dir)
+        f0_tts(sentence, voices, tts_dir, 'TODO path to reaper')
+
+
+# find all the recordings of a given sentence
+# listed in the corpus metadata.
+# sentence should be provided lowercase without punctuation
+def get_recordings(sentence, corpusdb):
+    with open(corpusdb,'r') as handle:
+        meta = handle.read().splitlines()
+    meta = [l.split('\t') for l in meta[1:]]
+    
+    # column index 4 of db is normalised sentence text
+    smeta = [l for l in meta if l[4] == sentence]
+    
+    if len(smeta) < 10:
+        if len(smeta) < 1:
+            print('This sentence does not exist in the corpus')
+        else:
+            print('Under 10 copies of the sentence: skipping.')
+        return []
+    else:
+        print(f'{len(smeta)} recordings of sentence <{sentence}>')
+        return smeta
+
+
+
+# check if word alignments exist for a set of human speech recordings
+# if not, warn, and make them with ctcalign.
+def align_human(meta,align_dir,speech_dir,model_path):
+
+    model_word_sep = '|'
+    model_blank_tk = '[PAD]'
+    
+    no_align = []
+    
+    for rec in meta:
+        apath = align_dir + rec[2].replace('.wav','.tsv')
+        if not os.path.exists(apath):
+            no_align.append(rec)
+            
+    if no_align:
+        print(f'Need to run alignment for {len(no_align)} files')
+        caligner = aligner(model_path,model_word_sep,model_blank_tk)
+        for rec in no_align:
+            wav_path = f'{speech_dir}{rec[1]}/{rec[2]}'
+            word_aln = caligner(wav16m(wav_path),rec[4],is_normed=True)
+            apath = align_dir + rec[2].replace('.wav','.tsv')
+            word_aln = [[str(x) for x in l] for l in word_aln]
+            with open(apath,'w') as handle:
+                handle.write(''.join(['\t'.join(l)+'\n' for l in word_aln]))
+    else:
+        print('All alignments existed')
+        
+        
+
+# check if f0s exist for all of those files.
+# if not, warn, and make them with TODO reaper
+def f0_human(meta, f0_dir, speech_dir, reaper_path):
+    no_f0 = []
+    
+    for rec in meta:
+        fpath = f0_dir + rec[2].replace('.wav','.f0')
+        if not os.path.exists(fpath):
+            no_f0.append(rec)
+            
+    if no_f0:
+        print(f'Need to estimate pitch for {len(no_f0)} recordings')
+        #TODO
+        
+    else:
+        print('All speech pitch trackings existed')
+
+
+
+# # # # # # # # # 
+#################
+# TODO
+# IMPLEMENT GOOD 2 STEP PITCH ESTIMATION
+# TODO
+#################
+# # # # # # # # # 
+
+
+
+
+# check if the TTS wavs + align jsons exist for this sentence
+# if not, warn and make them with TAPI ******
+def get_tts(sentence,voices,ttsdir):
+
+    # assume the first 64 chars of sentence are enough
+    dpath = sentence.replace(' ','_')[:65]
+    
+    no_voice = []
+
+    for v in voices:
+        wpath = f'{ttsdir}{dpath}/{v}.wav'
+        jpath = f'{ttsdir}{dpath}/{v}.json'
+        if not (os.path.exists(wpath) and os.path.exists(jpath)):
+            no_voice.append(v)
+            
+    if no_voice:
+        print(f'Need to generate TTS for {len(no_voice)} voices')
+        if not os.path.exists(f'{ttsdir}{dpath}'):
+            os.mkdir(f'{ttsdir}{dpath}')
+        for v in voices:
+            wf, af = tiro(sentence,v,save=f'{ttsdir}{dpath}/')
+        
+    else:
+        print('TTS for all voices existed')
+    
+
+
+# check if the TTS f0s exist
+# if not warn + make
+# TODO collapse functions
+def f0_tts(sentence, voices, ttsdir, reaper_path):
+
+    # assume the first 64 chars of sentence are enough
+    dpath = sentence.replace(' ','_')[:65]
+    
+    no_f0 = []
+    
+    for v in voices:
+        fpath = f'{ttsdir}{dpath}/{v}.f0'
+        if not os.path.exists(fpath):
+            no_f0.append(v)
+            
+    if no_f0:
+        print(f'Need to estimate pitch for {len(no_f0)} voices')
+        #TODO
+        
+    else:
+        print('All TTS pitch trackings existed')
+
+
+
+
+
+
+run()
+
+
+
+
+
+
+# https://colab.research.google.com/drive/1RApnJEocx3-mqdQC2h5SH8vucDkSlQYt?authuser=1#scrollTo=410ecd91fa29bc73
+
+# CLUSTER the humans
+# - read energy and pitch, to alignments
+# - dtw based with selected chunking ? code should exist.
+
+# ... experimental variants? 
+# ** 1 dimension at a time vs 2 on top of each other
+# ** 25 points resampling (euclidean, kmeans, i guess....) vs all points dtw kmediods
+#  +/or maybe some intermediate parts of that??? like 25 points dtw medoids particularly **
+# --different normings for pitch? different settings for energy (tbqh i hope not too much?)
+# TODO '''replacement with a constant low value''' ********
+# errrrrrrrm duration?
+#   duration feature vector will have a different length than the others, BUT, 
+ # besides the single clustering,, 
+ # i SUPPOSE one could TRY assigning the phone's 'speech rate' value to every frame of the phone, so it doesn't change while the other 2 values do change.... like it would still VAGUELY represent that 2 people elongating the same vowel/syllable are doing similar things with duration while someone eliding that vowel is doing a different durational thing right there?
+ # might want to z-score this dimension across ALL speakers tho not within a speaker
+  # try doing it both ways at least. bc not sure to what extent i want absolute vs. relative rate info here.
+   #(note - unless chengs dur metric is of a kind where only rel makes sense in the first place. idr.)
+
+
+
+
+# GRAPH the humans.
+# - probably modify this code a bit to centre on boundary.
+# - idk.
+
+
+# TEST each TTS
+# - structure its features
+# - find its avg dist for each human cluster
+# - find the lowest dist cluster
+# - report the dist for i guess this and all clusters
+# - GRAPH the tts with its best cluster
+
+
+
+# EVALUATION
+# - of the tts
+# - of the method: consistency? coherency / interpretability of 'best' voice across different features; alt. ability to recover good & problematic features from a combined method if that is chosen as the best? 
+# - how similar are the results across different sentences? are any voices consistently good or bad; if multiple are good, are they good in the same way or good in different ways; do humans agree.
+#  >> bc hey THAT could at least be an argument for the method, u might have to take time for human judgement once but then you can keep re using it free for new voices. or to select among alternative generations given you might know a context and know what you're going for in that context. etc.
+
+

scripts/tapi.py

@@ -0,0 +1,69 @@
+import json, os, requests, warnings, wave
+warnings.filterwarnings("ignore")
+
+
+
+# synthesise speech
+#   save 16khz mono wav file
+#   and word-level timestamps
+# return paths to wave and alignment files
+def tiro(text,voice,save='./'):
+
+    # endpoint working 2023
+    url = 'https://tts.tiro.is/v0/speech'
+    headers = {'Content-Type': 'application/json'}
+
+
+    # synthesis
+    payload_tts = {
+    "Engine": "standard",
+    "LanguageCode": "is-IS",
+    "OutputFormat": "pcm",
+    "SampleRate":"16000",
+    "Text": text,
+    "VoiceId": voice
+    }
+    
+    # word time alignments
+    payload_aln = {
+    "Engine": "standard",
+    "LanguageCode": "is-IS",
+    "OutputFormat": "json",
+    "SpeechMarkTypes": ["word"],
+    "Text": text,
+    "VoiceId": voice
+    }
+
+        
+    tts_data = requests.post(url, headers=headers, json=payload_tts, verify=False)
+    aln_data = requests.post(url, headers=headers, json=payload_aln, verify=False)
+
+    
+    #fname = save+text.replace(':','').replace('/','-')
+    #wname = fname+'.wav'
+    #aname = fname+'.json'
+    wname = save+voice+'.wav'
+    aname = save+voice+'.json'
+    
+    with wave.open(wname,'wb') as f:
+        f.setnchannels(1)
+        f.setframerate(16000)
+        f.setsampwidth(2)
+        f.writeframes(tts_data.content)
+
+    with open(aname,'w') as f:
+        f.write('{"alignments": [')
+        f.write(aln_data.content.decode().replace('}\n{','},\n {'))
+        f.write(']}')
+
+    return(os.path.abspath(wname),os.path.abspath(aname))
+
+
+
+        
+#sentence = "Hæ hæ hæ hæ! Ég heiti Gervimaður Finnland, en þú?"
+#voice = "Alfur"
+
+#wf, af = tiro(sentence,voice)
+
+#print(wf, af)