Computing word timestamps from start and end offsets of hypotheses object
State: closed | Pull request: no
Labels: name: bug

Hi, 

I am trying to compute word level timestamps with CTCBPEDecoder and able to get the Hypotheses with timestep dictionary. But the dictionary has start_offset and end_offset for each word. I would like to know the offset value and stride multplication factor for computing final timestamp. Formulation for it would be great.
 Here is a sample hypothesis object with timestep dictionary:

`[Hypothesis(score=tensor(-1.0438), y_sequence=tensor([[-5.6491e+01, -1.8834e+01, -1.4659e+01,  ..., -2.6665e+01,
         -3.1068e+01, -2.8610e-06],
        [-5.7072e+01, -1.8568e+01, -1.7445e+01,  ..., -2.4695e+01,
         -3.3793e+01, -5.9605e-07],
        [-5.7386e+01, -1.8189e+01, -1.9202e+01,  ..., -2.3949e+01,
         -3.3627e+01, -8.3446e-07],
        ...,
        [-5.5325e+01, -1.4839e+01, -1.9966e+01,  ..., -2.1065e+01,
         -3.3125e+01, -1.4305e-05],
        [-5.6287e+01, -1.4943e+01, -2.2234e+01,  ..., -2.3138e+01,
         -3.3042e+01, -7.3909e-06],
        [-5.2566e+01, -1.3785e+01, -2.1438e+01,  ..., -2.2653e+01,
         -2.8750e+01, -1.4543e-05]], device='cuda:0'), text="well i don't wish to see it any more observed phoebe turning away her eyes it is certainly very like the old portrait", dec_out=None, dec_state=None, timestep={'timestep': [12, 15, 19, 21, 23, 24, 25, 27, 29, 30, 31, 33, 36, 37, 39, 41, 44, 46, 47, 50, 52, 56, 57, 59, 61, 63, 64, 66, 67, 68, 69, 70, 71, 72, 74, 83, 85, 87, 90, 92, 94, 96, 97, 100, 102, 105, 107, 110, 125, 130, 135, 136, 138, 140, 141, 143, 146, 148, 150, 152, 158, 161, 163, 164, 167, 169, 171, 172, 174, 176], 'char': [{'char': 'we', 'start_offset': 11, 'end_offset': 12}, {'char': 'll', 'start_offset': 12, 'end_offset': 15}, {'char': 'i', 'start_offset': 15, 'end_offset': 19}, {'char': 'do', 'start_offset': 19, 'end_offset': 21}, {'char': 'n', 'start_offset': 21, 'end_offset': 23}, {'char': "'", 'start_offset': 23, 'end_offset': 24}, {'char': 't', 'start_offset': 24, 'end_offset': 25}, {'char': 'w', 'start_offset': 25, 'end_offset': 27}, {'char': 'i', 'start_offset': 27, 'end_offset': 29}, {'char': 's', 'start_offset': 29, 'end_offset': 30}, {'char': 'h', 'start_offset': 30, 'end_offset': 31}, {'char': 'to', 'start_offset': 31, 'end_offset': 33}, {'char': 's', 'start_offset': 33, 'end_offset': 36}, {'char': 'e', 'start_offset': 36, 'end_offset': 37}, {'char': 'e', 'start_offset': 37, 'end_offset': 39}, {'char': 'it', 'start_offset': 39, 'end_offset': 41}, {'char': 'a', 'start_offset': 41, 'end_offset': 44}, {'char': 'n', 'start_offset': 44, 'end_offset': 46}, {'char': 'y', 'start_offset': 46, 'end_offset': 47}, {'char': 'mo', 'start_offset': 47, 'end_offset': 50}, {'char': 're', 'start_offset': 50, 'end_offset': 52}, {'char': 'o', 'start_offset': 52, 'end_offset': 56}, {'char': 'b', 'start_offset': 56, 'end_offset': 57}, {'char': 's', 'start_offset': 57, 'end_offset': 59}, {'char': 'er', 'start_offset': 59, 'end_offset': 61}, {'char': 've', 'start_offset': 61, 'end_offset': 63}, {'char': 'd', 'start_offset': 63, 'end_offset': 64}, {'char': 'p', 'start_offset': 64, 'end_offset': 66}, {'char': 'h', 'start_offset': 66, 'end_offset': 67}, {'char': 'o', 'start_offset': 67, 'end_offset': 69}, {'char': 'e', 'start_offset': 69, 'end_offset': 70}, {'char': 'b', 'start_offset': 70, 'end_offset': 72}, {'char': 'e', 'start_offset': 72, 'end_offset': 74}, {'char': 't', 'start_offset': 74, 'end_offset': 83}, {'char': 'ur', 'start_offset': 83, 'end_offset': 85}, {'char': 'n', 'start_offset': 85, 'end_offset': 87}, {'char': 'ing', 'start_offset': 87, 'end_offset': 90}, {'char': 'a', 'start_offset': 90, 'end_offset': 92}, {'char': 'w', 'start_offset': 92, 'end_offset': 94}, {'char': 'a', 'start_offset': 94, 'end_offset': 96}, {'char': 'y', 'start_offset': 96, 'end_offset': 97}, {'char': 'he', 'start_offset': 97, 'end_offset': 100}, {'char': 'r', 'start_offset': 100, 'end_offset': 102}, {'char': 'e', 'start_offset': 102, 'end_offset': 105}, {'char': 'y', 'start_offset': 105, 'end_offset': 107}, {'char': 'es', 'start_offset': 107, 'end_offset': 110}, {'char': 'it', 'start_offset': 110, 'end_offset': 125}, {'char': 'is', 'start_offset': 125, 'end_offset': 130}, {'char': 'c', 'start_offset': 130, 'end_offset': 135}, {'char': 'er', 'start_offset': 135, 'end_offset': 136}, {'char': 't', 'start_offset': 136, 'end_offset': 138}, {'char': 'a', 'start_offset': 138, 'end_offset': 140}, {'char': 'in', 'start_offset': 140, 'end_offset': 141}, {'char': 'ly', 'start_offset': 141, 'end_offset': 143}, {'char': '', 'start_offset': 143, 'end_offset': 146}, {'char': 'ver', 'start_offset': 146, 'end_offset': 148}, {'char': 'y', 'start_offset': 148, 'end_offset': 150}, {'char': 'like', 'start_offset': 150, 'end_offset': 152}, {'char': 'the', 'start_offset': 152, 'end_offset': 158}, {'char': 'o', 'start_offset': 158, 'end_offset': 161}, {'char': 'l', 'start_offset': 161, 'end_offset': 163}, {'char': 'd', 'start_offset': 163, 'end_offset': 164}, {'char': 'p', 'start_offset': 164, 'end_offset': 167}, {'char': 'or', 'start_offset': 167, 'end_offset': 169}, {'char': 't', 'start_offset': 169, 'end_offset': 171}, {'char': 'ra', 'start_offset': 171, 'end_offset': 174}, {'char': 'it', 'start_offset': 174, 'end_offset': 176}], 'word': [{'word': 'well', 'start_offset': 11, 'end_offset': 19}, {'word': 'i', 'start_offset': 19, 'end_offset': 21}, {'word': "don't", 'start_offset': 25, 'end_offset': 27}, {'word': 'wish', 'start_offset': 31, 'end_offset': 33}, {'word': 'to', 'start_offset': 33, 'end_offset': 36}, {'word': 'see', 'start_offset': 39, 'end_offset': 41}, {'word': 'it', 'start_offset': 41, 'end_offset': 44}, {'word': 'any', 'start_offset': 47, 'end_offset': 50}, {'word': 'more', 'start_offset': 52, 'end_offset': 56}, {'word': 'observed', 'start_offset': 64, 'end_offset': 66}, {'word': 'phoebe', 'start_offset': 74, 'end_offset': 83}, {'word': 'turning', 'start_offset': 90, 'end_offset': 92}, {'word': 'away', 'start_offset': 97, 'end_offset': 100}, {'word': 'her', 'start_offset': 102, 'end_offset': 105}, {'word': 'eyes', 'start_offset': 110, 'end_offset': 125}, {'word': 'it', 'start_offset': 125, 'end_offset': 130}, {'word': 'is', 'start_offset': 130, 'end_offset': 135}, {'word': 'certainly', 'start_offset': 143, 'end_offset': 146}, {'word': 'very', 'start_offset': 150, 'end_offset': 152}, {'word': 'like', 'start_offset': 152, 'end_offset': 158}, {'word': 'the', 'start_offset': 158, 'end_offset': 161}, {'word': 'old', 'start_offset': 164, 'end_offset': 167}, {'word': 'portrait', 'start_offset': 164, 'end_offset': 176}]}, alignments=tensor([[-5.6491e+01, -1.8834e+01, -1.4659e+01,  ..., -2.6665e+01,
         -3.1068e+01, -2.8610e-06],
        [-5.7072e+01, -1.8568e+01, -1.7445e+01,  ..., -2.4695e+01,
         -3.3793e+01, -5.9605e-07],
        [-5.7386e+01, -1.8189e+01, -1.9202e+01,  ..., -2.3949e+01,
         -3.3627e+01, -8.3446e-07],
        ...,
        [-5.5325e+01, -1.4839e+01, -1.9966e+01,  ..., -2.1065e+01,
         -3.3125e+01, -1.4305e-05],
        [-5.6287e+01, -1.4943e+01, -2.2234e+01,  ..., -2.3138e+01,
         -3.3042e+01, -7.3909e-06],
        [-5.2566e+01, -1.3785e+01, -2.1438e+01,  ..., -2.2653e+01,
         -2.8750e+01, -1.4543e-05]], device='cuda:0'), length=tensor(186, dtype=torch.int32), y=None, lm_state=None, lm_scores=None, tokens=None, last_token=None)]`

Thanks in advance
Sagar