Repository: Natooz/MidiTok

Issue:
Possible bug when loading saved `PerTok` tokenizer
Hi, many thanks for this great package!

I'm encountering an issue when 1) training the PerTok tokenizer 2) saving the tokenizer 3) re-loading the trained tokenizer 4) encoding a MIDI file. It seems like the `tokenizer.microtiming_tick_values` parameter isn't set when loading from a JSON. 

Here is a minimal example that shows what I mean:

# Example

First, we train the tokenizer:

```
from miditok import PerTok, TokenizerConfig

# Using the parameters from https://miditok.readthedocs.io/en/latest/tokenizations.html#pertok
TOKENIZER_PARAMS = {
    "pitch_range": (21, 109),
    "beat_res": {(0, 4): 4, (0, 4): 3},
    "special_tokens": ["PAD", "BOS", "EOS", "MASK"],
    "use_chords": False,
    "use_rests": False,
    "use_tempos": False,
    "use_time_signatures": True,
    "use_programs": False,
    "use_microtiming": True,
    "ticks_per_quarter": 320,
    "max_microtiming_shift": 0.125,
    "num_microtiming_bins": 30,
}
config = TokenizerConfig(**TOKENIZER_PARAMS)
files_paths = [
    "tests/test_resources/test_midi1/piano_midi.mid",
    "tests/test_resources/test_midi2/piano_midi.mid",
    "tests/test_resources/test_midi3/piano_midi.mid"
]
tokenizer = PerTok(config)
tokenizer.train(vocab_size=500, model="BPE", files_paths=files_paths)
tokenizer.save("tmp.json")
```

After training, it has the `microtiming_tick_values` attribute:

```
hasattr(tokenizer, "microtiming_tick_values")    
>>> True
```

Now, we reload the tokenizer from the JSON we saved, but it no longer has the attribute.

```
newtokenizer = PerTok(params="tmp.json")
hasattr(newtokenizer, "microtiming_tick_values")    
>>> False
```

This causes the following issue when encoding a score:

```
from symusic import Score

score = Score(files_paths[0])
newtokenizer.encode(score)
>>> AttributeError: 'PerTok' object has no attribute 'microtiming_tick_values'
```

My work around is to do something like this (adapting the code [at this line](https://github.com/Natooz/MidiTok/blob/446c0417eb1851cca2063719f2b14e9b5cfcb867/src/miditok/tokenizations/pertok.py#L138)):

```
import numpy as np

newtokenizer = PerTok(params="tmp.json")
mt_bins = newtokenizer.config.additional_params["num_microtiming_bins"]
newtokenizer.microtiming_tick_values = np.linspace(
    -newtokenizer.max_mt_shift,
    newtokenizer.max_mt_shift,
    mt_bins + 1,
    dtype=np.intc
)
score = Score(files_paths[0])
# This will now work
newtokenizer.encode(score)
```

# System information
- Ubuntu 22.04.1
- Python 3.10.12
- Miditok 3.0.5.post1

Happy to send the MIDI files I am using if that will help with debugging.

Hints from issue comments:
@github-actions: This issue is stale because it has been open for 30 days with no activity.

Solution patch:
diff --git a/src/miditok/tokenizations/pertok.py b/src/miditok/tokenizations/pertok.py
index bb3f9e9..b5ed73e 100644
--- a/src/miditok/tokenizations/pertok.py
+++ b/src/miditok/tokenizations/pertok.py
@@ -98,6 +98,12 @@ class PerTok(MusicTokenizer):
             "PitchDrum",
             "Program",
         ]
+        # This will be hit when we're using microtiming
+        # and have loaded a TRAINED tokenizer
+        if self.config.additional_params["use_microtiming"] and not hasattr(
+            self, "microtiming_tick_values"
+        ):
+            self.microtiming_tick_values = self.create_microtiming_tick_values()
 
     def _tweak_config_before_creating_voc(self) -> None:
         self.tpq = self.config.additional_params["ticks_per_quarter"]
@@ -134,11 +140,7 @@ class PerTok(MusicTokenizer):
 
         # Microtiming
         if self.config.additional_params["use_microtiming"]:
-            mt_bins = self.config.additional_params["num_microtiming_bins"]
-            self.microtiming_tick_values = np.linspace(
-                -self.max_mt_shift, self.max_mt_shift, mt_bins + 1, dtype=np.intc
-            )
-
+            self.microtiming_tick_values = self.create_microtiming_tick_values()
             vocab += [
                 f"MicroTiming_{microtiming!s}"
                 for microtiming in self.microtiming_tick_values
@@ -181,6 +183,20 @@ class PerTok(MusicTokenizer):
             desc=f"duration {note.duration}",
         )
 
+    def create_microtiming_tick_values(self) -> NDArray:
+        """
+        Generate tick-based microtiming tokens.
+
+        Returns
+        -------
+            NDArray: Array of available microtiming values
+
+        """
+        mt_bins = self.config.additional_params["num_microtiming_bins"]
+        return np.linspace(
+            -self.max_mt_shift, self.max_mt_shift, mt_bins + 1, dtype=np.intc
+        )
+
     def create_timeshift_tick_values(self) -> NDArray:
         """
         Generate tick-based timeshift tokens.


Test patch:
diff --git a/tests/test_saving_loading_config.py b/tests/test_saving_loading_config.py
index c5594dd..68ae00d 100644
--- a/tests/test_saving_loading_config.py
+++ b/tests/test_saving_loading_config.py
@@ -8,7 +8,12 @@ from typing import TYPE_CHECKING, Any
 import miditok
 import pytest
 
-from .utils_tests import ALL_TOKENIZATIONS, MAX_BAR_EMBEDDING, MIDI_PATHS_MULTITRACK
+from .utils_tests import (
+    ALL_TOKENIZATIONS,
+    MAX_BAR_EMBEDDING,
+    MIDI_PATHS_MULTITRACK,
+    MIDI_PATHS_ONE_TRACK,
+)
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -118,3 +123,24 @@ def test_multitrack_midi_to_tokens_to_midi(
 
     # Assert tokens are the same
     assert tokens == tokens_loaded
+
+
+@pytest.mark.parametrize("file_path", MIDI_PATHS_ONE_TRACK[:3], ids=lambda p: p.name)
+def test_pertok_microtiming_tick_values(file_path: Path):
+    # Create the pertok tokenizer
+    cfg = miditok.TokenizerConfig(
+        use_chords=False,
+        use_microtiming=True,
+        ticks_per_quarter=480,
+        max_microtiming_shift=0.25,
+        num_microtiming_bins=110,
+    )
+    tok = miditok.PerTok(cfg)
+    # Train the tokenizer
+    tok.train(files_paths=[file_path], vocab_size=1000)
+    # Dump the tokenizer to a JSON
+    tok.save("tmp.json")
+    # Reload the tokenizer
+    newtok = miditok.PerTok(params="tmp.json")
+    # Should still have the microtiming_tick_values parameter
+    assert hasattr(newtok, "microtiming_tick_values")