Repository: huggingface/tokenizers (language: rust)

Issue:
sequence_ids and token_to_sequence return incorrect result for left padding
When using left padding, the sequence ids are not shifted accordingly.

# minimal upstream example:

```
from transformers import GPT2TokenizerFast
t = GPT2TokenizerFast.from_pretrained("gpt2")
t.add_special_tokens({'pad_token': '[PAD]'})
t.padding_side = "left"
e = t("ABC","DEF",padding="max_length",max_length=3)
print(list(zip(e['input_ids'],e.sequence_ids())))
```

# expected output:
`[(50257, None), (24694, 0), (32988, 1)]`

# actual output:
`[(50257, 0), (24694, 1), (32988, None)]`


Discussion / hints:
@Narsil looks like you know something about GPT2 tokenizers 👍 

Fix patch:
diff --git a/tokenizers/src/tokenizer/encoding.rs b/tokenizers/src/tokenizer/encoding.rs
index c6274c2f2..40576efea 100644
--- a/tokenizers/src/tokenizer/encoding.rs
+++ b/tokenizers/src/tokenizer/encoding.rs
@@ -512,6 +512,11 @@ impl Encoding {
                     .map(|_| (0, 0))
                     .chain(self.offsets.drain(..))
                     .collect();
+                self.sequence_ranges
+                    .iter_mut()
+                    .for_each(|(_seq_id, range)| {
+                        *range = (range.start + pad_length)..(range.end + pad_length)
+                    });
             }
             PaddingDirection::Right => {
                 self.ids.extend((0..pad_length).map(|_| pad_id));
@@ -874,4 +879,31 @@ mod tests {
         assert_eq!(encoding.char_to_word(2, 1), Some(0));
         assert_eq!(encoding.char_to_word(9, 1), Some(2));
     }
+
+    #[test]
+    fn padding() {
+        let mut a = Encoding {
+            ids: vec![1],
+            type_ids: vec![0],
+            tokens: vec![String::from("Hello ")],
+            words: vec![Some(0)],
+            offsets: vec![(0, 6)],
+            special_tokens_mask: vec![0],
+            attention_mask: vec![1],
+            sequence_ranges: HashMap::from([(0, 0..1)]),
+            ..Default::default()
+        };
+        let target_length = 2;
+        let pad_id = 99;
+        let pad_type_id = 0;
+        let pad_token = "[PAD]";
+        a.pad(
+            target_length,
+            pad_id,
+            pad_type_id,
+            pad_token,
+            PaddingDirection::Left,
+        );
+        assert_eq!(a.sequence_ranges, HashMap::from([(0, 1..2)]));
+    }
 }


Test patch: