Add the initial models

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,3 @@
+models/**/*.npy
+models/**/*.zst
+models/**/*.json

README.md

@@ -1,3 +1,67 @@
----
-license: mpl-2.0
----
+---
+tags:
+- static-embeddings
+---
+# Static Embeddings
+
+This project contains multilingual static embeddings that are appropriate for generating
+quick embeddings in edge devices. They are re-packaged from other projects in production
+ready assets.
+
+## Models
+
+* [minishlab/potion-retrieval-32M/](models/minishlab/potion-retrieval-32M/README.md)
+* [minishlab/potion-multilingual-128M/](models/minishlab/potion-multilingual-128M/README.md)
+* [sentence-transformers/static-retrieval-mrl-en-v1/](models/sentence-transformers/static-retrieval-mrl-en-v1/README.md)
+* [sentence-transformers/static-similarity-mrl-multilingual-v1](models/sentence-transformers/static-similarity-mrl-multilingual-v1/README.md)
+
+## Updating
+
+Add models to `scripts/build_models.py`.
+
+```sh
+# Install dependencies and login to huggingface:
+pipx install huggingface_hub
+huggingface-cli login
+
+# Re-build the models:
+uv run scripts/build_models.py
+
+# Version control:
+git add .
+git commit -m 'Updated the models'
+git push
+git tag v1.0.0 -m 'Model release description'
+git push origin tag v1.0.0
+
+# Upload the models
+uv run scripts/upload_models.py --tag v1.0.0
+```
+
+## Precision
+
+For static embeddings and cosine similarity, precision isn't as important. For an end
+to end to test in Firefox on some vectors here was the cosine similarity for the same
+mean pooled result. Note that the vector math happens in the f32 space, but storage
+for the embeddings is in a lower precision.
+
+> f32 vs f16: cosine similarity = 1.00000000<br/>
+> → They are essentially identical in direction.
+>
+> f32 vs f8: cosine similarity = 0.99956375<br/>
+> → Very close, only tiny quantization effects.
+
+Note that this was done on the `torch.float8_e4m3fn`, while `torch.float8_e5m2` generally
+has more loss.
+
+Precision also affects download size. For instance with larger
+[minishlab/potion-multilingual-128M/](models/minishlab/potion-multilingual-128M/README.md)
+model. The `fp32` is 228M compressed, while only 51M for `fp8_e4m3`, which has competetive
+quantization values.
+
+| precision     | dimensions | size    |
+| ------------- | ---------- | ------- |
+| fp32          | 128        | 228M    |
+| fp16          | 128        | 114M    |
+| **fp8_e4m3**  | 128        | **51M** |
+| fp8_e5m2      | 128        |  44M    |

build_models.py

@@ -0,0 +1,411 @@
+from dataclasses import dataclass
+import shutil
+from textwrap import dedent, indent
+from typing import Any
+import numpy as np
+from zstandard import ZstdCompressor
+from pathlib import Path
+import io
+from sentence_transformers import SentenceTransformer
+from torch.nn import EmbeddingBag
+import torch
+from model2vec import StaticModel
+from tokenizers import Encoding, Tokenizer
+
+models_path = Path("models")
+
+
+@dataclass
+class ModelCard:
+    owner: str
+    repo: str
+    # The dimensions that were applied with Matroyshka Loss.
+    matroyshka_dims: list[int]
+    description: str
+    license: str
+
+    def name(self):
+        return f"{self.owner}/{self.repo}"
+
+    def path(self):
+        return models_path / self.owner / self.repo
+
+    def get_description(self):
+        return dedent(self.description).strip()
+
+
+def zst_compress_file(input: Path):
+    cctx = ZstdCompressor()
+    output = input.parent / f"{input.name}.zst"
+    print(f"Compressing {output}")
+    with open(input, "rb") as fin, open(output, "wb") as fout:
+        cctx.copy_stream(fin, fout)
+
+
+def save_data(path: Path, tensor: torch.Tensor):
+    """Writes out the static embeddings to a .npy and .npy.zst file"""
+    buffer = io.BytesIO()
+
+    if tensor.dtype in (torch.float8_e4m3fn, torch.float8_e5m2):
+        # Store as the raw bytes.
+        np.save(buffer, tensor.detach().view(torch.uint8).numpy())
+    else:
+        np.save(buffer, tensor.detach().numpy())
+
+    print(f"Saving {path}")
+    with (open(path, "wb") as outfile,):
+        outfile.write(buffer.getvalue())
+
+    zst_compress_file(path)
+
+
+def quantization_loss_mse(tensor: torch.Tensor, dtype: torch.dtype):
+    """
+    Compute reconstruction loss when converting embeddings to a datatype and back using
+    the mean squared error, which punishes big errors more than small ones.
+    """
+
+    # Original → quantize → dequantize
+    roundtrip = tensor.detach().to(dtype).to(tensor.dtype)
+
+    # Mean squared error
+    return torch.mean((tensor - roundtrip) ** 2).item()
+
+
+def quantization_loss_mae(tensor: torch.Tensor, dtype: torch.dtype):
+    """
+    Compute reconstruction loss when converting embeddings to a datatype and back using
+    the mean absolute error, which is less sensitive to outliers than MSE.
+    """
+
+    # Original → quantize → dequantize
+    roundtrip = tensor.detach().to(dtype).to(tensor.dtype)
+
+    # Mean absolute error
+    return torch.mean(torch.abs(tensor - roundtrip)).item()
+
+
+def quantization_loss_cosine(tensor: torch.Tensor, dtype: torch.dtype):
+    """
+    Compute reconstruction loss when converting embeddings to a datatype and back using
+    cosine similarity. This measures whether the embedding directions are preserved
+    after quantization, independent of their magnitudes.
+    """
+
+    # Original → quantize → dequantize
+    roundtrip = tensor.detach().to(dtype).to(tensor.dtype)
+
+    # Flatten both to 2D (num_vectors, dimensions) in case tensor is 1D or higher-D
+    if tensor.ndim == 1:
+        orig = tensor.unsqueeze(0)
+        recon = roundtrip.unsqueeze(0)
+    else:
+        orig = tensor.view(tensor.shape[0], -1)
+        recon = roundtrip.view(roundtrip.shape[0], -1)
+
+    # Cosine similarity per vector, then average
+    cos = torch.nn.functional.cosine_similarity(orig, recon, dim=1)
+    return cos.mean().item()
+
+
+def export_embeddings(model_card: ModelCard, embeddings: torch.Tensor) -> None:
+    vocab_size, dimensions = embeddings.shape
+
+    # This logic can always be adjusted for models with different shapes.
+    assert (
+        embeddings.dtype == torch.float32
+    ), f"The embeddings {embeddings.dtype} are assumed to be float32."
+
+    for dim in model_card.matroyshka_dims:
+        assert (
+            dim <= dimensions
+        ), f"The Matroyshka dimensions {dim} were bigger than the models dimensions of {dimensions}"
+
+        truncated = embeddings[:, :dim]
+        assert truncated.shape == torch.Size([vocab_size, dim])
+
+        save_data(model_card.path() / f"fp32.d{dim}.npy", truncated)
+        save_data(
+            model_card.path() / f"fp16.d{dim}.npy",
+            truncated.to(dtype=torch.float16),
+        )
+        save_data(
+            model_card.path() / f"fp8_e5m2.d{dim}.npy",
+            truncated.to(dtype=torch.float8_e5m2),
+        )
+        save_data(
+            model_card.path() / f"fp8_e4m3.d{dim}.npy",
+            truncated.to(dtype=torch.float8_e4m3fn),
+        )
+
+
+def normalized_mean_pooling(x: torch.Tensor) -> torch.Tensor:
+    pooled = x.mean(dim=0)
+    normalized = torch.nn.functional.normalize(pooled, dim=0)
+    return normalized
+
+
+def export_readme(
+    model_card: ModelCard,
+    embeddings: torch.Tensor,
+    tokenizer: Tokenizer,
+):
+    vocab_size, dimensions = embeddings.shape
+    norms = torch.norm(embeddings, dim=1)  # shape: [vocab_size]
+
+    phrases = [
+        "The committee approved the proposal after hours of heated discussion and several last-minute amendments."
+        "When training large neural networks, careful tuning of hyperparameters can significantly affect performance and stability."
+        "Despite the heavy rain, the concert continued as planned and the crowd stayed enthusiastic until the final encore."
+        "In ancient mythology, heroes often embarked on perilous journeys to discover hidden truths about themselves and their world."
+        "The new smartphone model features an improved camera system, faster processing, and extended battery life compared to its predecessor."
+        "He tried to explain the concept using simple analogies, but the underlying mathematics remained difficult to grasp for most listeners."
+        "After weeks of negotiations, the two countries signed a historic trade agreement aimed at reducing tariffs and boosting cooperation."
+        "She paused for a moment before answering, choosing her words carefully to avoid misunderstanding in such a delicate situation."
+        "The detective pieced together the timeline of events, realizing that the key witness had provided a contradictory statement."
+        "Remote work has changed the way teams collaborate, with online tools replacing traditional office routines and in-person meetings."
+    ]
+
+    cosine_similarity = {
+        torch.float16: [],
+        torch.float8_e4m3fn: [],
+        torch.float8_e5m2: [],
+    }
+
+    for phrase in phrases:
+        encoding: Encoding = tokenizer.encode(phrase)
+        embedded_phrase = embeddings[torch.tensor(encoding.ids, dtype=torch.long)]
+
+        for dtype in cosine_similarity.keys():
+            pooling_unquantized = normalized_mean_pooling(embedded_phrase)
+            pooling_roundtrip = normalized_mean_pooling(
+                embedded_phrase.to(dtype).to(torch.float32)
+            )
+            cosine = torch.dot(pooling_unquantized, pooling_roundtrip).item()
+            cosine_similarity[dtype].append(cosine)
+
+    avg_cosine_similarity = {
+        dtype: sum(values) / len(values) for dtype, values in cosine_similarity.items()
+    }
+
+    tokenizer_examples = []
+    for text in [
+        "This is an example of encoding",
+        "The quick brown fox jumps over the lazy dog.",
+        "Curaçao, naïve fiancé, jalapeño, déjà vu.",
+        "Привет, как дела?",
+        "Бързата кафява лисица прескача мързеливото куче.",
+        "Γρήγορη καφέ αλεπού πηδάει πάνω από τον τεμπέλη σκύλο.",
+        "اللغة العربية جميلة وغنية بالتاريخ.",
+        "مرحبا بالعالم!",
+        "Simplified: 快速的棕色狐狸跳过懒狗。",
+        "Traditional: 快速的棕色狐狸跳過懶狗。",
+        "素早い茶色の狐が怠け者の犬を飛び越える。",
+        "コンピュータープログラミング",
+        "빠른 갈색 여우가 게으른 개를 뛰어넘습니다.",
+        "तेज़ भूरी लोमड़ी आलसी कुत्ते के ऊपर कूदती है।",
+        "দ্রুত বাদামী শিয়াল অলস কুকুরের উপর দিয়ে লাফ দেয়।",
+        "வேகமான பழுப்பு நரி சோம்பேறி நாயின் மேல் குதிக்கிறது.",
+        "สุนัขจิ้งจอกสีน้ำตาลกระโดดข้ามสุนัขขี้เกียจ.",
+        "ብሩክ ቡናማ ቀበሮ ሰነፍ ውሻን ተዘልሏል።",
+        "Hello 世界 مرحبا 🌍",
+        "123, αβγ, абв, العربية, 中文, हिन्दी.",
+    ]:
+        encoding = tokenizer.encode(text)
+        tokens = [f"`{token}`" for token in encoding.tokens]
+
+        tokenizer_examples.append(f"**Input:** {text}<br/>")
+        tokenizer_examples.append(f"**Tokens**: {' '.join(tokens)}")
+        tokenizer_examples.append("")
+
+    tokenizer_output = "\n".join(tokenizer_examples)
+
+    with (model_card.path() / "README.md").open("wt") as file:
+        prefix = "                "
+
+        file.write(
+            dedent(
+                f"""
+                # [{model_card.name()}](https://huggingface.co/{model_card.name()})
+                
+                License: [{model_card.license}](https://choosealicense.com/licenses/{model_card.license}/)
+                
+                {indent(model_card.get_description(), prefix).strip()}
+                
+                ## Model Stats
+                
+                Stats that describe the embeddings tensor shapes and value distribution.
+
+                | item          | metric                  | value |
+                | --------------| ----------------------- | ----- |
+                | vocab         | size                    | {vocab_size:,.0f} |
+                | embedding     | dimensions              | {dimensions:,.0f} |
+                | vector length | mean                    | {norms.mean().item():.2f} |
+                | vector length | median                  | {norms.median().item():.2f} |
+                | vector length | stddev                  | {norms.std().item():.2f} |
+                | values        | mean                    | {embeddings.mean().item():.2f} |
+                | values        | median                  | {embeddings.median().item():.2f} |
+                | values        | stddev                  | {embeddings.std().item():.2f} |
+                
+                ## Mean Pooled Quantization Loss
+                
+                This test roundtrips the vectors through quantization, but performs the
+                mean pooling arithmetic in float32 space. The quantized and unquantized
+                mean pooled vectors are compared to each other to determine their cosine
+                similarity, to show how much the meaning of the vector has changed due
+                to quantization.
+                
+                | Precision     | Cosine Similarity |
+                | ------------- | ----------------- |
+                | fp16          | {avg_cosine_similarity[torch.float16]:.5f} |
+                | fp8 e4m3      | {avg_cosine_similarity[torch.float8_e4m3fn]:.5f} |
+                | fp8 e5m2      | {avg_cosine_similarity[torch.float8_e5m2]:.5f} |
+                
+                ## Quantization Loss Per Vector
+                
+                While ultimately the embedding vectors will be mean pooled together, it's
+                still useful to look at the loss per-vector in the embedding table to see
+                which quantization strategies retain the most vector meaning.
+                
+                - **Cosine Similarity** — measures how well the *direction* of embedding vectors
+                is preserved after quantization, independent of scale. This is especially
+                relevant when embeddings are used for similarity search or retrieval.
+                - **MSE (Mean Squared Error)** — emphasizes large errors by squaring the
+                differences. Useful for detecting whether any values are badly distorted.
+                - **MAE (Mean Absolute Error)** — the average absolute difference between
+                original and quantized values. Easier to interpret, less sensitive to outliers.
+
+                | Precision     | Metric | Value |
+                | ------------- | ------ | ----- |
+                | fp16          | cosine similarity | {quantization_loss_cosine(embeddings, torch.float16):.5f} |
+                | fp8 e4m3      | cosine similarity | {quantization_loss_cosine(embeddings, torch.float8_e4m3fn):.5f} |
+                | fp8 e5m2      | cosine similarity | {quantization_loss_cosine(embeddings, torch.float8_e5m2):.5f} |
+                | fp16          | MSE    | {quantization_loss_mse(embeddings, torch.float16):.5f} |
+                | fp8 e4m3      | MSE    | {quantization_loss_mse(embeddings, torch.float8_e4m3fn):.5f} |
+                | fp8 e5m2      | MSE    | {quantization_loss_mse(embeddings, torch.float8_e5m2):.5f} |
+                | fp16          | MAE    | {quantization_loss_mae(embeddings, torch.float16):.5f} |
+                | fp8 e4m3      | MAE    | {quantization_loss_mae(embeddings, torch.float8_e4m3fn):.5f} |
+                | fp8 e5m2      | MAE    | {quantization_loss_mae(embeddings, torch.float8_e5m2):.5f} |
+                
+                ## Tokenizer Examples
+                
+                {indent(tokenizer_output, prefix).strip()}
+                """
+            ).strip()
+        )
+
+
+def export_tokenizer(model_card: ModelCard, tokenizer: Tokenizer) -> None:
+    tokenizer_path = model_card.path() / "tokenizer.json"
+    print(f"Exporting tokenizer: {tokenizer_path}")
+    tokenizer.save(str(tokenizer_path))
+    zst_compress_file(tokenizer_path)
+
+
+def export_sentence_transformers(model_card: ModelCard) -> None:
+    """Extract the embeddings and tokenizer from SentenceTransformers"""
+
+    print("Processing", model_card.name())
+
+    model = SentenceTransformer(model_card.name(), device="cpu")
+    embedding_bag: EmbeddingBag = model[0].embedding  # type: ignore
+    model_card.path().mkdir(exist_ok=True, parents=True)
+    embeddings = torch.Tensor(embedding_bag.weight)
+
+    export_embeddings(model_card, embeddings)
+    export_tokenizer(model_card, model.tokenizer)
+    export_readme(model_card, embeddings, model.tokenizer)
+
+
+def export_model2vec(model_card: ModelCard) -> None:
+    """Extract the embeddings and tokenizer from model2vec"""
+
+    print("Processing", model_card.name())
+
+    model = StaticModel.from_pretrained(model_card.name())
+    model_card.path().mkdir(exist_ok=True, parents=True)
+    embeddings = torch.from_numpy(model.embedding)
+    export_embeddings(model_card, embeddings)
+    export_tokenizer(model_card, model.tokenizer)
+    export_readme(model_card, embeddings, model.tokenizer)
+
+
+def main() -> None:
+    # Static embedders that use sentence_transformers models.
+    sentence_transformers_models = [
+        ModelCard(
+            owner="sentence-transformers",
+            repo="static-similarity-mrl-multilingual-v1",
+            description="""
+            Multi-lingual similarity embeddings that were trained with Matroyshka loss
+            that allows for more effective truncation of the embedding vectors. It
+            was trained on a variety of domains of multilingual datasets.
+            
+            It's a general purpose model that can be used for semantic textual similarity,
+            paraphrase mining, text classification, clustering, and more
+            """,
+            matroyshka_dims=[32, 64, 128, 256, 512, 1024],
+            license="apache-2.0",
+        ),
+        ModelCard(
+            owner="sentence-transformers",
+            repo="static-retrieval-mrl-en-v1",
+            description="""
+            English-only uncased similarity embeddings that were trained with Matroyshka
+            loss that allows for more effective truncation of the embedding vectors. It
+            was trained on a variety of domains of monolingual datasets. I was designed
+            specifically for similarity retrieval.
+            """,
+            matroyshka_dims=[32, 64, 128, 256, 512, 1024],
+            license="apache-2.0",
+        ),
+    ]
+    # Static embedders that use model2vec.
+    model2vec_models = [
+        ModelCard(
+            owner="minishlab",
+            repo="potion-multilingual-128M",
+            # These are assumed as their is no python reference implementation:
+            matroyshka_dims=[32, 64, 128, 256],
+            description="""
+            A multilingual embedder. The details are a bit scant on how it's trained as
+            there is no source code for it. However, it's likely a close architecture
+            to the potion-retrieval-32M model, but trained on Common Crawl data.
+            
+            The 128M references the number of parameters in the embeddings:
+            
+            256 dimensions * 500,353 vocab.
+            """,
+            license="mit",
+        ),
+        ModelCard(
+            owner="minishlab",
+            repo="potion-retrieval-32M",
+            matroyshka_dims=[32, 64, 128, 256, 512],
+            description="""
+            The token embeddings from a monolingual English 32M parameter model that was
+            distilled from embeddings that were initialized from the the multi-domain
+            [BAAI/bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5)
+            
+            The 32M references the number of parameters in the embeddings:
+            
+            512 dimension * 63,091 vocab.
+            """,
+            license="mit",
+        ),
+    ]
+
+    if models_path.exists():
+        print(f"Removing the old models folder: {models_path}")
+        shutil.rmtree(models_path)
+        models_path.mkdir()
+
+    for model_card in sentence_transformers_models:
+        export_sentence_transformers(model_card)
+
+    for model_card in model2vec_models:
+        export_model2vec(model_card)
+
+
+if __name__ == "__main__":
+    main()

example.mjs

@@ -0,0 +1,75 @@
+import { pipeline, AutoTokenizer, AutoModel, TokenizerModel, PreTrainedTokenizer } from '@huggingface/transformers';
+import fs from 'node:fs/promises';
+import { constants } from 'node:fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const DIR = path.dirname(fileURLToPath(import.meta.url));
+
+await main()
+
+async function main() {
+  const url = "https://huggingface.co/sentence-transformers/static-similarity-mrl-multilingual-v1/resolve/main/0_StaticEmbedding/tokenizer.json"
+
+  const config = await ensureTokenizerJson(url)
+  const tokenizer = new PreTrainedTokenizer(config, {})
+  
+  const examples = [
+    "This is an example of encoding",
+    "The quick brown fox jumps over the lazy dog.",
+    "Curaçao, naïve fiancé, jalapeño, déjà vu.",
+    "Привет, как дела?",
+    "Бързата кафява лисица прескача мързеливото куче.",
+    "Γρήγορη καφέ αλεπού πηδάει πάνω από τον τεμπέλη σκύλο.",
+    "اللغة العربية جميلة وغنية بالتاريخ.",
+    "مرحبا بالعالم!",
+    "Simplified: 快速的棕色狐狸跳过懒狗。",
+    "Traditional: 快速的棕色狐狸跳過懶狗。",
+    "素早い茶色の狐が怠け者の犬を飛び越える。",
+    "コンピュータープログラミング",
+    "빠른 갈색 여우가 게으른 개를 뛰어넘습니다.",
+    "तेज़ भूरी लोमड़ी आलसी कुत्ते के ऊपर कूदती है।",
+    "দ্রুত বাদামী শিয়াল অলস কুকুরের উপর দিয়ে লাফ দেয়।",
+    "வேகமான பழுப்பு நரி சோம்பேறி நாயின் மேல் குதிக்கிறது.",
+    "สุนัขจิ้งจอกสีน้ำตาลกระโดดข้ามสุนัขขี้เกียจ.",
+    "ብሩክ ቡናማ ቀበሮ ሰነፍ ውሻን ተዘልሏል።",
+    // Mixed scripts:
+    "Hello 世界 مرحبا 🌍",
+    "123, αβγ, абв, العربية, 中文, हिन्दी.",
+  ];
+  for (const example of examples) {
+    console.log(tokenizer.tokenize(example))
+  }
+}
+
+/**
+ * @param {string} path
+ * @returns {Promise<string>}
+ */
+async function loadJSON(path) {
+  return JSON.parse(await fs.readFile(path, { encoding: 'utf8' }));
+}
+
+/**
+ * Download tokenizer.json if it does not already exist.
+ *
+ * @param {string} url - The URL to download tokenizer.json from
+ * @returns {Promise<any>} - Path to tokenizer.json
+ */
+export async function ensureTokenizerJson(url) {
+  const tokenizerPath = path.join(DIR, 'tokenizer.json');
+
+  try {
+    await fs.access(tokenizerPath, constants.F_OK);
+    console.log('Using', tokenizerPath);
+    return loadJSON(tokenizerPath);
+  } catch {}
+  
+  console.log("Downloading", url);
+  const response = await fetch(url);
+  const data = Buffer.from(await response.arrayBuffer());
+  await fs.writeFile(tokenizerPath, data);
+  
+  return loadJSON(tokenizerPath);
+
+}

experiments/multilingual.py

@@ -0,0 +1,59 @@
+from sentence_transformers import SentenceTransformer
+from tokenizers import Encoding, Tokenizer
+from torch.nn import EmbeddingBag
+import torch
+
+
+def test_tokenizer():
+    examples = [
+        "This is an example of encoding",
+        "The quick brown fox jumps over the lazy dog.",
+        "Curaçao, naïve fiancé, jalapeño, déjà vu.",
+        "Привет, как дела?",
+        "Бързата кафява лисица прескача мързеливото куче.",
+        "Γρήγορη καφέ αλεπού πηδάει πάνω από τον τεμπέλη σκύλο.",
+        "اللغة العربية جميلة وغنية بالتاريخ.",
+        "مرحبا بالعالم!",
+        "Simplified: 快速的棕色狐狸跳过懒狗。",
+        "Traditional: 快速的棕色狐狸跳過懶狗。",
+        "素早い茶色の狐が怠け者の犬を飛び越える。",
+        "コンピュータープログラミング",
+        "빠른 갈색 여우가 게으른 개를 뛰어넘습니다.",
+        "तेज़ भूरी लोमड़ी आलसी कुत्ते के ऊपर कूदती है।",
+        "দ্রুত বাদামী শিয়াল অলস কুকুরের উপর দিয়ে লাফ দেয়।",
+        "வேகமான பழுப்பு நரி சோம்பேறி நாயின் மேல் குதிக்கிறது.",
+        "สุนัขจิ้งจอกสีน้ำตาลกระโดดข้ามสุนัขขี้เกียจ.",
+        "ብሩክ ቡናማ ቀበሮ ሰነፍ ውሻን ተዘልሏል።",
+        "Hello 世界 مرحبا 🌍",
+        "123, αβγ, абв, العربية, 中文, हिन्दी.",
+    ]
+
+    tokenizer: Tokenizer = Tokenizer.from_file("js/tokenizer.json")
+
+    for example in examples:
+        encoding: Encoding = tokenizer.encode(example)
+        print(example)
+        print(encoding.tokens)
+        print()
+
+
+# https://huggingface.co/sentence-transformers/static-similarity-mrl-multilingual-v1
+model = SentenceTransformer(
+    "sentence-transformers/static-similarity-mrl-multilingual-v1", device="cpu"
+)
+
+embedding_bag: EmbeddingBag = model[0].embedding  # type: ignore
+embeddings = torch.Tensor(embedding_bag.weight)
+
+print(embeddings.shape)
+assert embeddings.shape == torch.Size([105879, 1024])
+
+print("float32")
+print(f"  1024 dim - {embeddings.shape[0] * 1024 * 4 / 1024 / 1024:,.1f} MiB")
+print(f"   512 dim - {embeddings.shape[0] * 512 * 4 / 1024 / 1024:,.1f} MiB")
+print(f"   256 dim - {embeddings.shape[0] * 256 * 4 / 1024 / 1024:,.1f} MiB")
+
+print("float16")
+print(f"  1024 dim - {embeddings.shape[0] * 1024 * 2 / 1024 / 1024:,.1f} MiB")
+print(f"   512 dim - {embeddings.shape[0] * 512 * 2 / 1024 / 1024:,.1f} MiB")
+print(f"   256 dim - {embeddings.shape[0] * 256 * 2 / 1024 / 1024:,.1f} MiB")

experiments/potion.py

@@ -0,0 +1,14 @@
+from model2vec import StaticModel
+from tokenizers import Tokenizer
+import torch
+
+model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M")
+embeddings = torch.from_numpy(model.embedding)
+
+print("Embedding shape:", embeddings.shape)
+bytes = embeddings.shape[0] * embeddings.shape[1] * 4
+
+print("MiB:", bytes / 1024 / 1024)
+
+tokenizer: Tokenizer = model.tokenizer
+print(tokenizer.to_str())

experiments/tomaarsen.py

@@ -0,0 +1,15 @@
+from sentence_transformers import SentenceTransformer
+from torch.nn import EmbeddingBag
+import torch
+
+model = SentenceTransformer("tomaarsen/static-retrieval-mrl-en-v1")
+embedding_bag: EmbeddingBag = model[0].embedding  # type: ignore
+embeddings = torch.Tensor(embedding_bag.weight)
+
+assert embeddings.shape == torch.Size([30522, 1024])
+
+print(f"1024 dim - {embeddings.shape[0] * 1024 * 4 / 1024 / 1024:,.1f} MiB:")
+print(f"512 dim - {embeddings.shape[0] * 512 * 4 / 1024 / 1024:,.1f} MiB:")
+print(f"256 dim - {embeddings.shape[0] * 256 * 4 / 1024 / 1024:,.1f} MiB:")
+
+print("Embeddings[0]", embeddings[0])

js/example.mjs

@@ -0,0 +1,75 @@
+import { pipeline, AutoTokenizer, AutoModel, TokenizerModel, PreTrainedTokenizer } from '@huggingface/transformers';
+import fs from 'node:fs/promises';
+import { constants } from 'node:fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const DIR = path.dirname(fileURLToPath(import.meta.url));
+
+await main()
+
+async function main() {
+  const url = "https://huggingface.co/sentence-transformers/static-similarity-mrl-multilingual-v1/resolve/main/0_StaticEmbedding/tokenizer.json"
+
+  const config = await ensureTokenizerJson(url)
+  const tokenizer = new PreTrainedTokenizer(config, {})
+  
+  const examples = [
+    "This is an example of encoding",
+    "The quick brown fox jumps over the lazy dog.",
+    "Curaçao, naïve fiancé, jalapeño, déjà vu.",
+    "Привет, как дела?",
+    "Бързата кафява лисица прескача мързеливото куче.",
+    "Γρήγορη καφέ αλεπού πηδάει πάνω από τον τεμπέλη σκύλο.",
+    "اللغة العربية جميلة وغنية بالتاريخ.",
+    "مرحبا بالعالم!",
+    "Simplified: 快速的棕色狐狸跳过懒狗。",
+    "Traditional: 快速的棕色狐狸跳過懶狗。",
+    "素早い茶色の狐が怠け者の犬を飛び越える。",
+    "コンピュータープログラミング",
+    "빠른 갈색 여우가 게으른 개를 뛰어넘습니다.",
+    "तेज़ भूरी लोमड़ी आलसी कुत्ते के ऊपर कूदती है।",
+    "দ্রুত বাদামী শিয়াল অলস কুকুরের উপর দিয়ে লাফ দেয়।",
+    "வேகமான பழுப்பு நரி சோம்பேறி நாயின் மேல் குதிக்கிறது.",
+    "สุนัขจิ้งจอกสีน้ำตาลกระโดดข้ามสุนัขขี้เกียจ.",
+    "ብሩክ ቡናማ ቀበሮ ሰነፍ ውሻን ተዘልሏል።",
+    // Mixed scripts:
+    "Hello 世界 مرحبا 🌍",
+    "123, αβγ, абв, العربية, 中文, हिन्दी.",
+  ];
+  for (const example of examples) {
+    console.log(tokenizer.tokenize(example))
+  }
+}
+
+/**
+ * @param {string} path
+ * @returns {Promise<string>}
+ */
+async function loadJSON(path) {
+  return JSON.parse(await fs.readFile(path, { encoding: 'utf8' }));
+}
+
+/**
+ * Download tokenizer.json if it does not already exist.
+ *
+ * @param {string} url - The URL to download tokenizer.json from
+ * @returns {Promise<any>} - Path to tokenizer.json
+ */
+export async function ensureTokenizerJson(url) {
+  const tokenizerPath = path.join(DIR, 'tokenizer.json');
+
+  try {
+    await fs.access(tokenizerPath, constants.F_OK);
+    console.log('Using', tokenizerPath);
+    return loadJSON(tokenizerPath);
+  } catch {}
+  
+  console.log("Downloading", url);
+  const response = await fetch(url);
+  const data = Buffer.from(await response.arrayBuffer());
+  await fs.writeFile(tokenizerPath, data);
+  
+  return loadJSON(tokenizerPath);
+
+}

js/package-lock.json

@@ -0,0 +1,1067 @@
+{
+  "name": "js",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "js",
+      "version": "1.0.0",
+      "license": "ISC",
+      "dependencies": {
+        "@huggingface/transformers": "^3.7.2"
+      }
+    },
+    "node_modules/@emnapi/runtime": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.5.0.tgz",
+      "integrity": "sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==",
+      "license": "MIT",
+      "optional": true,
+      "dependencies": {
+        "tslib": "^2.4.0"
+      }
+    },
+    "node_modules/@huggingface/jinja": {
+      "version": "0.5.1",
+      "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.1.tgz",
+      "integrity": "sha512-yUZLld4lrM9iFxHCwFQ7D1HW2MWMwSbeB7WzWqFYDWK+rEb+WldkLdAJxUPOmgICMHZLzZGVcVjFh3w/YGubng==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@huggingface/transformers": {
+      "version": "3.7.2",
+      "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.7.2.tgz",
+      "integrity": "sha512-6SOxo6XziupnQ5Vs5vbbs74CNB6ViHLHGQJjY6zj88JeiDtJ2d/ADKxaay688Sf2KcjtdF3dyBL11C5pJS2NxQ==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@huggingface/jinja": "^0.5.1",
+        "onnxruntime-node": "1.21.0",
+        "onnxruntime-web": "1.22.0-dev.20250409-89f8206ba4",
+        "sharp": "^0.34.1"
+      }
+    },
+    "node_modules/@img/sharp-darwin-arm64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.3.tgz",
+      "integrity": "sha512-ryFMfvxxpQRsgZJqBd4wsttYQbCxsJksrv9Lw/v798JcQ8+w84mBWuXwl+TT0WJ/WrYOLaYpwQXi3sA9nTIaIg==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-darwin-arm64": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-darwin-x64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.3.tgz",
+      "integrity": "sha512-yHpJYynROAj12TA6qil58hmPmAwxKKC7reUqtGLzsOHfP7/rniNGTL8tjWX6L3CTV4+5P4ypcS7Pp+7OB+8ihA==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-darwin-x64": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-libvips-darwin-arm64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.0.tgz",
+      "integrity": "sha512-sBZmpwmxqwlqG9ueWFXtockhsxefaV6O84BMOrhtg/YqbTaRdqDE7hxraVE3y6gVM4eExmfzW4a8el9ArLeEiQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-darwin-x64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.0.tgz",
+      "integrity": "sha512-M64XVuL94OgiNHa5/m2YvEQI5q2cl9d/wk0qFTDVXcYzi43lxuiFTftMR1tOnFQovVXNZJ5TURSDK2pNe9Yzqg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linux-arm": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.0.tgz",
+      "integrity": "sha512-mWd2uWvDtL/nvIzThLq3fr2nnGfyr/XMXlq8ZJ9WMR6PXijHlC3ksp0IpuhK6bougvQrchUAfzRLnbsen0Cqvw==",
+      "cpu": [
+        "arm"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linux-arm64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.0.tgz",
+      "integrity": "sha512-RXwd0CgG+uPRX5YYrkzKyalt2OJYRiJQ8ED/fi1tq9WQW2jsQIn0tqrlR5l5dr/rjqq6AHAxURhj2DVjyQWSOA==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linux-ppc64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.0.tgz",
+      "integrity": "sha512-Xod/7KaDDHkYu2phxxfeEPXfVXFKx70EAFZ0qyUdOjCcxbjqyJOEUpDe6RIyaunGxT34Anf9ue/wuWOqBW2WcQ==",
+      "cpu": [
+        "ppc64"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linux-s390x": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.0.tgz",
+      "integrity": "sha512-eMKfzDxLGT8mnmPJTNMcjfO33fLiTDsrMlUVcp6b96ETbnJmd4uvZxVJSKPQfS+odwfVaGifhsB07J1LynFehw==",
+      "cpu": [
+        "s390x"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linux-x64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.0.tgz",
+      "integrity": "sha512-ZW3FPWIc7K1sH9E3nxIGB3y3dZkpJlMnkk7z5tu1nSkBoCgw2nSRTFHI5pB/3CQaJM0pdzMF3paf9ckKMSE9Tg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linuxmusl-arm64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.0.tgz",
+      "integrity": "sha512-UG+LqQJbf5VJ8NWJ5Z3tdIe/HXjuIdo4JeVNADXBFuG7z9zjoegpzzGIyV5zQKi4zaJjnAd2+g2nna8TZvuW9Q==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linuxmusl-x64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.0.tgz",
+      "integrity": "sha512-SRYOLR7CXPgNze8akZwjoGBoN1ThNZoqpOgfnOxmWsklTGVfJiGJoC/Lod7aNMGA1jSsKWM1+HRX43OP6p9+6Q==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-linux-arm": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.3.tgz",
+      "integrity": "sha512-oBK9l+h6KBN0i3dC8rYntLiVfW8D8wH+NPNT3O/WBHeW0OQWCjfWksLUaPidsrDKpJgXp3G3/hkmhptAW0I3+A==",
+      "cpu": [
+        "arm"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linux-arm": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-linux-arm64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.3.tgz",
+      "integrity": "sha512-QdrKe3EvQrqwkDrtuTIjI0bu6YEJHTgEeqdzI3uWJOH6G1O8Nl1iEeVYRGdj1h5I21CqxSvQp1Yv7xeU3ZewbA==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linux-arm64": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-linux-ppc64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.3.tgz",
+      "integrity": "sha512-GLtbLQMCNC5nxuImPR2+RgrviwKwVql28FWZIW1zWruy6zLgA5/x2ZXk3mxj58X/tszVF69KK0Is83V8YgWhLA==",
+      "cpu": [
+        "ppc64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linux-ppc64": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-linux-s390x": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.3.tgz",
+      "integrity": "sha512-3gahT+A6c4cdc2edhsLHmIOXMb17ltffJlxR0aC2VPZfwKoTGZec6u5GrFgdR7ciJSsHT27BD3TIuGcuRT0KmQ==",
+      "cpu": [
+        "s390x"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linux-s390x": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-linux-x64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.3.tgz",
+      "integrity": "sha512-8kYso8d806ypnSq3/Ly0QEw90V5ZoHh10yH0HnrzOCr6DKAPI6QVHvwleqMkVQ0m+fc7EH8ah0BB0QPuWY6zJQ==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linux-x64": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-linuxmusl-arm64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.3.tgz",
+      "integrity": "sha512-vAjbHDlr4izEiXM1OTggpCcPg9tn4YriK5vAjowJsHwdBIdx0fYRsURkxLG2RLm9gyBq66gwtWI8Gx0/ov+JKQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linuxmusl-arm64": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-linuxmusl-x64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.3.tgz",
+      "integrity": "sha512-gCWUn9547K5bwvOn9l5XGAEjVTTRji4aPTqLzGXHvIr6bIDZKNTA34seMPgM0WmSf+RYBH411VavCejp3PkOeQ==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linuxmusl-x64": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-wasm32": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.3.tgz",
+      "integrity": "sha512-+CyRcpagHMGteySaWos8IbnXcHgfDn7pO2fiC2slJxvNq9gDipYBN42/RagzctVRKgxATmfqOSulgZv5e1RdMg==",
+      "cpu": [
+        "wasm32"
+      ],
+      "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT",
+      "optional": true,
+      "dependencies": {
+        "@emnapi/runtime": "^1.4.4"
+      },
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-win32-arm64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.3.tgz",
+      "integrity": "sha512-MjnHPnbqMXNC2UgeLJtX4XqoVHHlZNd+nPt1kRPmj63wURegwBhZlApELdtxM2OIZDRv/DFtLcNhVbd1z8GYXQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0 AND LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-win32-ia32": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.3.tgz",
+      "integrity": "sha512-xuCdhH44WxuXgOM714hn4amodJMZl3OEvf0GVTm0BEyMeA2to+8HEdRPShH0SLYptJY1uBw+SCFP9WVQi1Q/cw==",
+      "cpu": [
+        "ia32"
+      ],
+      "license": "Apache-2.0 AND LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-win32-x64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.3.tgz",
+      "integrity": "sha512-OWwz05d++TxzLEv4VnsTz5CmZ6mI6S05sfQGEMrNrQcOEERbX46332IvE7pO/EUiw7jUrrS40z/M7kPyjfl04g==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0 AND LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@isaacs/fs-minipass": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz",
+      "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==",
+      "license": "ISC",
+      "dependencies": {
+        "minipass": "^7.0.4"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      }
+    },
+    "node_modules/@protobufjs/aspromise": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
+      "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/base64": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
+      "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/codegen": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
+      "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/eventemitter": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
+      "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/fetch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
+      "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "@protobufjs/aspromise": "^1.1.1",
+        "@protobufjs/inquire": "^1.1.0"
+      }
+    },
+    "node_modules/@protobufjs/float": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
+      "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/inquire": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
+      "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/path": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
+      "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/pool": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
+      "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/utf8": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
+      "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@types/node": {
+      "version": "24.3.1",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.3.1.tgz",
+      "integrity": "sha512-3vXmQDXy+woz+gnrTvuvNrPzekOi+Ds0ReMxw0LzBiK3a+1k0kQn9f2NWk+lgD4rJehFUmYy2gMhJ2ZI+7YP9g==",
+      "license": "MIT",
+      "dependencies": {
+        "undici-types": "~7.10.0"
+      }
+    },
+    "node_modules/boolean": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz",
+      "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==",
+      "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
+      "license": "MIT"
+    },
+    "node_modules/chownr": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
+      "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==",
+      "license": "BlueOak-1.0.0",
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/color": {
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
+      "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==",
+      "license": "MIT",
+      "dependencies": {
+        "color-convert": "^2.0.1",
+        "color-string": "^1.9.0"
+      },
+      "engines": {
+        "node": ">=12.5.0"
+      }
+    },
+    "node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "license": "MIT",
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "license": "MIT"
+    },
+    "node_modules/color-string": {
+      "version": "1.9.1",
+      "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz",
+      "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==",
+      "license": "MIT",
+      "dependencies": {
+        "color-name": "^1.0.0",
+        "simple-swizzle": "^0.2.2"
+      }
+    },
+    "node_modules/define-data-property": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz",
+      "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==",
+      "license": "MIT",
+      "dependencies": {
+        "es-define-property": "^1.0.0",
+        "es-errors": "^1.3.0",
+        "gopd": "^1.0.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/define-properties": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz",
+      "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==",
+      "license": "MIT",
+      "dependencies": {
+        "define-data-property": "^1.0.1",
+        "has-property-descriptors": "^1.0.0",
+        "object-keys": "^1.1.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/detect-libc": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz",
+      "integrity": "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/detect-node": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz",
+      "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==",
+      "license": "MIT"
+    },
+    "node_modules/es-define-property": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
+      "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-errors": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
+      "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es6-error": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz",
+      "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==",
+      "license": "MIT"
+    },
+    "node_modules/escape-string-regexp": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
+      "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/flatbuffers": {
+      "version": "25.2.10",
+      "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.2.10.tgz",
+      "integrity": "sha512-7JlN9ZvLDG1McO3kbX0k4v+SUAg48L1rIwEvN6ZQl/eCtgJz9UylTMzE9wrmYrcorgxm3CX/3T/w5VAub99UUw==",
+      "license": "Apache-2.0"
+    },
+    "node_modules/global-agent": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz",
+      "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "boolean": "^3.0.1",
+        "es6-error": "^4.1.1",
+        "matcher": "^3.0.0",
+        "roarr": "^2.15.3",
+        "semver": "^7.3.2",
+        "serialize-error": "^7.0.1"
+      },
+      "engines": {
+        "node": ">=10.0"
+      }
+    },
+    "node_modules/globalthis": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz",
+      "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==",
+      "license": "MIT",
+      "dependencies": {
+        "define-properties": "^1.2.1",
+        "gopd": "^1.0.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/gopd": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
+      "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/guid-typescript": {
+      "version": "1.0.9",
+      "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz",
+      "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==",
+      "license": "ISC"
+    },
+    "node_modules/has-property-descriptors": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz",
+      "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==",
+      "license": "MIT",
+      "dependencies": {
+        "es-define-property": "^1.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-arrayish": {
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz",
+      "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==",
+      "license": "MIT"
+    },
+    "node_modules/json-stringify-safe": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
+      "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==",
+      "license": "ISC"
+    },
+    "node_modules/long": {
+      "version": "5.3.2",
+      "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz",
+      "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==",
+      "license": "Apache-2.0"
+    },
+    "node_modules/matcher": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz",
+      "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==",
+      "license": "MIT",
+      "dependencies": {
+        "escape-string-regexp": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/minipass": {
+      "version": "7.1.2",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
+      "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      }
+    },
+    "node_modules/minizlib": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.2.tgz",
+      "integrity": "sha512-oG62iEk+CYt5Xj2YqI5Xi9xWUeZhDI8jjQmC5oThVH5JGCTgIjr7ciJDzC7MBzYd//WvR1OTmP5Q38Q8ShQtVA==",
+      "license": "MIT",
+      "dependencies": {
+        "minipass": "^7.1.2"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/mkdirp": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz",
+      "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==",
+      "license": "MIT",
+      "bin": {
+        "mkdirp": "dist/cjs/src/bin.js"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/object-keys": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
+      "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/onnxruntime-common": {
+      "version": "1.21.0",
+      "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.21.0.tgz",
+      "integrity": "sha512-Q632iLLrtCAVOTO65dh2+mNbQir/QNTVBG3h/QdZBpns7mZ0RYbLRBgGABPbpU9351AgYy7SJf1WaeVwMrBFPQ==",
+      "license": "MIT"
+    },
+    "node_modules/onnxruntime-node": {
+      "version": "1.21.0",
+      "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.21.0.tgz",
+      "integrity": "sha512-NeaCX6WW2L8cRCSqy3bInlo5ojjQqu2fD3D+9W5qb5irwxhEyWKXeH2vZ8W9r6VxaMPUan+4/7NDwZMtouZxEw==",
+      "hasInstallScript": true,
+      "license": "MIT",
+      "os": [
+        "win32",
+        "darwin",
+        "linux"
+      ],
+      "dependencies": {
+        "global-agent": "^3.0.0",
+        "onnxruntime-common": "1.21.0",
+        "tar": "^7.0.1"
+      }
+    },
+    "node_modules/onnxruntime-web": {
+      "version": "1.22.0-dev.20250409-89f8206ba4",
+      "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.22.0-dev.20250409-89f8206ba4.tgz",
+      "integrity": "sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ==",
+      "license": "MIT",
+      "dependencies": {
+        "flatbuffers": "^25.1.24",
+        "guid-typescript": "^1.0.9",
+        "long": "^5.2.3",
+        "onnxruntime-common": "1.22.0-dev.20250409-89f8206ba4",
+        "platform": "^1.3.6",
+        "protobufjs": "^7.2.4"
+      }
+    },
+    "node_modules/onnxruntime-web/node_modules/onnxruntime-common": {
+      "version": "1.22.0-dev.20250409-89f8206ba4",
+      "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.22.0-dev.20250409-89f8206ba4.tgz",
+      "integrity": "sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ==",
+      "license": "MIT"
+    },
+    "node_modules/platform": {
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz",
+      "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==",
+      "license": "MIT"
+    },
+    "node_modules/protobufjs": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz",
+      "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==",
+      "hasInstallScript": true,
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "@protobufjs/aspromise": "^1.1.2",
+        "@protobufjs/base64": "^1.1.2",
+        "@protobufjs/codegen": "^2.0.4",
+        "@protobufjs/eventemitter": "^1.1.0",
+        "@protobufjs/fetch": "^1.1.0",
+        "@protobufjs/float": "^1.0.2",
+        "@protobufjs/inquire": "^1.1.0",
+        "@protobufjs/path": "^1.1.2",
+        "@protobufjs/pool": "^1.1.0",
+        "@protobufjs/utf8": "^1.1.0",
+        "@types/node": ">=13.7.0",
+        "long": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
+    "node_modules/roarr": {
+      "version": "2.15.4",
+      "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz",
+      "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "boolean": "^3.0.1",
+        "detect-node": "^2.0.4",
+        "globalthis": "^1.0.1",
+        "json-stringify-safe": "^5.0.1",
+        "semver-compare": "^1.0.0",
+        "sprintf-js": "^1.1.2"
+      },
+      "engines": {
+        "node": ">=8.0"
+      }
+    },
+    "node_modules/semver": {
+      "version": "7.7.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz",
+      "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==",
+      "license": "ISC",
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/semver-compare": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz",
+      "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==",
+      "license": "MIT"
+    },
+    "node_modules/serialize-error": {
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz",
+      "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==",
+      "license": "MIT",
+      "dependencies": {
+        "type-fest": "^0.13.1"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/sharp": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.3.tgz",
+      "integrity": "sha512-eX2IQ6nFohW4DbvHIOLRB3MHFpYqaqvXd3Tp5e/T/dSH83fxaNJQRvDMhASmkNTsNTVF2/OOopzRCt7xokgPfg==",
+      "hasInstallScript": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "color": "^4.2.3",
+        "detect-libc": "^2.0.4",
+        "semver": "^7.7.2"
+      },
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-darwin-arm64": "0.34.3",
+        "@img/sharp-darwin-x64": "0.34.3",
+        "@img/sharp-libvips-darwin-arm64": "1.2.0",
+        "@img/sharp-libvips-darwin-x64": "1.2.0",
+        "@img/sharp-libvips-linux-arm": "1.2.0",
+        "@img/sharp-libvips-linux-arm64": "1.2.0",
+        "@img/sharp-libvips-linux-ppc64": "1.2.0",
+        "@img/sharp-libvips-linux-s390x": "1.2.0",
+        "@img/sharp-libvips-linux-x64": "1.2.0",
+        "@img/sharp-libvips-linuxmusl-arm64": "1.2.0",
+        "@img/sharp-libvips-linuxmusl-x64": "1.2.0",
+        "@img/sharp-linux-arm": "0.34.3",
+        "@img/sharp-linux-arm64": "0.34.3",
+        "@img/sharp-linux-ppc64": "0.34.3",
+        "@img/sharp-linux-s390x": "0.34.3",
+        "@img/sharp-linux-x64": "0.34.3",
+        "@img/sharp-linuxmusl-arm64": "0.34.3",
+        "@img/sharp-linuxmusl-x64": "0.34.3",
+        "@img/sharp-wasm32": "0.34.3",
+        "@img/sharp-win32-arm64": "0.34.3",
+        "@img/sharp-win32-ia32": "0.34.3",
+        "@img/sharp-win32-x64": "0.34.3"
+      }
+    },
+    "node_modules/simple-swizzle": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz",
+      "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==",
+      "license": "MIT",
+      "dependencies": {
+        "is-arrayish": "^0.3.1"
+      }
+    },
+    "node_modules/sprintf-js": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz",
+      "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/tar": {
+      "version": "7.4.3",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz",
+      "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==",
+      "license": "ISC",
+      "dependencies": {
+        "@isaacs/fs-minipass": "^4.0.0",
+        "chownr": "^3.0.0",
+        "minipass": "^7.1.2",
+        "minizlib": "^3.0.1",
+        "mkdirp": "^3.0.1",
+        "yallist": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/tslib": {
+      "version": "2.8.1",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+      "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
+      "license": "0BSD",
+      "optional": true
+    },
+    "node_modules/type-fest": {
+      "version": "0.13.1",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz",
+      "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==",
+      "license": "(MIT OR CC0-1.0)",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/undici-types": {
+      "version": "7.10.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.10.0.tgz",
+      "integrity": "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag==",
+      "license": "MIT"
+    },
+    "node_modules/yallist": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz",
+      "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==",
+      "license": "BlueOak-1.0.0",
+      "engines": {
+        "node": ">=18"
+      }
+    }
+  }
+}

js/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "js",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@huggingface/transformers": "^3.7.2"
+  }
+}

js/tokenizer_config.json

@@ -0,0 +1 @@
+{}

js/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "module": "ESNext",
+    "moduleResolution": "nodenext",
+    // Set the baseUrl to the root of the project.
+    "baseUrl": "src",
+    // Make the type checking as strict as possible.
+    "strict": true,
+    // TypeScript will check JS files only if they have a @ts-check comment in them.
+    "allowJs": true,
+    "checkJs": true,
+    // Only type check, don't emit files.
+    "noEmit": true,
+    // Allow esnext syntax. Otherwise the default is ES5 only.
+    "target": "esnext",
+    "lib": ["esnext", "dom"],
+    "esModuleInterop": true
+  },
+  // Add a @ts-check comment to a JS file to start type checking it.
+  "include": ["example.mjs"],
+  // "files": ["src/@types/globals.d.ts"],
+  "exclude": []
+}

models/minishlab/potion-multilingual-128M/README.md

@@ -0,0 +1,128 @@
+# [minishlab/potion-multilingual-128M](https://huggingface.co/minishlab/potion-multilingual-128M)
+
+License: [mit](https://choosealicense.com/licenses/mit/)
+
+A multilingual embedder. The details are a bit scant on how it's trained as
+there is no source code for it. However, it's likely a close architecture
+to the potion-retrieval-32M model, but trained on Common Crawl data.
+
+The 128M references the number of parameters in the embeddings:
+
+256 dimensions * 500,353 vocab.
+
+## Model Stats
+
+Stats that describe the embeddings tensor shapes and value distribution.
+
+| item          | metric                  | value |
+| --------------| ----------------------- | ----- |
+| vocab         | size                    | 500,353 |
+| embedding     | dimensions              | 256 |
+| vector length | mean                    | 12.73 |
+| vector length | median                  | 11.94 |
+| vector length | stddev                  | 5.12 |
+| values        | mean                    | -0.00 |
+| values        | median                  | -0.00 |
+| values        | stddev                  | 0.86 |
+
+## Mean Pooled Quantization Loss
+
+This test roundtrips the vectors through quantization, but performs the
+mean pooling arithmetic in float32 space. The quantized and unquantized
+mean pooled vectors are compared to each other to determine their cosine
+similarity, to show how much the meaning of the vector has changed due
+to quantization.
+
+| Precision     | Cosine Similarity |
+| ------------- | ----------------- |
+| fp16          | 1.00000 |
+| fp8 e4m3      | 0.99993 |
+| fp8 e5m2      | 0.99973 |
+
+## Quantization Loss Per Vector
+
+While ultimately the embedding vectors will be mean pooled together, it's
+still useful to look at the loss per-vector in the embedding table to see
+which quantization strategies retain the most vector meaning.
+
+- **Cosine Similarity** — measures how well the *direction* of embedding vectors
+is preserved after quantization, independent of scale. This is especially
+relevant when embeddings are used for similarity search or retrieval.
+- **MSE (Mean Squared Error)** — emphasizes large errors by squaring the
+differences. Useful for detecting whether any values are badly distorted.
+- **MAE (Mean Absolute Error)** — the average absolute difference between
+original and quantized values. Easier to interpret, less sensitive to outliers.
+
+| Precision     | Metric | Value |
+| ------------- | ------ | ----- |
+| fp16          | cosine similarity | 1.00000 |
+| fp8 e4m3      | cosine similarity | 0.99965 |
+| fp8 e5m2      | cosine similarity | 0.99863 |
+| fp16          | MSE    | 0.00000 |
+| fp8 e4m3      | MSE    | 0.00052 |
+| fp8 e5m2      | MSE    | 0.00205 |
+| fp16          | MAE    | 0.00011 |
+| fp8 e4m3      | MAE    | 0.01364 |
+| fp8 e5m2      | MAE    | 0.02717 |
+
+## Tokenizer Examples
+
+**Input:** This is an example of encoding<br/>
+**Tokens**: `▁This` `▁is` `▁an` `▁example` `▁of` `▁encoding`
+
+**Input:** The quick brown fox jumps over the lazy dog.<br/>
+**Tokens**: `▁The` `▁quick` `▁brown` `▁fox` `▁jumps` `▁over` `▁the` `▁lazy` `▁dog` `▁` `.`
+
+**Input:** Curaçao, naïve fiancé, jalapeño, déjà vu.<br/>
+**Tokens**: `▁Cura` `ça` `o` `▁` `,` `▁na` `ï` `ve` `▁fiancé` `▁` `,` `▁ja` `lap` `eño` `▁` `,` `▁déjà` `▁vu` `▁` `.`
+
+**Input:** Привет, как дела?<br/>
+**Tokens**: `▁При` `вет` `▁` `,` `▁как` `▁дела` `▁?`
+
+**Input:** Бързата кафява лисица прескача мързеливото куче.<br/>
+**Tokens**: `▁Бър` `за` `та` `▁кафяв` `а` `▁лис` `ица` `▁пре` `ска` `ча` `▁` `мър` `зе` `ливо` `то` `▁куче` `▁` `.`
+
+**Input:** Γρήγορη καφέ αλεπού πηδάει πάνω από τον τεμπέλη σκύλο.<br/>
+**Tokens**: `▁Γ` `ρή` `γο` `ρη` `▁καφέ` `▁α` `λε` `πού` `▁` `πη` `δά` `ει` `▁πάνω` `▁από` `▁τον` `▁τε` `μπ` `έλη` `▁σκύλο` `▁` `.`
+
+**Input:** اللغة العربية جميلة وغنية بالتاريخ.<br/>
+**Tokens**: `▁اللغة` `▁العربية` `▁جميلة` `▁وغ` `نية` `▁بال` `تاريخ` `▁` `.`
+
+**Input:** مرحبا بالعالم!<br/>
+**Tokens**: `▁مرحبا` `▁بالعالم` `▁!`
+
+**Input:** Simplified: 快速的棕色狐狸跳过懒狗。<br/>
+**Tokens**: `▁Simp` `l` `ified` `▁:` `▁` `快速` `的` `棕` `色` `狐` `狸` `跳` `过` `懒` `狗` `。`
+
+**Input:** Traditional: 快速的棕色狐狸跳過懶狗。<br/>
+**Tokens**: `▁Tradition` `al` `▁:` `▁` `快速` `的` `棕` `色` `狐` `狸` `跳` `過` `懶` `狗` `。`
+
+**Input:** 素早い茶色の狐が怠け者の犬を飛び越える。<br/>
+**Tokens**: `▁素` `早い` `茶` `色` `の` `狐` `が` `怠` `け` `者の` `犬` `を` `飛び` `越` `える` `。`
+
+**Input:** コンピュータープログラミング<br/>
+**Tokens**: `▁` `コンピュ��タ` `ー` `プロ` `グラ` `ミ` `ング`
+
+**Input:** 빠른 갈색 여우가 게으른 개를 뛰어넘습니다.<br/>
+**Tokens**: `▁빠른` `▁갈` `색` `▁여` `우` `가` `▁게` `으` `른` `▁` `개를` `▁뛰어` `넘` `습니다` `▁` `.`
+
+**Input:** तेज़ भूरी लोमड़ी आलसी कुत्ते के ऊपर कूदती है।<br/>
+**Tokens**: `▁तेज़` `▁भू` `री` `▁लो` `म` `ड़ी` `▁आ` `ल` `सी` `▁कुत्ते` `▁के` `▁ऊपर` `▁` `कू` `द` `ती` `▁है` `।`
+
+**Input:** দ্রুত বাদামী শিয়াল অলস কুকুরের উপর দিয়ে লাফ দেয়।<br/>
+**Tokens**: `▁দ্রুত` `▁বাদাম` `ী` `▁শি` `য়াল` `▁অ` `ল` `স` `▁কু` `কুর` `ের` `▁উপর` `▁দিয়ে` `▁লা` `ফ` `▁দেয়` `।`
+
+**Input:** வேகமான பழுப்பு நரி சோம்பேறி நாயின் மேல் குதிக்கிறது.<br/>
+**Tokens**: `▁வேக` `மான` `▁பழ` `ு` `ப்பு` `▁ந` `ரி` `▁சோ` `ம்` `பே` `றி` `▁நா` `யின்` `▁மேல்` `▁கு` `தி` `க்கிறது` `▁` `.`
+
+**Input:** สุนัขจิ้งจอกสีน้ำตาลกระโดดข้ามสุนัขขี้เกียจ.<br/>
+**Tokens**: `▁` `สุนัข` `จิ` `้ง` `จอ` `ก` `สีน้ําตาล` `กระโดด` `ข้าม` `สุนัข` `ขี้` `เกีย` `จ` `▁` `.`
+
+**Input:** ብሩክ ቡናማ ቀበሮ ሰነፍ ውሻን ተዘልሏል።<br/>
+**Tokens**: `▁` `ብሩ` `ክ` `▁ቡና` `ማ` `▁` `ቀበ` `ሮ` `▁ሰ` `ነፍ` `▁` `ው` `ሻ` `ን` `▁ተ` `ዘ` `ል` `ሏል` `።`
+
+**Input:** Hello 世界 مرحبا 🌍<br/>
+**Tokens**: `▁Hello` `▁世界` `▁مرحبا` `▁🌍`
+
+**Input:** 123, αβγ, абв, العربية, 中文, हिन्दी.<br/>
+**Tokens**: `▁123` `▁` `,` `▁α` `β` `γ` `▁` `,` `▁аб` `в` `▁` `,` `▁العربية` `▁` `,` `▁中文` `▁` `,` `▁हिन्दी` `▁` `.`

models/minishlab/potion-retrieval-32M/README.md

@@ -0,0 +1,128 @@
+# [minishlab/potion-retrieval-32M](https://huggingface.co/minishlab/potion-retrieval-32M)
+
+License: [mit](https://choosealicense.com/licenses/mit/)
+
+The token embeddings from a monolingual English 32M parameter model that was
+distilled from embeddings that were initialized from the the multi-domain
+[BAAI/bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5)
+
+The 32M references the number of parameters in the embeddings:
+
+512 dimension * 63,091 vocab.
+
+## Model Stats
+
+Stats that describe the embeddings tensor shapes and value distribution.
+
+| item          | metric                  | value |
+| --------------| ----------------------- | ----- |
+| vocab         | size                    | 63,091 |
+| embedding     | dimensions              | 512 |
+| vector length | mean                    | 130.27 |
+| vector length | median                  | 130.39 |
+| vector length | stddev                  | 30.43 |
+| values        | mean                    | 0.01 |
+| values        | median                  | 0.01 |
+| values        | stddev                  | 5.91 |
+
+## Mean Pooled Quantization Loss
+
+This test roundtrips the vectors through quantization, but performs the
+mean pooling arithmetic in float32 space. The quantized and unquantized
+mean pooled vectors are compared to each other to determine their cosine
+similarity, to show how much the meaning of the vector has changed due
+to quantization.
+
+| Precision     | Cosine Similarity |
+| ------------- | ----------------- |
+| fp16          | 1.00000 |
+| fp8 e4m3      | 0.99970 |
+| fp8 e5m2      | 0.99887 |
+
+## Quantization Loss Per Vector
+
+While ultimately the embedding vectors will be mean pooled together, it's
+still useful to look at the loss per-vector in the embedding table to see
+which quantization strategies retain the most vector meaning.
+
+- **Cosine Similarity** — measures how well the *direction* of embedding vectors
+is preserved after quantization, independent of scale. This is especially
+relevant when embeddings are used for similarity search or retrieval.
+- **MSE (Mean Squared Error)** — emphasizes large errors by squaring the
+differences. Useful for detecting whether any values are badly distorted.
+- **MAE (Mean Absolute Error)** — the average absolute difference between
+original and quantized values. Easier to interpret, less sensitive to outliers.
+
+| Precision     | Metric | Value |
+| ------------- | ------ | ----- |
+| fp16          | cosine similarity | 1.00000 |
+| fp8 e4m3      | cosine similarity | 0.99965 |
+| fp8 e5m2      | cosine similarity | 0.99862 |
+| fp16          | MSE    | 0.00000 |
+| fp8 e4m3      | MSE    | 0.02454 |
+| fp8 e5m2      | MSE    | 0.09720 |
+| fp16          | MAE    | 0.00076 |
+| fp8 e4m3      | MAE    | 0.09763 |
+| fp8 e5m2      | MAE    | 0.19461 |
+
+## Tokenizer Examples
+
+**Input:** This is an example of encoding<br/>
+**Tokens**: `[CLS]` `this` `is` `an` `example` `of` `encoding` `[SEP]`
+
+**Input:** The quick brown fox jumps over the lazy dog.<br/>
+**Tokens**: `[CLS]` `the` `quick` `brown` `fox` `jumps` `over` `the` `lazy` `dog` `.` `[SEP]`
+
+**Input:** Curaçao, naïve fiancé, jalapeño, déjà vu.<br/>
+**Tokens**: `[CLS]` `curacao` `,` `naive` `fiance` `,` `jalapeno` `,` `deja` `vu` `.` `[SEP]`
+
+**Input:** Привет, как дела?<br/>
+**Tokens**: `[CLS]` `п` `##р` `##и` `##в` `##е` `##т` `,` `как` `д` `##е` `##л` `##а` `?` `[SEP]`
+
+**Input:** Бързата кафява лисица прескача мързеливото куче.<br/>
+**Tokens**: `[CLS]` `б` `##ъ` `##р` `##з` `##а` `##т` `##а` `к` `##а` `##ф` `##я` `##в` `##а` `л` `##и` `##с` `##и` `##ц` `##а` `п` `##р` `##е` `##с` `##ка` `##ч` `##а` `м` `##ъ` `##р` `##з` `##е` `##л` `##и` `##в` `##о` `##т` `##о` `к` `##у` `##ч` `##е` `.` `[SEP]`
+
+**Input:** Γρήγορη καφέ αλεπού πηδάει πάνω από τον τεμπέλη σκύλο.<br/>
+**Tokens**: `[CLS]` `γ` `##ρ` `##η` `##γ` `##ο` `##ρ` `##η` `κ` `##α` `##φ` `##ε` `α` `##λ` `##ε` `##π` `##ου` `π` `##η` `##δ` `##α` `##ε` `##ι` `π` `##α` `##ν` `##ω` `α` `##π` `##ο` `τ` `##ο` `##ν` `τ` `##ε` `##μ` `##π` `##ε` `##λ` `##η` `σ` `##κ` `##υ` `##λ` `##ο` `.` `[SEP]`
+
+**Input:** اللغة العربية جميلة وغنية بالتاريخ.<br/>
+**Tokens**: `[CLS]` `ا` `##ل` `##ل` `##غ` `##ة` `ا` `##ل` `##ع` `##ر` `##ب` `##ي` `##ة` `ج` `##م` `##ي` `##ل` `##ة` `و` `##غ` `##ن` `##ي` `##ة` `با` `##ل` `##ت` `##ا` `##ر` `##ي` `##خ` `.` `[SEP]`
+
+**Input:** مرحبا بالعالم!<br/>
+**Tokens**: `[CLS]` `م` `##ر` `##ح` `##ب` `##ا` `با` `##ل` `##ع` `##ا` `##ل` `##م` `!` `[SEP]`
+
+**Input:** Simplified: 快速的棕色狐狸跳过懒狗。<br/>
+**Tokens**: `[CLS]` `simplified` `:` `[UNK]` `[UNK]` `的` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `。` `[SEP]`
+
+**Input:** Traditional: 快速的棕色狐狸跳過懶狗。<br/>
+**Tokens**: `[CLS]` `traditional` `:` `[UNK]` `[UNK]` `的` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `。` `[SEP]`
+
+**Input:** 素早い茶色の狐が怠け者の犬を飛び越える。<br/>
+**Tokens**: `[CLS]` `[UNK]` `[UNK]` `い` `[UNK]` `[UNK]` `の` `[UNK]` `か` `[UNK]` `け` `[UNK]` `の` `犬` `を` `[UNK]` `ひ` `[UNK]` `え` `##る` `。` `[SEP]`
+
+**Input:** コンピュータープログラミング<br/>
+**Tokens**: `[CLS]` `コ` `##ン` `##ヒ` `##ュ` `##ー` `##タ` `##ー` `##フ` `##ロ` `##ク` `##ラ` `##ミ` `##ン` `##ク` `[SEP]`
+
+**Input:** 빠른 갈색 여우가 게으른 개를 뛰어넘습니다.<br/>
+**Tokens**: `[CLS]` `[UNK]` `ᄀ` `##ᅡ` `##ᆯ` `##ᄉ` `##ᅢ` `##ᆨ` `ᄋ` `##ᅧ` `##ᄋ` `##ᅮ` `##ᄀ` `##ᅡ` `ᄀ` `##ᅦ` `##ᄋ` `##ᅳ` `##ᄅ` `##ᅳ` `##ᆫ` `ᄀ` `##ᅢ` `##ᄅ` `##ᅳ` `##ᆯ` `[UNK]` `.` `[SEP]`
+
+**Input:** तेज़ भूरी लोमड़ी आलसी कुत्ते के ऊपर कूदती है।<br/>
+**Tokens**: `[CLS]` `त` `##ज` `भ` `##र` `##ी` `ल` `##ो` `##म` `##ड` `##ी` `आ` `##ल` `##स` `##ी` `क` `##त` `##त` `क` `[UNK]` `क` `##द` `##त` `##ी` `ह` `।` `[SEP]`
+
+**Input:** দ্রুত বাদামী শিয়াল অলস কুকুরের উপর দিয়ে লাফ দেয়।<br/>
+**Tokens**: `[CLS]` `দ` `##র` `##ত` `ব` `##া` `##দ` `##া` `##ম` `##ী` `শ` `##ি` `##য` `##া` `##ল` `অ` `##ল` `##স` `ক` `##ক` `##র` `##ে` `##র` `উ` `##প` `##র` `দ` `##ি` `##য` `##ে` `[UNK]` `দ` `##ে` `##য` `।` `[SEP]`
+
+**Input:** வேகமான பழுப்பு நரி சோம்பேறி நாயின் மேல் குதிக்கிறது.<br/>
+**Tokens**: `[CLS]` `வ` `##ே` `##க` `##ம` `##ா` `##ன` `[UNK]` `ந` `##ர` `##ி` `[UNK]` `ந` `##ா` `##ய` `##ி` `##ன` `ம` `##ே` `##ல` `[UNK]` `.` `[SEP]`
+
+**Input:** สุนัขจิ้งจอกสีน้ำตาลกระโดดข้ามสุนัขขี้เกียจ.<br/>
+**Tokens**: `[CLS]` `[UNK]` `.` `[SEP]`
+
+**Input:** ብሩክ ቡናማ ቀበሮ ሰነፍ ውሻን ተዘልሏል።<br/>
+**Tokens**: `[CLS]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[SEP]`
+
+**Input:** Hello 世界 مرحبا 🌍<br/>
+**Tokens**: `[CLS]` `hello` `世` `[UNK]` `م` `##ر` `##ح` `##ب` `##ا` `[UNK]` `[SEP]`
+
+**Input:** 123, αβγ, абв, العربية, 中文, हिन्दी.<br/>
+**Tokens**: `[CLS]` `123` `,` `α` `##β` `##γ` `,` `а` `##б` `##в` `,` `ا` `##ل` `##ع` `##ر` `##ب` `##ي` `##ة` `,` `中` `文` `,` `ह` `##ि` `##न` `##द` `##ी` `.` `[SEP]`

models/sentence-transformers/static-retrieval-mrl-en-v1/README.md

@@ -0,0 +1,125 @@
+# [sentence-transformers/static-retrieval-mrl-en-v1](https://huggingface.co/sentence-transformers/static-retrieval-mrl-en-v1)
+
+License: [apache-2.0](https://choosealicense.com/licenses/apache-2.0/)
+
+English-only uncased similarity embeddings that were trained with Matroyshka
+loss that allows for more effective truncation of the embedding vectors. It
+was trained on a variety of domains of monolingual datasets. I was designed
+specifically for similarity retrieval.
+
+## Model Stats
+
+Stats that describe the embeddings tensor shapes and value distribution.
+
+| item          | metric                  | value |
+| --------------| ----------------------- | ----- |
+| vocab         | size                    | 30,522 |
+| embedding     | dimensions              | 1,024 |
+| vector length | mean                    | 555.04 |
+| vector length | median                  | 573.92 |
+| vector length | stddev                  | 219.06 |
+| values        | mean                    | 0.02 |
+| values        | median                  | 0.01 |
+| values        | stddev                  | 18.65 |
+
+## Mean Pooled Quantization Loss
+
+This test roundtrips the vectors through quantization, but performs the
+mean pooling arithmetic in float32 space. The quantized and unquantized
+mean pooled vectors are compared to each other to determine their cosine
+similarity, to show how much the meaning of the vector has changed due
+to quantization.
+
+| Precision     | Cosine Similarity |
+| ------------- | ----------------- |
+| fp16          | 1.00000 |
+| fp8 e4m3      | 0.99972 |
+| fp8 e5m2      | 0.99887 |
+
+## Quantization Loss Per Vector
+
+While ultimately the embedding vectors will be mean pooled together, it's
+still useful to look at the loss per-vector in the embedding table to see
+which quantization strategies retain the most vector meaning.
+
+- **Cosine Similarity** — measures how well the *direction* of embedding vectors
+is preserved after quantization, independent of scale. This is especially
+relevant when embeddings are used for similarity search or retrieval.
+- **MSE (Mean Squared Error)** — emphasizes large errors by squaring the
+differences. Useful for detecting whether any values are badly distorted.
+- **MAE (Mean Absolute Error)** — the average absolute difference between
+original and quantized values. Easier to interpret, less sensitive to outliers.
+
+| Precision     | Metric | Value |
+| ------------- | ------ | ----- |
+| fp16          | cosine similarity | 1.00000 |
+| fp8 e4m3      | cosine similarity | 0.99965 |
+| fp8 e5m2      | cosine similarity | 0.99861 |
+| fp16          | MSE    | 0.00001 |
+| fp8 e4m3      | MSE    | 0.24369 |
+| fp8 e5m2      | MSE    | 0.96497 |
+| fp16          | MAE    | 0.00244 |
+| fp8 e4m3      | MAE    | 0.31206 |
+| fp8 e5m2      | MAE    | 0.62205 |
+
+## Tokenizer Examples
+
+**Input:** This is an example of encoding<br/>
+**Tokens**: `[CLS]` `this` `is` `an` `example` `of` `encoding` `[SEP]`
+
+**Input:** The quick brown fox jumps over the lazy dog.<br/>
+**Tokens**: `[CLS]` `the` `quick` `brown` `fox` `jumps` `over` `the` `lazy` `dog` `.` `[SEP]`
+
+**Input:** Curaçao, naïve fiancé, jalapeño, déjà vu.<br/>
+**Tokens**: `[CLS]` `cu` `##rac` `##ao` `,` `naive` `fiance` `,` `ja` `##la` `##pen` `##o` `,` `de` `##ja` `vu` `.` `[SEP]`
+
+**Input:** Привет, как дела?<br/>
+**Tokens**: `[CLS]` `п` `##р` `##и` `##в` `##е` `##т` `,` `к` `##а` `##к` `д` `##е` `##л` `##а` `?` `[SEP]`
+
+**Input:** Бързата кафява лисица прескача мързеливото куче.<br/>
+**Tokens**: `[CLS]` `б` `##ъ` `##р` `##з` `##а` `##т` `##а` `к` `##а` `##ф` `##я` `##в` `##а` `л` `##и` `##с` `##и` `##ц` `##а` `п` `##р` `##е` `##с` `##ка` `##ч` `##а` `м` `##ъ` `##р` `##з` `##е` `##л` `##и` `##в` `##о` `##т` `##о` `к` `##у` `##ч` `##е` `.` `[SEP]`
+
+**Input:** Γρήγορη καφέ αλεπού πηδάει πάνω από τον τεμπέλη σκύλο.<br/>
+**Tokens**: `[CLS]` `γ` `##ρ` `##η` `##γ` `##ο` `##ρ` `##η` `κ` `##α` `##φ` `##ε` `α` `##λ` `##ε` `##π` `##ου` `π` `##η` `##δ` `##α` `##ε` `##ι` `π` `##α` `##ν` `##ω` `α` `##π` `##ο` `τ` `##ο` `##ν` `τ` `##ε` `##μ` `##π` `##ε` `##λ` `##η` `σ` `##κ` `##υ` `##λ` `##ο` `.` `[SEP]`
+
+**Input:** اللغة العربية جميلة وغنية بالتاريخ.<br/>
+**Tokens**: `[CLS]` `ا` `##ل` `##ل` `##غ` `##ة` `ا` `##ل` `##ع` `##ر` `##ب` `##ي` `##ة` `ج` `##م` `##ي` `##ل` `##ة` `و` `##غ` `##ن` `##ي` `##ة` `ب` `##ا` `##ل` `##ت` `##ا` `##ر` `##ي` `##خ` `.` `[SEP]`
+
+**Input:** مرحبا بالعالم!<br/>
+**Tokens**: `[CLS]` `م` `##ر` `##ح` `##ب` `##ا` `ب` `##ا` `##ل` `##ع` `##ا` `##ل` `##م` `!` `[SEP]`
+
+**Input:** Simplified: 快速的棕色狐狸跳过懒狗。<br/>
+**Tokens**: `[CLS]` `simplified` `:` `[UNK]` `[UNK]` `的` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `。` `[SEP]`
+
+**Input:** Traditional: 快速的棕色狐狸跳過懶狗。<br/>
+**Tokens**: `[CLS]` `traditional` `:` `[UNK]` `[UNK]` `的` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `。` `[SEP]`
+
+**Input:** 素早い茶色の狐が怠け者の犬を飛び越える。<br/>
+**Tokens**: `[CLS]` `[UNK]` `[UNK]` `い` `[UNK]` `[UNK]` `の` `[UNK]` `か` `[UNK]` `け` `[UNK]` `の` `犬` `を` `[UNK]` `ひ` `[UNK]` `え` `##る` `。` `[SEP]`
+
+**Input:** コンピュータープログラミング<br/>
+**Tokens**: `[CLS]` `コ` `##ン` `##ヒ` `##ュ` `##ー` `##タ` `##ー` `##フ` `##ロ` `##ク` `##ラ` `##ミ` `##ン` `##ク` `[SEP]`
+
+**Input:** 빠른 갈색 여우가 게으른 개를 뛰어넘습니다.<br/>
+**Tokens**: `[CLS]` `[UNK]` `ᄀ` `##ᅡ` `##ᆯ` `##ᄉ` `##ᅢ` `##ᆨ` `ᄋ` `##ᅧ` `##ᄋ` `##ᅮ` `##ᄀ` `##ᅡ` `ᄀ` `##ᅦ` `##ᄋ` `##ᅳ` `##ᄅ` `##ᅳ` `##ᆫ` `ᄀ` `##ᅢ` `##ᄅ` `##ᅳ` `##ᆯ` `[UNK]` `.` `[SEP]`
+
+**Input:** तेज़ भूरी लोमड़ी आलसी कुत्ते के ऊपर कूदती है।<br/>
+**Tokens**: `[CLS]` `त` `##ज` `भ` `##र` `##ी` `ल` `##ो` `##म` `##ड` `##ी` `आ` `##ल` `##स` `##ी` `क` `##त` `##त` `क` `[UNK]` `क` `##द` `##त` `##ी` `ह` `।` `[SEP]`
+
+**Input:** দ্রুত বাদামী শিয়াল অলস কুকুরের উপর দিয়ে লাফ দেয়।<br/>
+**Tokens**: `[CLS]` `দ` `##র` `##ত` `ব` `##া` `##দ` `##া` `##ম` `##ী` `শ` `##ি` `##য` `##া` `##ল` `অ` `##ল` `##স` `ক` `##ক` `##র` `##ে` `##র` `উ` `##প` `##র` `দ` `##ি` `##য` `##ে` `[UNK]` `দ` `##ে` `##য` `।` `[SEP]`
+
+**Input:** வேகமான பழுப்பு நரி சோம்பேறி நாயின் மேல் குதிக்கிறது.<br/>
+**Tokens**: `[CLS]` `வ` `##ே` `##க` `##ம` `##ா` `##ன` `[UNK]` `ந` `##ர` `##ி` `[UNK]` `ந` `##ா` `##ய` `##ி` `##ன` `ம` `##ே` `##ல` `[UNK]` `.` `[SEP]`
+
+**Input:** สุนัขจิ้งจอกสีน้ำตาลกระโดดข้ามสุนัขขี้เกียจ.<br/>
+**Tokens**: `[CLS]` `[UNK]` `.` `[SEP]`
+
+**Input:** ብሩክ ቡናማ ቀበሮ ሰነፍ ውሻን ተዘልሏል።<br/>
+**Tokens**: `[CLS]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[SEP]`
+
+**Input:** Hello 世界 مرحبا 🌍<br/>
+**Tokens**: `[CLS]` `hello` `世` `[UNK]` `م` `##ر` `##ح` `##ب` `##ا` `[UNK]` `[SEP]`
+
+**Input:** 123, αβγ, абв, العربية, 中文, हिन्दी.<br/>
+**Tokens**: `[CLS]` `123` `,` `α` `##β` `##γ` `,` `а` `##б` `##в` `,` `ا` `##ل` `##ع` `##ر` `##ب` `##ي` `##ة` `,` `中` `文` `,` `ह` `##ि` `##न` `##द` `##ी` `.` `[SEP]`

models/sentence-transformers/static-similarity-mrl-multilingual-v1/README.md

@@ -0,0 +1,127 @@
+# [sentence-transformers/static-similarity-mrl-multilingual-v1](https://huggingface.co/sentence-transformers/static-similarity-mrl-multilingual-v1)
+
+License: [apache-2.0](https://choosealicense.com/licenses/apache-2.0/)
+
+Multi-lingual similarity embeddings that were trained with Matroyshka loss
+that allows for more effective truncation of the embedding vectors. It
+was trained on a variety of domains of multilingual datasets.
+
+It's a general purpose model that can be used for semantic textual similarity,
+paraphrase mining, text classification, clustering, and more
+
+## Model Stats
+
+Stats that describe the embeddings tensor shapes and value distribution.
+
+| item          | metric                  | value |
+| --------------| ----------------------- | ----- |
+| vocab         | size                    | 105,879 |
+| embedding     | dimensions              | 1,024 |
+| vector length | mean                    | 413.61 |
+| vector length | median                  | 437.74 |
+| vector length | stddev                  | 195.51 |
+| values        | mean                    | -0.02 |
+| values        | median                  | -0.01 |
+| values        | stddev                  | 14.30 |
+
+## Mean Pooled Quantization Loss
+
+This test roundtrips the vectors through quantization, but performs the
+mean pooling arithmetic in float32 space. The quantized and unquantized
+mean pooled vectors are compared to each other to determine their cosine
+similarity, to show how much the meaning of the vector has changed due
+to quantization.
+
+| Precision     | Cosine Similarity |
+| ------------- | ----------------- |
+| fp16          | 1.00000 |
+| fp8 e4m3      | 0.99980 |
+| fp8 e5m2      | 0.99921 |
+
+## Quantization Loss Per Vector
+
+While ultimately the embedding vectors will be mean pooled together, it's
+still useful to look at the loss per-vector in the embedding table to see
+which quantization strategies retain the most vector meaning.
+
+- **Cosine Similarity** — measures how well the *direction* of embedding vectors
+is preserved after quantization, independent of scale. This is especially
+relevant when embeddings are used for similarity search or retrieval.
+- **MSE (Mean Squared Error)** — emphasizes large errors by squaring the
+differences. Useful for detecting whether any values are badly distorted.
+- **MAE (Mean Absolute Error)** — the average absolute difference between
+original and quantized values. Easier to interpret, less sensitive to outliers.
+
+| Precision     | Metric | Value |
+| ------------- | ------ | ----- |
+| fp16          | cosine similarity | 1.00000 |
+| fp8 e4m3      | cosine similarity | 0.99965 |
+| fp8 e5m2      | cosine similarity | 0.99861 |
+| fp16          | MSE    | 0.00001 |
+| fp8 e4m3      | MSE    | 0.14369 |
+| fp8 e5m2      | MSE    | 0.56917 |
+| fp16          | MAE    | 0.00183 |
+| fp8 e4m3      | MAE    | 0.23372 |
+| fp8 e5m2      | MAE    | 0.46585 |
+
+## Tokenizer Examples
+
+**Input:** This is an example of encoding<br/>
+**Tokens**: `[CLS]` `this` `is` `an` `example` `of` `en` `##co` `##ding` `[SEP]`
+
+**Input:** The quick brown fox jumps over the lazy dog.<br/>
+**Tokens**: `[CLS]` `the` `quick` `brown` `fox` `jump` `##s` `over` `the` `la` `##zy` `dog` `.` `[SEP]`
+
+**Input:** Curaçao, naïve fiancé, jalapeño, déjà vu.<br/>
+**Tokens**: `[CLS]` `curacao` `,` `nai` `##ve` `fia` `##nce` `,` `ja` `##lap` `##eno` `,` `deja` `vu` `.` `[SEP]`
+
+**Input:** Привет, как дела?<br/>
+**Tokens**: `[CLS]` `при` `##вет` `,` `как` `дела` `?` `[SEP]`
+
+**Input:** Бързата кафява лисица прескача мързеливото куче.<br/>
+**Tokens**: `[CLS]` `б` `##ър` `##за` `##та` `ка` `##ф` `##ява` `ли` `##си` `##ца` `пре` `##ска` `##ча` `м` `##ър` `##зе` `##ливо` `##то` `к` `##уч` `##е` `.` `[SEP]`
+
+**Input:** Γρήγορη καφέ αλεπού πηδάει πάνω από τον τεμπέλη σκύλο.<br/>
+**Tokens**: `[CLS]` `γ` `##ρη` `##γο` `##ρη` `κ` `##α` `##φ` `##ε` `α` `##λε` `##που` `π` `##η` `##δα` `##ει` `πανω` `απο` `τον` `τ` `##ε` `##μ` `##πε` `##λη` `σ` `##κ` `##υλο` `.` `[SEP]`
+
+**Input:** اللغة العربية جميلة وغنية بالتاريخ.<br/>
+**Tokens**: `[CLS]` `اللغة` `العربية` `ج` `##ميل` `##ة` `و` `##غنية` `با` `##لت` `##اري` `##خ` `.` `[SEP]`
+
+**Input:** مرحبا بالعالم!<br/>
+**Tokens**: `[CLS]` `م` `##رح` `##با` `با` `##ل` `##عا` `##لم` `!` `[SEP]`
+
+**Input:** Simplified: 快速的棕色狐狸跳过懒狗。<br/>
+**Tokens**: `[CLS]` `simplified` `:` `快` `速` `的` `棕` `色` `狐` `狸` `跳` `过` `懒` `狗` `。` `[SEP]`
+
+**Input:** Traditional: 快速的棕色狐狸跳過懶狗。<br/>
+**Tokens**: `[CLS]` `traditional` `:` `快` `速` `的` `棕` `色` `狐` `狸` `跳` `過` `懶` `狗` `。` `[SEP]`
+
+**Input:** 素早い茶色の狐が怠け者の犬を飛び越える。<br/>
+**Tokens**: `[CLS]` `素` `早` `い` `茶` `色` `の` `狐` `か` `怠` `け` `者` `の` `犬` `を` `飛` `ひ` `越` `える` `。` `[SEP]`
+
+**Input:** コンピュータープログラミング<br/>
+**Tokens**: `[CLS]` `コ` `##ン` `##ヒ` `##ュー` `##ター` `##フロ` `##ク` `##ラ` `##ミ` `##ンク` `[SEP]`
+
+**Input:** 빠른 갈색 여우가 게으른 개를 뛰어넘습니다.<br/>
+**Tokens**: `[CLS]` `ᄈ` `##ᅡ른` `가` `##ᆯ` `##색` `ᄋ` `##ᅧ` `##우` `##가` `ᄀ` `##ᅦ` `##ᄋ` `##ᅳ` `##른` `ᄀ` `##ᅢ를` `ᄄ` `##ᅱ` `##어` `##너` `##ᆷ` `##스` `##ᆸ니다` `.` `[SEP]`
+
+**Input:** तेज़ भूरी लोमड़ी आलसी कुत्ते के ऊपर कूदती है।<br/>
+**Tokens**: `[CLS]` `त` `##ज` `भर` `##ी` `ल` `##ो` `##म` `##डी` `आल` `##सी` `क` `##तत` `क` `ऊपर` `क` `##द` `##ती` `ह` `।` `[SEP]`
+
+**Input:** দ্রুত বাদামী শিয়াল অলস কুকুরের উপর দিয়ে লাফ দেয়।<br/>
+**Tokens**: `[CLS]` `দ` `##রত` `বা` `##দা` `##মী` `শ` `##িযা` `##ল` `অ` `##ল` `##স` `ক` `##কর` `##ের` `উপর` `দিযে` `ল` `##া` `##ফ` `দেয` `।` `[SEP]`
+
+**Input:** வேகமான பழுப்பு நரி சோம்பேறி நாயின் மேல் குதிக்கிறது.<br/>
+**Tokens**: `[CLS]` `வ` `##ே` `##கம` `##ான` `ப` `##ழு` `##பபு` `நர` `##ி` `ச` `##ோ` `##ம` `##ப` `##ே` `##றி` `ந` `##ாய` `##ின` `மேல` `க` `##ு` `##தி` `##ககிறது` `.` `[SEP]`
+
+**Input:** สุนัขจิ้งจอกสีน้ำตาลกระโดดข้ามสุนัขขี้เกียจ.<br/>
+**Tokens**: `[CLS]` `[UNK]` `.` `[SEP]`
+
+**Input:** ብሩክ ቡናማ ቀበሮ ሰነፍ ውሻን ተዘልሏል።<br/>
+**Tokens**: `[CLS]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[UNK]` `[SEP]`
+
+**Input:** Hello 世界 مرحبا 🌍<br/>
+**Tokens**: `[CLS]` `hello` `世` `界` `م` `##رح` `##با` `[UNK]` `[SEP]`
+
+**Input:** 123, αβγ, абв, العربية, 中文, हिन्दी.<br/>
+**Tokens**: `[CLS]` `123` `,` `α` `##β` `##γ` `,` `аб` `##в` `,` `العربية` `,` `中` `文` `,` `हिनदी` `.` `[SEP]`

multilingual.py

@@ -0,0 +1,59 @@
+from sentence_transformers import SentenceTransformer
+from tokenizers import Encoding, Tokenizer
+from torch.nn import EmbeddingBag
+import torch
+
+
+def test_tokenizer():
+    examples = [
+        "This is an example of encoding",
+        "The quick brown fox jumps over the lazy dog.",
+        "Curaçao, naïve fiancé, jalapeño, déjà vu.",
+        "Привет, как дела?",
+        "Бързата кафява лисица прескача мързеливото куче.",
+        "Γρήγορη καφέ αλεπού πηδάει πάνω από τον τεμπέλη σκύλο.",
+        "اللغة العربية جميلة وغنية بالتاريخ.",
+        "مرحبا بالعالم!",
+        "Simplified: 快速的棕色狐狸跳过懒狗。",
+        "Traditional: 快速的棕色狐狸跳過懶狗。",
+        "素早い茶色の狐が怠け者の犬を飛び越える。",
+        "コンピュータープログラミング",
+        "빠른 갈색 여우가 게으른 개를 뛰어넘습니다.",
+        "तेज़ भूरी लोमड़ी आलसी कुत्ते के ऊपर कूदती है।",
+        "দ্রুত বাদামী শিয়াল অলস কুকুরের উপর দিয়ে লাফ দেয়।",
+        "வேகமான பழுப்பு நரி சோம்பேறி நாயின் மேல் குதிக்கிறது.",
+        "สุนัขจิ้งจอกสีน้ำตาลกระโดดข้ามสุนัขขี้เกียจ.",
+        "ብሩክ ቡናማ ቀበሮ ሰነፍ ውሻን ተዘልሏል።",
+        "Hello 世界 مرحبا 🌍",
+        "123, αβγ, абв, العربية, 中文, हिन्दी.",
+    ]
+
+    tokenizer: Tokenizer = Tokenizer.from_file("js/tokenizer.json")
+
+    for example in examples:
+        encoding: Encoding = tokenizer.encode(example)
+        print(example)
+        print(encoding.tokens)
+        print()
+
+
+# https://huggingface.co/sentence-transformers/static-similarity-mrl-multilingual-v1
+model = SentenceTransformer(
+    "sentence-transformers/static-similarity-mrl-multilingual-v1", device="cpu"
+)
+
+embedding_bag: EmbeddingBag = model[0].embedding  # type: ignore
+embeddings = torch.Tensor(embedding_bag.weight)
+
+print(embeddings.shape)
+assert embeddings.shape == torch.Size([105879, 1024])
+
+print("float32")
+print(f"  1024 dim - {embeddings.shape[0] * 1024 * 4 / 1024 / 1024:,.1f} MiB")
+print(f"   512 dim - {embeddings.shape[0] * 512 * 4 / 1024 / 1024:,.1f} MiB")
+print(f"   256 dim - {embeddings.shape[0] * 256 * 4 / 1024 / 1024:,.1f} MiB")
+
+print("float16")
+print(f"  1024 dim - {embeddings.shape[0] * 1024 * 2 / 1024 / 1024:,.1f} MiB")
+print(f"   512 dim - {embeddings.shape[0] * 512 * 2 / 1024 / 1024:,.1f} MiB")
+print(f"   256 dim - {embeddings.shape[0] * 256 * 2 / 1024 / 1024:,.1f} MiB")

package-lock.json

@@ -0,0 +1,1067 @@
+{
+  "name": "js",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "js",
+      "version": "1.0.0",
+      "license": "ISC",
+      "dependencies": {
+        "@huggingface/transformers": "^3.7.2"
+      }
+    },
+    "node_modules/@emnapi/runtime": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.5.0.tgz",
+      "integrity": "sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==",
+      "license": "MIT",
+      "optional": true,
+      "dependencies": {
+        "tslib": "^2.4.0"
+      }
+    },
+    "node_modules/@huggingface/jinja": {
+      "version": "0.5.1",
+      "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.1.tgz",
+      "integrity": "sha512-yUZLld4lrM9iFxHCwFQ7D1HW2MWMwSbeB7WzWqFYDWK+rEb+WldkLdAJxUPOmgICMHZLzZGVcVjFh3w/YGubng==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@huggingface/transformers": {
+      "version": "3.7.2",
+      "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.7.2.tgz",
+      "integrity": "sha512-6SOxo6XziupnQ5Vs5vbbs74CNB6ViHLHGQJjY6zj88JeiDtJ2d/ADKxaay688Sf2KcjtdF3dyBL11C5pJS2NxQ==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@huggingface/jinja": "^0.5.1",
+        "onnxruntime-node": "1.21.0",
+        "onnxruntime-web": "1.22.0-dev.20250409-89f8206ba4",
+        "sharp": "^0.34.1"
+      }
+    },
+    "node_modules/@img/sharp-darwin-arm64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.3.tgz",
+      "integrity": "sha512-ryFMfvxxpQRsgZJqBd4wsttYQbCxsJksrv9Lw/v798JcQ8+w84mBWuXwl+TT0WJ/WrYOLaYpwQXi3sA9nTIaIg==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-darwin-arm64": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-darwin-x64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.3.tgz",
+      "integrity": "sha512-yHpJYynROAj12TA6qil58hmPmAwxKKC7reUqtGLzsOHfP7/rniNGTL8tjWX6L3CTV4+5P4ypcS7Pp+7OB+8ihA==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-darwin-x64": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-libvips-darwin-arm64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.0.tgz",
+      "integrity": "sha512-sBZmpwmxqwlqG9ueWFXtockhsxefaV6O84BMOrhtg/YqbTaRdqDE7hxraVE3y6gVM4eExmfzW4a8el9ArLeEiQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-darwin-x64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.0.tgz",
+      "integrity": "sha512-M64XVuL94OgiNHa5/m2YvEQI5q2cl9d/wk0qFTDVXcYzi43lxuiFTftMR1tOnFQovVXNZJ5TURSDK2pNe9Yzqg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linux-arm": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.0.tgz",
+      "integrity": "sha512-mWd2uWvDtL/nvIzThLq3fr2nnGfyr/XMXlq8ZJ9WMR6PXijHlC3ksp0IpuhK6bougvQrchUAfzRLnbsen0Cqvw==",
+      "cpu": [
+        "arm"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linux-arm64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.0.tgz",
+      "integrity": "sha512-RXwd0CgG+uPRX5YYrkzKyalt2OJYRiJQ8ED/fi1tq9WQW2jsQIn0tqrlR5l5dr/rjqq6AHAxURhj2DVjyQWSOA==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linux-ppc64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.0.tgz",
+      "integrity": "sha512-Xod/7KaDDHkYu2phxxfeEPXfVXFKx70EAFZ0qyUdOjCcxbjqyJOEUpDe6RIyaunGxT34Anf9ue/wuWOqBW2WcQ==",
+      "cpu": [
+        "ppc64"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linux-s390x": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.0.tgz",
+      "integrity": "sha512-eMKfzDxLGT8mnmPJTNMcjfO33fLiTDsrMlUVcp6b96ETbnJmd4uvZxVJSKPQfS+odwfVaGifhsB07J1LynFehw==",
+      "cpu": [
+        "s390x"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linux-x64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.0.tgz",
+      "integrity": "sha512-ZW3FPWIc7K1sH9E3nxIGB3y3dZkpJlMnkk7z5tu1nSkBoCgw2nSRTFHI5pB/3CQaJM0pdzMF3paf9ckKMSE9Tg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linuxmusl-arm64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.0.tgz",
+      "integrity": "sha512-UG+LqQJbf5VJ8NWJ5Z3tdIe/HXjuIdo4JeVNADXBFuG7z9zjoegpzzGIyV5zQKi4zaJjnAd2+g2nna8TZvuW9Q==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linuxmusl-x64": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.0.tgz",
+      "integrity": "sha512-SRYOLR7CXPgNze8akZwjoGBoN1ThNZoqpOgfnOxmWsklTGVfJiGJoC/Lod7aNMGA1jSsKWM1+HRX43OP6p9+6Q==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-linux-arm": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.3.tgz",
+      "integrity": "sha512-oBK9l+h6KBN0i3dC8rYntLiVfW8D8wH+NPNT3O/WBHeW0OQWCjfWksLUaPidsrDKpJgXp3G3/hkmhptAW0I3+A==",
+      "cpu": [
+        "arm"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linux-arm": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-linux-arm64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.3.tgz",
+      "integrity": "sha512-QdrKe3EvQrqwkDrtuTIjI0bu6YEJHTgEeqdzI3uWJOH6G1O8Nl1iEeVYRGdj1h5I21CqxSvQp1Yv7xeU3ZewbA==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linux-arm64": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-linux-ppc64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.3.tgz",
+      "integrity": "sha512-GLtbLQMCNC5nxuImPR2+RgrviwKwVql28FWZIW1zWruy6zLgA5/x2ZXk3mxj58X/tszVF69KK0Is83V8YgWhLA==",
+      "cpu": [
+        "ppc64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linux-ppc64": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-linux-s390x": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.3.tgz",
+      "integrity": "sha512-3gahT+A6c4cdc2edhsLHmIOXMb17ltffJlxR0aC2VPZfwKoTGZec6u5GrFgdR7ciJSsHT27BD3TIuGcuRT0KmQ==",
+      "cpu": [
+        "s390x"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linux-s390x": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-linux-x64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.3.tgz",
+      "integrity": "sha512-8kYso8d806ypnSq3/Ly0QEw90V5ZoHh10yH0HnrzOCr6DKAPI6QVHvwleqMkVQ0m+fc7EH8ah0BB0QPuWY6zJQ==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linux-x64": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-linuxmusl-arm64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.3.tgz",
+      "integrity": "sha512-vAjbHDlr4izEiXM1OTggpCcPg9tn4YriK5vAjowJsHwdBIdx0fYRsURkxLG2RLm9gyBq66gwtWI8Gx0/ov+JKQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linuxmusl-arm64": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-linuxmusl-x64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.3.tgz",
+      "integrity": "sha512-gCWUn9547K5bwvOn9l5XGAEjVTTRji4aPTqLzGXHvIr6bIDZKNTA34seMPgM0WmSf+RYBH411VavCejp3PkOeQ==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linuxmusl-x64": "1.2.0"
+      }
+    },
+    "node_modules/@img/sharp-wasm32": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.3.tgz",
+      "integrity": "sha512-+CyRcpagHMGteySaWos8IbnXcHgfDn7pO2fiC2slJxvNq9gDipYBN42/RagzctVRKgxATmfqOSulgZv5e1RdMg==",
+      "cpu": [
+        "wasm32"
+      ],
+      "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT",
+      "optional": true,
+      "dependencies": {
+        "@emnapi/runtime": "^1.4.4"
+      },
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-win32-arm64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.3.tgz",
+      "integrity": "sha512-MjnHPnbqMXNC2UgeLJtX4XqoVHHlZNd+nPt1kRPmj63wURegwBhZlApELdtxM2OIZDRv/DFtLcNhVbd1z8GYXQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0 AND LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-win32-ia32": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.3.tgz",
+      "integrity": "sha512-xuCdhH44WxuXgOM714hn4amodJMZl3OEvf0GVTm0BEyMeA2to+8HEdRPShH0SLYptJY1uBw+SCFP9WVQi1Q/cw==",
+      "cpu": [
+        "ia32"
+      ],
+      "license": "Apache-2.0 AND LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-win32-x64": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.3.tgz",
+      "integrity": "sha512-OWwz05d++TxzLEv4VnsTz5CmZ6mI6S05sfQGEMrNrQcOEERbX46332IvE7pO/EUiw7jUrrS40z/M7kPyjfl04g==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0 AND LGPL-3.0-or-later",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@isaacs/fs-minipass": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz",
+      "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==",
+      "license": "ISC",
+      "dependencies": {
+        "minipass": "^7.0.4"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      }
+    },
+    "node_modules/@protobufjs/aspromise": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
+      "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/base64": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
+      "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/codegen": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
+      "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/eventemitter": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
+      "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/fetch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
+      "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "@protobufjs/aspromise": "^1.1.1",
+        "@protobufjs/inquire": "^1.1.0"
+      }
+    },
+    "node_modules/@protobufjs/float": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
+      "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/inquire": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
+      "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/path": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
+      "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/pool": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
+      "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/utf8": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
+      "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@types/node": {
+      "version": "24.3.1",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.3.1.tgz",
+      "integrity": "sha512-3vXmQDXy+woz+gnrTvuvNrPzekOi+Ds0ReMxw0LzBiK3a+1k0kQn9f2NWk+lgD4rJehFUmYy2gMhJ2ZI+7YP9g==",
+      "license": "MIT",
+      "dependencies": {
+        "undici-types": "~7.10.0"
+      }
+    },
+    "node_modules/boolean": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz",
+      "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==",
+      "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
+      "license": "MIT"
+    },
+    "node_modules/chownr": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
+      "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==",
+      "license": "BlueOak-1.0.0",
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/color": {
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
+      "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==",
+      "license": "MIT",
+      "dependencies": {
+        "color-convert": "^2.0.1",
+        "color-string": "^1.9.0"
+      },
+      "engines": {
+        "node": ">=12.5.0"
+      }
+    },
+    "node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "license": "MIT",
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "license": "MIT"
+    },
+    "node_modules/color-string": {
+      "version": "1.9.1",
+      "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz",
+      "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==",
+      "license": "MIT",
+      "dependencies": {
+        "color-name": "^1.0.0",
+        "simple-swizzle": "^0.2.2"
+      }
+    },
+    "node_modules/define-data-property": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz",
+      "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==",
+      "license": "MIT",
+      "dependencies": {
+        "es-define-property": "^1.0.0",
+        "es-errors": "^1.3.0",
+        "gopd": "^1.0.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/define-properties": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz",
+      "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==",
+      "license": "MIT",
+      "dependencies": {
+        "define-data-property": "^1.0.1",
+        "has-property-descriptors": "^1.0.0",
+        "object-keys": "^1.1.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/detect-libc": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz",
+      "integrity": "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/detect-node": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz",
+      "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==",
+      "license": "MIT"
+    },
+    "node_modules/es-define-property": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
+      "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-errors": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
+      "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es6-error": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz",
+      "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==",
+      "license": "MIT"
+    },
+    "node_modules/escape-string-regexp": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
+      "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/flatbuffers": {
+      "version": "25.2.10",
+      "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.2.10.tgz",
+      "integrity": "sha512-7JlN9ZvLDG1McO3kbX0k4v+SUAg48L1rIwEvN6ZQl/eCtgJz9UylTMzE9wrmYrcorgxm3CX/3T/w5VAub99UUw==",
+      "license": "Apache-2.0"
+    },
+    "node_modules/global-agent": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz",
+      "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "boolean": "^3.0.1",
+        "es6-error": "^4.1.1",
+        "matcher": "^3.0.0",
+        "roarr": "^2.15.3",
+        "semver": "^7.3.2",
+        "serialize-error": "^7.0.1"
+      },
+      "engines": {
+        "node": ">=10.0"
+      }
+    },
+    "node_modules/globalthis": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz",
+      "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==",
+      "license": "MIT",
+      "dependencies": {
+        "define-properties": "^1.2.1",
+        "gopd": "^1.0.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/gopd": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
+      "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/guid-typescript": {
+      "version": "1.0.9",
+      "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz",
+      "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==",
+      "license": "ISC"
+    },
+    "node_modules/has-property-descriptors": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz",
+      "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==",
+      "license": "MIT",
+      "dependencies": {
+        "es-define-property": "^1.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/is-arrayish": {
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz",
+      "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==",
+      "license": "MIT"
+    },
+    "node_modules/json-stringify-safe": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
+      "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==",
+      "license": "ISC"
+    },
+    "node_modules/long": {
+      "version": "5.3.2",
+      "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz",
+      "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==",
+      "license": "Apache-2.0"
+    },
+    "node_modules/matcher": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz",
+      "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==",
+      "license": "MIT",
+      "dependencies": {
+        "escape-string-regexp": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/minipass": {
+      "version": "7.1.2",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
+      "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      }
+    },
+    "node_modules/minizlib": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.2.tgz",
+      "integrity": "sha512-oG62iEk+CYt5Xj2YqI5Xi9xWUeZhDI8jjQmC5oThVH5JGCTgIjr7ciJDzC7MBzYd//WvR1OTmP5Q38Q8ShQtVA==",
+      "license": "MIT",
+      "dependencies": {
+        "minipass": "^7.1.2"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/mkdirp": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz",
+      "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==",
+      "license": "MIT",
+      "bin": {
+        "mkdirp": "dist/cjs/src/bin.js"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/object-keys": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
+      "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/onnxruntime-common": {
+      "version": "1.21.0",
+      "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.21.0.tgz",
+      "integrity": "sha512-Q632iLLrtCAVOTO65dh2+mNbQir/QNTVBG3h/QdZBpns7mZ0RYbLRBgGABPbpU9351AgYy7SJf1WaeVwMrBFPQ==",
+      "license": "MIT"
+    },
+    "node_modules/onnxruntime-node": {
+      "version": "1.21.0",
+      "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.21.0.tgz",
+      "integrity": "sha512-NeaCX6WW2L8cRCSqy3bInlo5ojjQqu2fD3D+9W5qb5irwxhEyWKXeH2vZ8W9r6VxaMPUan+4/7NDwZMtouZxEw==",
+      "hasInstallScript": true,
+      "license": "MIT",
+      "os": [
+        "win32",
+        "darwin",
+        "linux"
+      ],
+      "dependencies": {
+        "global-agent": "^3.0.0",
+        "onnxruntime-common": "1.21.0",
+        "tar": "^7.0.1"
+      }
+    },
+    "node_modules/onnxruntime-web": {
+      "version": "1.22.0-dev.20250409-89f8206ba4",
+      "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.22.0-dev.20250409-89f8206ba4.tgz",
+      "integrity": "sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ==",
+      "license": "MIT",
+      "dependencies": {
+        "flatbuffers": "^25.1.24",
+        "guid-typescript": "^1.0.9",
+        "long": "^5.2.3",
+        "onnxruntime-common": "1.22.0-dev.20250409-89f8206ba4",
+        "platform": "^1.3.6",
+        "protobufjs": "^7.2.4"
+      }
+    },
+    "node_modules/onnxruntime-web/node_modules/onnxruntime-common": {
+      "version": "1.22.0-dev.20250409-89f8206ba4",
+      "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.22.0-dev.20250409-89f8206ba4.tgz",
+      "integrity": "sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ==",
+      "license": "MIT"
+    },
+    "node_modules/platform": {
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz",
+      "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==",
+      "license": "MIT"
+    },
+    "node_modules/protobufjs": {
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz",
+      "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==",
+      "hasInstallScript": true,
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "@protobufjs/aspromise": "^1.1.2",
+        "@protobufjs/base64": "^1.1.2",
+        "@protobufjs/codegen": "^2.0.4",
+        "@protobufjs/eventemitter": "^1.1.0",
+        "@protobufjs/fetch": "^1.1.0",
+        "@protobufjs/float": "^1.0.2",
+        "@protobufjs/inquire": "^1.1.0",
+        "@protobufjs/path": "^1.1.2",
+        "@protobufjs/pool": "^1.1.0",
+        "@protobufjs/utf8": "^1.1.0",
+        "@types/node": ">=13.7.0",
+        "long": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
+    "node_modules/roarr": {
+      "version": "2.15.4",
+      "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz",
+      "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "boolean": "^3.0.1",
+        "detect-node": "^2.0.4",
+        "globalthis": "^1.0.1",
+        "json-stringify-safe": "^5.0.1",
+        "semver-compare": "^1.0.0",
+        "sprintf-js": "^1.1.2"
+      },
+      "engines": {
+        "node": ">=8.0"
+      }
+    },
+    "node_modules/semver": {
+      "version": "7.7.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz",
+      "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==",
+      "license": "ISC",
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/semver-compare": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz",
+      "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==",
+      "license": "MIT"
+    },
+    "node_modules/serialize-error": {
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz",
+      "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==",
+      "license": "MIT",
+      "dependencies": {
+        "type-fest": "^0.13.1"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/sharp": {
+      "version": "0.34.3",
+      "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.3.tgz",
+      "integrity": "sha512-eX2IQ6nFohW4DbvHIOLRB3MHFpYqaqvXd3Tp5e/T/dSH83fxaNJQRvDMhASmkNTsNTVF2/OOopzRCt7xokgPfg==",
+      "hasInstallScript": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "color": "^4.2.3",
+        "detect-libc": "^2.0.4",
+        "semver": "^7.7.2"
+      },
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-darwin-arm64": "0.34.3",
+        "@img/sharp-darwin-x64": "0.34.3",
+        "@img/sharp-libvips-darwin-arm64": "1.2.0",
+        "@img/sharp-libvips-darwin-x64": "1.2.0",
+        "@img/sharp-libvips-linux-arm": "1.2.0",
+        "@img/sharp-libvips-linux-arm64": "1.2.0",
+        "@img/sharp-libvips-linux-ppc64": "1.2.0",
+        "@img/sharp-libvips-linux-s390x": "1.2.0",
+        "@img/sharp-libvips-linux-x64": "1.2.0",
+        "@img/sharp-libvips-linuxmusl-arm64": "1.2.0",
+        "@img/sharp-libvips-linuxmusl-x64": "1.2.0",
+        "@img/sharp-linux-arm": "0.34.3",
+        "@img/sharp-linux-arm64": "0.34.3",
+        "@img/sharp-linux-ppc64": "0.34.3",
+        "@img/sharp-linux-s390x": "0.34.3",
+        "@img/sharp-linux-x64": "0.34.3",
+        "@img/sharp-linuxmusl-arm64": "0.34.3",
+        "@img/sharp-linuxmusl-x64": "0.34.3",
+        "@img/sharp-wasm32": "0.34.3",
+        "@img/sharp-win32-arm64": "0.34.3",
+        "@img/sharp-win32-ia32": "0.34.3",
+        "@img/sharp-win32-x64": "0.34.3"
+      }
+    },
+    "node_modules/simple-swizzle": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz",
+      "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==",
+      "license": "MIT",
+      "dependencies": {
+        "is-arrayish": "^0.3.1"
+      }
+    },
+    "node_modules/sprintf-js": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz",
+      "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/tar": {
+      "version": "7.4.3",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz",
+      "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==",
+      "license": "ISC",
+      "dependencies": {
+        "@isaacs/fs-minipass": "^4.0.0",
+        "chownr": "^3.0.0",
+        "minipass": "^7.1.2",
+        "minizlib": "^3.0.1",
+        "mkdirp": "^3.0.1",
+        "yallist": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/tslib": {
+      "version": "2.8.1",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+      "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
+      "license": "0BSD",
+      "optional": true
+    },
+    "node_modules/type-fest": {
+      "version": "0.13.1",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz",
+      "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==",
+      "license": "(MIT OR CC0-1.0)",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/undici-types": {
+      "version": "7.10.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.10.0.tgz",
+      "integrity": "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag==",
+      "license": "MIT"
+    },
+    "node_modules/yallist": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz",
+      "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==",
+      "license": "BlueOak-1.0.0",
+      "engines": {
+        "node": ">=18"
+      }
+    }
+  }
+}

package.json

@@ -0,0 +1,15 @@
+{
+  "name": "js",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@huggingface/transformers": "^3.7.2"
+  }
+}

potion.py

@@ -0,0 +1,14 @@
+from model2vec import StaticModel
+from tokenizers import Tokenizer
+import torch
+
+model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M")
+embeddings = torch.from_numpy(model.embedding)
+
+print("Embedding shape:", embeddings.shape)
+bytes = embeddings.shape[0] * embeddings.shape[1] * 4
+
+print("MiB:", bytes / 1024 / 1024)
+
+tokenizer: Tokenizer = model.tokenizer
+print(tokenizer.to_str())

pyproject.toml

@@ -0,0 +1,12 @@
+[project]
+name = "sentence-embeddings"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "model2vec>=0.6.0",
+    "numpy>=2.3.2",
+    "sentence-transformers>=5.1.0",
+    "zstandard>=0.24.0",
+]

scripts/build_models.py

@@ -0,0 +1,411 @@
+from dataclasses import dataclass
+import shutil
+from textwrap import dedent, indent
+from typing import Any
+import numpy as np
+from zstandard import ZstdCompressor
+from pathlib import Path
+import io
+from sentence_transformers import SentenceTransformer
+from torch.nn import EmbeddingBag
+import torch
+from model2vec import StaticModel
+from tokenizers import Encoding, Tokenizer
+
+models_path = Path("models")
+
+
+@dataclass
+class ModelCard:
+    owner: str
+    repo: str
+    # The dimensions that were applied with Matroyshka Loss.
+    matroyshka_dims: list[int]
+    description: str
+    license: str
+
+    def name(self):
+        return f"{self.owner}/{self.repo}"
+
+    def path(self):
+        return models_path / self.owner / self.repo
+
+    def get_description(self):
+        return dedent(self.description).strip()
+
+
+def zst_compress_file(input: Path):
+    cctx = ZstdCompressor()
+    output = input.parent / f"{input.name}.zst"
+    print(f"Compressing {output}")
+    with open(input, "rb") as fin, open(output, "wb") as fout:
+        cctx.copy_stream(fin, fout)
+
+
+def save_data(path: Path, tensor: torch.Tensor):
+    """Writes out the static embeddings to a .npy and .npy.zst file"""
+    buffer = io.BytesIO()
+
+    if tensor.dtype in (torch.float8_e4m3fn, torch.float8_e5m2):
+        # Store as the raw bytes.
+        np.save(buffer, tensor.detach().view(torch.uint8).numpy())
+    else:
+        np.save(buffer, tensor.detach().numpy())
+
+    print(f"Saving {path}")
+    with (open(path, "wb") as outfile,):
+        outfile.write(buffer.getvalue())
+
+    zst_compress_file(path)
+
+
+def quantization_loss_mse(tensor: torch.Tensor, dtype: torch.dtype):
+    """
+    Compute reconstruction loss when converting embeddings to a datatype and back using
+    the mean squared error, which punishes big errors more than small ones.
+    """
+
+    # Original → quantize → dequantize
+    roundtrip = tensor.detach().to(dtype).to(tensor.dtype)
+
+    # Mean squared error
+    return torch.mean((tensor - roundtrip) ** 2).item()
+
+
+def quantization_loss_mae(tensor: torch.Tensor, dtype: torch.dtype):
+    """
+    Compute reconstruction loss when converting embeddings to a datatype and back using
+    the mean absolute error, which is less sensitive to outliers than MSE.
+    """
+
+    # Original → quantize → dequantize
+    roundtrip = tensor.detach().to(dtype).to(tensor.dtype)
+
+    # Mean absolute error
+    return torch.mean(torch.abs(tensor - roundtrip)).item()
+
+
+def quantization_loss_cosine(tensor: torch.Tensor, dtype: torch.dtype):
+    """
+    Compute reconstruction loss when converting embeddings to a datatype and back using
+    cosine similarity. This measures whether the embedding directions are preserved
+    after quantization, independent of their magnitudes.
+    """
+
+    # Original → quantize → dequantize
+    roundtrip = tensor.detach().to(dtype).to(tensor.dtype)
+
+    # Flatten both to 2D (num_vectors, dimensions) in case tensor is 1D or higher-D
+    if tensor.ndim == 1:
+        orig = tensor.unsqueeze(0)
+        recon = roundtrip.unsqueeze(0)
+    else:
+        orig = tensor.view(tensor.shape[0], -1)
+        recon = roundtrip.view(roundtrip.shape[0], -1)
+
+    # Cosine similarity per vector, then average
+    cos = torch.nn.functional.cosine_similarity(orig, recon, dim=1)
+    return cos.mean().item()
+
+
+def export_embeddings(model_card: ModelCard, embeddings: torch.Tensor) -> None:
+    vocab_size, dimensions = embeddings.shape
+
+    # This logic can always be adjusted for models with different shapes.
+    assert (
+        embeddings.dtype == torch.float32
+    ), f"The embeddings {embeddings.dtype} are assumed to be float32."
+
+    for dim in model_card.matroyshka_dims:
+        assert (
+            dim <= dimensions
+        ), f"The Matroyshka dimensions {dim} were bigger than the models dimensions of {dimensions}"
+
+        truncated = embeddings[:, :dim]
+        assert truncated.shape == torch.Size([vocab_size, dim])
+
+        save_data(model_card.path() / f"fp32.d{dim}.npy", truncated)
+        save_data(
+            model_card.path() / f"fp16.d{dim}.npy",
+            truncated.to(dtype=torch.float16),
+        )
+        save_data(
+            model_card.path() / f"fp8_e5m2.d{dim}.npy",
+            truncated.to(dtype=torch.float8_e5m2),
+        )
+        save_data(
+            model_card.path() / f"fp8_e4m3.d{dim}.npy",
+            truncated.to(dtype=torch.float8_e4m3fn),
+        )
+
+
+def normalized_mean_pooling(x: torch.Tensor) -> torch.Tensor:
+    pooled = x.mean(dim=0)
+    normalized = torch.nn.functional.normalize(pooled, dim=0)
+    return normalized
+
+
+def export_readme(
+    model_card: ModelCard,
+    embeddings: torch.Tensor,
+    tokenizer: Tokenizer,
+):
+    vocab_size, dimensions = embeddings.shape
+    norms = torch.norm(embeddings, dim=1)  # shape: [vocab_size]
+
+    phrases = [
+        "The committee approved the proposal after hours of heated discussion and several last-minute amendments."
+        "When training large neural networks, careful tuning of hyperparameters can significantly affect performance and stability."
+        "Despite the heavy rain, the concert continued as planned and the crowd stayed enthusiastic until the final encore."
+        "In ancient mythology, heroes often embarked on perilous journeys to discover hidden truths about themselves and their world."
+        "The new smartphone model features an improved camera system, faster processing, and extended battery life compared to its predecessor."
+        "He tried to explain the concept using simple analogies, but the underlying mathematics remained difficult to grasp for most listeners."
+        "After weeks of negotiations, the two countries signed a historic trade agreement aimed at reducing tariffs and boosting cooperation."
+        "She paused for a moment before answering, choosing her words carefully to avoid misunderstanding in such a delicate situation."
+        "The detective pieced together the timeline of events, realizing that the key witness had provided a contradictory statement."
+        "Remote work has changed the way teams collaborate, with online tools replacing traditional office routines and in-person meetings."
+    ]
+
+    cosine_similarity = {
+        torch.float16: [],
+        torch.float8_e4m3fn: [],
+        torch.float8_e5m2: [],
+    }
+
+    for phrase in phrases:
+        encoding: Encoding = tokenizer.encode(phrase)
+        embedded_phrase = embeddings[torch.tensor(encoding.ids, dtype=torch.long)]
+
+        for dtype in cosine_similarity.keys():
+            pooling_unquantized = normalized_mean_pooling(embedded_phrase)
+            pooling_roundtrip = normalized_mean_pooling(
+                embedded_phrase.to(dtype).to(torch.float32)
+            )
+            cosine = torch.dot(pooling_unquantized, pooling_roundtrip).item()
+            cosine_similarity[dtype].append(cosine)
+
+    avg_cosine_similarity = {
+        dtype: sum(values) / len(values) for dtype, values in cosine_similarity.items()
+    }
+
+    tokenizer_examples = []
+    for text in [
+        "This is an example of encoding",
+        "The quick brown fox jumps over the lazy dog.",
+        "Curaçao, naïve fiancé, jalapeño, déjà vu.",
+        "Привет, как дела?",
+        "Бързата кафява лисица прескача мързеливото куче.",
+        "Γρήγορη καφέ αλεπού πηδάει πάνω από τον τεμπέλη σκύλο.",
+        "اللغة العربية جميلة وغنية بالتاريخ.",
+        "مرحبا بالعالم!",
+        "Simplified: 快速的棕色狐狸跳过懒狗。",
+        "Traditional: 快速的棕色狐狸跳過懶狗。",
+        "素早い茶色の狐が怠け者の犬を飛び越える。",
+        "コンピュータープログラミング",
+        "빠른 갈색 여우가 게으른 개를 뛰어넘습니다.",
+        "तेज़ भूरी लोमड़ी आलसी कुत्ते के ऊपर कूदती है।",
+        "দ্রুত বাদামী শিয়াল অলস কুকুরের উপর দিয়ে লাফ দেয়।",
+        "வேகமான பழுப்பு நரி சோம்பேறி நாயின் மேல் குதிக்கிறது.",
+        "สุนัขจิ้งจอกสีน้ำตาลกระโดดข้ามสุนัขขี้เกียจ.",
+        "ብሩክ ቡናማ ቀበሮ ሰነፍ ውሻን ተዘልሏል።",
+        "Hello 世界 مرحبا 🌍",
+        "123, αβγ, абв, العربية, 中文, हिन्दी.",
+    ]:
+        encoding = tokenizer.encode(text)
+        tokens = [f"`{token}`" for token in encoding.tokens]
+
+        tokenizer_examples.append(f"**Input:** {text}<br/>")
+        tokenizer_examples.append(f"**Tokens**: {' '.join(tokens)}")
+        tokenizer_examples.append("")
+
+    tokenizer_output = "\n".join(tokenizer_examples)
+
+    with (model_card.path() / "README.md").open("wt") as file:
+        prefix = "                "
+
+        file.write(
+            dedent(
+                f"""
+                # [{model_card.name()}](https://huggingface.co/{model_card.name()})
+                
+                License: [{model_card.license}](https://choosealicense.com/licenses/{model_card.license}/)
+                
+                {indent(model_card.get_description(), prefix).strip()}
+                
+                ## Model Stats
+                
+                Stats that describe the embeddings tensor shapes and value distribution.
+
+                | item          | metric                  | value |
+                | --------------| ----------------------- | ----- |
+                | vocab         | size                    | {vocab_size:,.0f} |
+                | embedding     | dimensions              | {dimensions:,.0f} |
+                | vector length | mean                    | {norms.mean().item():.2f} |
+                | vector length | median                  | {norms.median().item():.2f} |
+                | vector length | stddev                  | {norms.std().item():.2f} |
+                | values        | mean                    | {embeddings.mean().item():.2f} |
+                | values        | median                  | {embeddings.median().item():.2f} |
+                | values        | stddev                  | {embeddings.std().item():.2f} |
+                
+                ## Mean Pooled Quantization Loss
+                
+                This test roundtrips the vectors through quantization, but performs the
+                mean pooling arithmetic in float32 space. The quantized and unquantized
+                mean pooled vectors are compared to each other to determine their cosine
+                similarity, to show how much the meaning of the vector has changed due
+                to quantization.
+                
+                | Precision     | Cosine Similarity |
+                | ------------- | ----------------- |
+                | fp16          | {avg_cosine_similarity[torch.float16]:.5f} |
+                | fp8 e4m3      | {avg_cosine_similarity[torch.float8_e4m3fn]:.5f} |
+                | fp8 e5m2      | {avg_cosine_similarity[torch.float8_e5m2]:.5f} |
+                
+                ## Quantization Loss Per Vector
+                
+                While ultimately the embedding vectors will be mean pooled together, it's
+                still useful to look at the loss per-vector in the embedding table to see
+                which quantization strategies retain the most vector meaning.
+                
+                - **Cosine Similarity** — measures how well the *direction* of embedding vectors
+                is preserved after quantization, independent of scale. This is especially
+                relevant when embeddings are used for similarity search or retrieval.
+                - **MSE (Mean Squared Error)** — emphasizes large errors by squaring the
+                differences. Useful for detecting whether any values are badly distorted.
+                - **MAE (Mean Absolute Error)** — the average absolute difference between
+                original and quantized values. Easier to interpret, less sensitive to outliers.
+
+                | Precision     | Metric | Value |
+                | ------------- | ------ | ----- |
+                | fp16          | cosine similarity | {quantization_loss_cosine(embeddings, torch.float16):.5f} |
+                | fp8 e4m3      | cosine similarity | {quantization_loss_cosine(embeddings, torch.float8_e4m3fn):.5f} |
+                | fp8 e5m2      | cosine similarity | {quantization_loss_cosine(embeddings, torch.float8_e5m2):.5f} |
+                | fp16          | MSE    | {quantization_loss_mse(embeddings, torch.float16):.5f} |
+                | fp8 e4m3      | MSE    | {quantization_loss_mse(embeddings, torch.float8_e4m3fn):.5f} |
+                | fp8 e5m2      | MSE    | {quantization_loss_mse(embeddings, torch.float8_e5m2):.5f} |
+                | fp16          | MAE    | {quantization_loss_mae(embeddings, torch.float16):.5f} |
+                | fp8 e4m3      | MAE    | {quantization_loss_mae(embeddings, torch.float8_e4m3fn):.5f} |
+                | fp8 e5m2      | MAE    | {quantization_loss_mae(embeddings, torch.float8_e5m2):.5f} |
+                
+                ## Tokenizer Examples
+                
+                {indent(tokenizer_output, prefix).strip()}
+                """
+            ).strip()
+        )
+
+
+def export_tokenizer(model_card: ModelCard, tokenizer: Tokenizer) -> None:
+    tokenizer_path = model_card.path() / "tokenizer.json"
+    print(f"Exporting tokenizer: {tokenizer_path}")
+    tokenizer.save(str(tokenizer_path))
+    zst_compress_file(tokenizer_path)
+
+
+def export_sentence_transformers(model_card: ModelCard) -> None:
+    """Extract the embeddings and tokenizer from SentenceTransformers"""
+
+    print("Processing", model_card.name())
+
+    model = SentenceTransformer(model_card.name(), device="cpu")
+    embedding_bag: EmbeddingBag = model[0].embedding  # type: ignore
+    model_card.path().mkdir(exist_ok=True, parents=True)
+    embeddings = torch.Tensor(embedding_bag.weight)
+
+    export_embeddings(model_card, embeddings)
+    export_tokenizer(model_card, model.tokenizer)
+    export_readme(model_card, embeddings, model.tokenizer)
+
+
+def export_model2vec(model_card: ModelCard) -> None:
+    """Extract the embeddings and tokenizer from model2vec"""
+
+    print("Processing", model_card.name())
+
+    model = StaticModel.from_pretrained(model_card.name())
+    model_card.path().mkdir(exist_ok=True, parents=True)
+    embeddings = torch.from_numpy(model.embedding)
+    export_embeddings(model_card, embeddings)
+    export_tokenizer(model_card, model.tokenizer)
+    export_readme(model_card, embeddings, model.tokenizer)
+
+
+def main() -> None:
+    # Static embedders that use sentence_transformers models.
+    sentence_transformers_models = [
+        ModelCard(
+            owner="sentence-transformers",
+            repo="static-similarity-mrl-multilingual-v1",
+            description="""
+            Multi-lingual similarity embeddings that were trained with Matroyshka loss
+            that allows for more effective truncation of the embedding vectors. It
+            was trained on a variety of domains of multilingual datasets.
+            
+            It's a general purpose model that can be used for semantic textual similarity,
+            paraphrase mining, text classification, clustering, and more
+            """,
+            matroyshka_dims=[32, 64, 128, 256, 512, 1024],
+            license="apache-2.0",
+        ),
+        ModelCard(
+            owner="sentence-transformers",
+            repo="static-retrieval-mrl-en-v1",
+            description="""
+            English-only uncased similarity embeddings that were trained with Matroyshka
+            loss that allows for more effective truncation of the embedding vectors. It
+            was trained on a variety of domains of monolingual datasets. I was designed
+            specifically for similarity retrieval.
+            """,
+            matroyshka_dims=[32, 64, 128, 256, 512, 1024],
+            license="apache-2.0",
+        ),
+    ]
+    # Static embedders that use model2vec.
+    model2vec_models = [
+        ModelCard(
+            owner="minishlab",
+            repo="potion-multilingual-128M",
+            # These are assumed as their is no python reference implementation:
+            matroyshka_dims=[32, 64, 128, 256],
+            description="""
+            A multilingual embedder. The details are a bit scant on how it's trained as
+            there is no source code for it. However, it's likely a close architecture
+            to the potion-retrieval-32M model, but trained on Common Crawl data.
+            
+            The 128M references the number of parameters in the embeddings:
+            
+            256 dimensions * 500,353 vocab.
+            """,
+            license="mit",
+        ),
+        ModelCard(
+            owner="minishlab",
+            repo="potion-retrieval-32M",
+            matroyshka_dims=[32, 64, 128, 256, 512],
+            description="""
+            The token embeddings from a monolingual English 32M parameter model that was
+            distilled from embeddings that were initialized from the the multi-domain
+            [BAAI/bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5)
+            
+            The 32M references the number of parameters in the embeddings:
+            
+            512 dimension * 63,091 vocab.
+            """,
+            license="mit",
+        ),
+    ]
+
+    if models_path.exists():
+        print(f"Removing the old models folder: {models_path}")
+        shutil.rmtree(models_path)
+        models_path.mkdir()
+
+    for model_card in sentence_transformers_models:
+        export_sentence_transformers(model_card)
+
+    for model_card in model2vec_models:
+        export_model2vec(model_card)
+
+
+if __name__ == "__main__":
+    main()

scripts/experiments/multilingual.py

@@ -0,0 +1,59 @@
+from sentence_transformers import SentenceTransformer
+from tokenizers import Encoding, Tokenizer
+from torch.nn import EmbeddingBag
+import torch
+
+
+def test_tokenizer():
+    examples = [
+        "This is an example of encoding",
+        "The quick brown fox jumps over the lazy dog.",
+        "Curaçao, naïve fiancé, jalapeño, déjà vu.",
+        "Привет, как дела?",
+        "Бързата кафява лисица прескача мързеливото куче.",
+        "Γρήγορη καφέ αλεπού πηδάει πάνω από τον τεμπέλη σκύλο.",
+        "اللغة العربية جميلة وغنية بالتاريخ.",
+        "مرحبا بالعالم!",
+        "Simplified: 快速的棕色狐狸跳过懒狗。",
+        "Traditional: 快速的棕色狐狸跳過懶狗。",
+        "素早い茶色の狐が怠け者の犬を飛び越える。",
+        "コンピュータープログラミング",
+        "빠른 갈색 여우가 게으른 개를 뛰어넘습니다.",
+        "तेज़ भूरी लोमड़ी आलसी कुत्ते के ऊपर कूदती है।",
+        "দ্রুত বাদামী শিয়াল অলস কুকুরের উপর দিয়ে লাফ দেয়।",
+        "வேகமான பழுப்பு நரி சோம்பேறி நாயின் மேல் குதிக்கிறது.",
+        "สุนัขจิ้งจอกสีน้ำตาลกระโดดข้ามสุนัขขี้เกียจ.",
+        "ብሩክ ቡናማ ቀበሮ ሰነፍ ውሻን ተዘልሏል።",
+        "Hello 世界 مرحبا 🌍",
+        "123, αβγ, абв, العربية, 中文, हिन्दी.",
+    ]
+
+    tokenizer: Tokenizer = Tokenizer.from_file("js/tokenizer.json")
+
+    for example in examples:
+        encoding: Encoding = tokenizer.encode(example)
+        print(example)
+        print(encoding.tokens)
+        print()
+
+
+# https://huggingface.co/sentence-transformers/static-similarity-mrl-multilingual-v1
+model = SentenceTransformer(
+    "sentence-transformers/static-similarity-mrl-multilingual-v1", device="cpu"
+)
+
+embedding_bag: EmbeddingBag = model[0].embedding  # type: ignore
+embeddings = torch.Tensor(embedding_bag.weight)
+
+print(embeddings.shape)
+assert embeddings.shape == torch.Size([105879, 1024])
+
+print("float32")
+print(f"  1024 dim - {embeddings.shape[0] * 1024 * 4 / 1024 / 1024:,.1f} MiB")
+print(f"   512 dim - {embeddings.shape[0] * 512 * 4 / 1024 / 1024:,.1f} MiB")
+print(f"   256 dim - {embeddings.shape[0] * 256 * 4 / 1024 / 1024:,.1f} MiB")
+
+print("float16")
+print(f"  1024 dim - {embeddings.shape[0] * 1024 * 2 / 1024 / 1024:,.1f} MiB")
+print(f"   512 dim - {embeddings.shape[0] * 512 * 2 / 1024 / 1024:,.1f} MiB")
+print(f"   256 dim - {embeddings.shape[0] * 256 * 2 / 1024 / 1024:,.1f} MiB")

scripts/experiments/potion.py

@@ -0,0 +1,14 @@
+from model2vec import StaticModel
+from tokenizers import Tokenizer
+import torch
+
+model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M")
+embeddings = torch.from_numpy(model.embedding)
+
+print("Embedding shape:", embeddings.shape)
+bytes = embeddings.shape[0] * embeddings.shape[1] * 4
+
+print("MiB:", bytes / 1024 / 1024)
+
+tokenizer: Tokenizer = model.tokenizer
+print(tokenizer.to_str())

scripts/experiments/tomaarsen.py

@@ -0,0 +1,15 @@
+from sentence_transformers import SentenceTransformer
+from torch.nn import EmbeddingBag
+import torch
+
+model = SentenceTransformer("tomaarsen/static-retrieval-mrl-en-v1")
+embedding_bag: EmbeddingBag = model[0].embedding  # type: ignore
+embeddings = torch.Tensor(embedding_bag.weight)
+
+assert embeddings.shape == torch.Size([30522, 1024])
+
+print(f"1024 dim - {embeddings.shape[0] * 1024 * 4 / 1024 / 1024:,.1f} MiB:")
+print(f"512 dim - {embeddings.shape[0] * 512 * 4 / 1024 / 1024:,.1f} MiB:")
+print(f"256 dim - {embeddings.shape[0] * 256 * 4 / 1024 / 1024:,.1f} MiB:")
+
+print("Embeddings[0]", embeddings[0])

scripts/upload_models.py

@@ -0,0 +1,47 @@
+import argparse
+import subprocess
+from pathlib import Path
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        # Preserves whitespace in the help text.
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+
+    parser.add_argument(
+        "--tag", type=str, required=True, help="The git tag for the release"
+    )
+
+    args = parser.parse_args()
+    tag: str = args.tag
+
+    try:
+        subprocess.run(
+            ["git", "rev-parse", "--verify", f"refs/tags/{tag}"],
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+    except subprocess.CalledProcessError:
+        raise SystemExit(f"Error: Git tag '{tag}' does not exist.")
+
+    repo_root = Path(__file__).parent.parent.resolve()
+
+    command = f"gsutil cp -r {repo_root / "models"} gs://moz-model-hub/mozilla/static-embeddings/{tag}/"
+
+    print(f"Uploading models")
+    print(command)
+
+    subprocess.run(
+        command,
+        shell=True,
+        check=True,
+    )
+
+    print("All models have been uploaded successfully.")
+
+
+if __name__ == "__main__":
+    main()

tokenizer_config.json

@@ -0,0 +1 @@
+{}

tomaarsen.py

@@ -0,0 +1,15 @@
+from sentence_transformers import SentenceTransformer
+from torch.nn import EmbeddingBag
+import torch
+
+model = SentenceTransformer("tomaarsen/static-retrieval-mrl-en-v1")
+embedding_bag: EmbeddingBag = model[0].embedding  # type: ignore
+embeddings = torch.Tensor(embedding_bag.weight)
+
+assert embeddings.shape == torch.Size([30522, 1024])
+
+print(f"1024 dim - {embeddings.shape[0] * 1024 * 4 / 1024 / 1024:,.1f} MiB:")
+print(f"512 dim - {embeddings.shape[0] * 512 * 4 / 1024 / 1024:,.1f} MiB:")
+print(f"256 dim - {embeddings.shape[0] * 256 * 4 / 1024 / 1024:,.1f} MiB:")
+
+print("Embeddings[0]", embeddings[0])

tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "module": "ESNext",
+    "moduleResolution": "nodenext",
+    // Set the baseUrl to the root of the project.
+    "baseUrl": "src",
+    // Make the type checking as strict as possible.
+    "strict": true,
+    // TypeScript will check JS files only if they have a @ts-check comment in them.
+    "allowJs": true,
+    "checkJs": true,
+    // Only type check, don't emit files.
+    "noEmit": true,
+    // Allow esnext syntax. Otherwise the default is ES5 only.
+    "target": "esnext",
+    "lib": ["esnext", "dom"],
+    "esModuleInterop": true
+  },
+  // Add a @ts-check comment to a JS file to start type checking it.
+  "include": ["example.mjs"],
+  // "files": ["src/@types/globals.d.ts"],
+  "exclude": []
+}