Upload 10 files

chatml_to_mistral.py

@@ -0,0 +1,103 @@
+import json
+import os
+import argparse
+from colorama import init, Fore, Style
+
+init()
+
+def load_json(path):
+    if not os.path.exists(path):
+        return {}
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            return json.load(f)
+    except Exception:
+        return {}
+
+def save_json(path, data):
+    with open(path, 'w', encoding='utf-8') as f:
+        json.dump(data, f, indent=2)
+
+def convert_model(model_path):
+    model_name = os.path.basename(model_path)
+    print(f"Scanning: {model_name}...")
+    
+    gen_path = os.path.join(model_path, "generation_config.json")
+    tok_conf_path = os.path.join(model_path, "tokenizer_config.json")
+    
+    # 1. Check if it is actually a ChatML/32000 model
+    gen_data = load_json(gen_path)
+    current_eos = gen_data.get("eos_token_id")
+    
+    # Handle list format (e.g. [32000, 2])
+    if isinstance(current_eos, list):
+        current_eos = current_eos[0]
+
+    if str(current_eos) != "32000":
+        # If it's already 2, we skip it (unless forced, but let's be safe)
+        if str(current_eos) == "2":
+            return # Already Mistral format
+        print(f"  Skipping: EOS ID is {current_eos} (Not 32000)")
+        return
+
+    print(f"{Fore.YELLOW}  -> Detected ChatML (EOS: 32000). Converting to Mistral format...{Style.RESET_ALL}")
+
+    # 2. Patch Generation Config
+    gen_data["eos_token_id"] = 2
+    gen_data["pad_token_id"] = 2 # Usually good practice to align pad/eos for base mistral
+    save_json(gen_path, gen_data)
+    print(f"     Fixed generation_config.json (ID: 2)")
+
+    # 3. Patch Tokenizer Config
+    if os.path.exists(tok_conf_path):
+        tok_data = load_json(tok_conf_path)
+        
+        # Change string to </s>
+        tok_data["eos_token"] = "</s>"
+        
+        # Remove chat_template if it exists (prevents auto-detection issues later)
+        if "chat_template" in tok_data:
+            del tok_data["chat_template"]
+            
+        save_json(tok_conf_path, tok_data)
+        print(f"     Fixed tokenizer_config.json (Str: </s>)")
+
+    # 4. Patch Special Tokens Map
+    spec_path = os.path.join(model_path, "special_tokens_map.json")
+    if os.path.exists(spec_path):
+        spec_data = load_json(spec_path)
+        spec_data["eos_token"] = "</s>"
+        save_json(spec_path, spec_data)
+        print(f"     Fixed special_tokens_map.json")
+
+    print(f"{Fore.GREEN}  -> Successfully converted {model_name}{Style.RESET_ALL}")
+
+def main():
+    parser = argparse.ArgumentParser(description="Convert ChatML models (EOS 32000) to Mistral format (EOS 2)")
+    parser.add_argument("base_dir", help="Directory containing the model folders (e.g. B:\\7B)")
+    args = parser.parse_args()
+
+    print(f"{Fore.CYAN}--- CHATML TO MISTRAL CONVERTER ---{Style.RESET_ALL}")
+    print("This script changes metadata only. It allows ChatML models to be merged")
+    print("using 'tokenizer: source: base' without errors.\n")
+
+    if not os.path.exists(args.base_dir):
+        print(f"Error: Directory {args.base_dir} does not exist.")
+        return
+
+    # Walk through all subdirectories
+    count = 0
+    for root, dirs, files in os.walk(args.base_dir):
+        for name in dirs:
+            # We look at every folder, check if it's a model inside convert_model
+            full_path = os.path.join(root, name)
+            # Simple check if it looks like a model folder
+            if os.path.exists(os.path.join(full_path, "config.json")):
+                convert_model(full_path)
+                count += 1
+    
+    print("-" * 60)
+    print("Scan complete.")
+
+if __name__ == "__main__":
+    main()

enable_fix_mistral_regex_true.md

@@ -0,0 +1,569 @@
+```
+architecture: MistralForCausalLM
+base_model: B:/12B/models--p-e-w--Mistral-Nemo-Instruct-2407-heretic-noslop
+models:
+  - model: B:/12B/models--allura-org--Tlacuilo-12B
+  - model: B:/12B/models--p-e-w--Mistral-Nemo-Instruct-2407-heretic-noslop
+  - model: B:/12B/models--EldritchLabs--Altair-Stock-12B-v1/MPOA_alt
+  - model: B:/12B/models--inflatebot--MN-12B-Mag-Mell-R1
+  - model: B:/12B/models--MuXodious--Irix-12B-Model_Stock-absolute-heresy
+  - model: B:/12B/models--aixonlab--Aether-12b
+  - model: B:/12B/models--SicariusSicariiStuff--Impish_Bloodmoon_12B
+  - model: B:/12B/models--SicariusSicariiStuff--Sweet_Dreams_12B
+  - model: B:/12B/models--Epiculous--Azure_Dusk-v0.2
+  # - model: B:/12B/models--SicariusSicariiStuff--Impish_Bloodmoon_12B.backup
+merge_method: karcher
+parameters:  
+  max_iter: 30 # Maximum iterations for Karcher mean algorithm (default 10 )
+  tol: 1e-9 # Convergence tolerance for Karcher mean algorithm (default 1e5)
+dtype: float32
+out_dtype: bfloat16
+tokenizer:  
+  source: "union"  
+  tokens:  
+    # Force ChatML EOS tokens  
+    "<|im_start|>":  
+      source: "B:/12B/models--allura-org--Tlacuilo-12B"  
+      force: true  
+    "<|im_end|>":  
+      source: "B:/12B/models--allura-org--Tlacuilo-12B"  
+      force: true  
+    # Keep Mistral tokens  
+    "[INST]":  
+      source: "B:/12B/models--p-e-w--Mistral-Nemo-Instruct-2407-heretic-noslop"  
+     #  source: "B:/12B/models--mistralai--Mistral-Nemo-Instruct-2407"    # The tokenizer system requires all models referenced in token configurations to be present in the merge's model list to build proper embedding permutations. 
+    "[/INST]":  
+      source: "B:/12B/models--p-e-w--Mistral-Nemo-Instruct-2407-heretic-noslop"  
+    # Force </s> as fallback EOS  
+    "</s>":  
+      source: "B:/12B/models--p-e-w--Mistral-Nemo-Instruct-2407-heretic-noslop"  
+      force: true
+
+chat_template: "chatml"
+name: 🧜‍ Nocturne-Nereid-12B-v1
+```
+
+```
+C:\mergekit-main>timeout /t 3 /nobreak && mergekit-yaml C:\mergekit-main\altair7.yaml C:\mergekit-main\merged_model_altair7 --copy-tokenizer --allow-crimes --out-shard-size 5B --trust-remote-code --lazy-unpickle --random-seed 420 --cuda
+
+Waiting for 0 seconds, press CTRL+C to quit ...
+`torch_dtype` is deprecated! Use `dtype` instead!
+Warmup loader cache: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:00<00:00, 1499.99it/s]
+Planning operations
+Executing graph:   0%|                                                                                                                                                                                             | 0/4361 [00:00<?, ?it/s]The tokenizer you are loading from 'B:/12B/models--p-e-w--Mistral-Nemo-Instruct-2407-heretic-noslop' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.
+Loading tokenizers
+The tokenizer you are loading from 'B:/12B/models--allura-org--Tlacuilo-12B' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.
+The tokenizer you are loading from 'B:/12B/models--aixonlab--Aether-12b' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.
+The tokenizer you are loading from 'B:/12B/models--Epiculous--Azure_Dusk-v0.2' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.
+The tokenizer you are loading from 'B:/12B/models--inflatebot--MN-12B-Mag-Mell-R1' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.
+The tokenizer you are loading from 'B:/12B/models--EldritchLabs--Altair-Stock-12B-v1/MPOA_alt' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.
+The tokenizer you are loading from 'B:/12B/models--SicariusSicariiStuff--Impish_Bloodmoon_12B' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.
+The tokenizer you are loading from 'B:/12B/models--SicariusSicariiStuff--Sweet_Dreams_12B' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.
+Building output tokenizer
+Token '<|im_start|>' added with multiple different settings, using first
+Building permutations
+Building tokenizer permutations: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:00<00:00,  9.54it/s]
+Executing graph:   0%|                                                                                                                                                                                   | 3/4361 [00:07<2:44:37,  2.27s/it]
+```
+
+---
+
+Based on the codebase, mergekit doesn't currently support passing `fix_mistral_regex=True` to tokenizer loading. The warnings you're seeing come from the transformers library, but mergekit only passes `trust_remote_code` to tokenizer loading calls, not the `fix_mistral_regex` parameter.
+
+## Current Tokenizer Loading in MergeKit
+
+The main tokenizer loading happens in `mergekit/tokenizer/build.py`:
+
+```python
+# In get_stripped_tokenizer()
+tokenizer = transformers.AutoTokenizer.from_pretrained(
+    path.path,
+    revision=path.revision,
+    trust_remote_code=trust_remote_code,
+    use_fast=True,
+)
+```
+
+And in `build_tokenizer()`:
+```python
+model_tok = transformers.AutoTokenizer.from_pretrained(
+    model.model.path,
+    revision=model.model.revision,
+    trust_remote_code=trust_remote_code,
+)
+```
+
+## How to Add Support
+
+To add `fix_mistral_regex=True` support, you would need to:
+
+1. **Add the parameter to MergeOptions** - This would require modifying the options structure to include a new field for `fix_mistral_regex`
+
+2. **Update all tokenizer loading calls** - Modify the `AutoTokenizer.from_pretrained()` calls throughout the codebase to pass this parameter:
+   - `mergekit/tokenizer/build.py` (lines 49-54, 203-207, 155-157)
+   - `mergekit/merge.py` (line 271-275)
+   - `mergekit/scripts/tokensurgeon.py` (line 148-152)
+
+3. **Add CLI flag** - Add a `--fix-mistral-regex` flag to the CLI that sets this option
+
+## Temporary Workaround
+
+Until mergekit adds native support, you could:
+
+1. **Modify the source locally** - Add `fix_mistral_regex=True` directly to the `AutoTokenizer.from_pretrained()` calls in your local mergekit installation
+
+2. **Use a different tokenizer source** - If one of your models has a correct tokenizer, you could specify it as the tokenizer source in your YAML config
+
+The warnings won't prevent your merge from completing, but as noted, they may lead to incorrect tokenization behavior with the affected models.
+
+## Notes
+
+The `trust_remote_code` parameter you're already using via `--trust-remote-code` follows the same pattern through the codebase - it's passed from CLI options to the tokenizer loading functions. Adding `fix_mistral_regex` support would require implementing the same pattern.
+
+---
+
+> Explain how to enable this for merges on my local copy, step by step.
+
+---
+
+To enable `fix_mistral_regex=True` for your local mergekit installation, you'll need to modify several files to add this parameter throughout the tokenizer loading pipeline.
+
+## Step-by-Step Local Modifications
+
+### 1. Add the option to MergeOptions
+
+Edit `mergekit/options.py` and add the new field:
+
+```python
+# In MergeOptions class (around line 41)
+fix_mistral_regex: bool = False
+```
+
+Also add it to `OPTION_HELP` and `OPTION_CATEGORIES` dictionaries:
+
+```python
+# Around line 114
+OPTION_HELP = {
+    # ... existing options ...
+    "fix_mistral_regex": "Fix incorrect regex pattern in Mistral tokenizers",
+}
+
+# Around line 140
+OPTION_CATEGORIES = {
+    # ... existing categories ...
+    "fix_mistral_regex": "Dangerous Options",
+}
+```
+
+### 2. Update tokenizer loading in mergekit/tokenizer/build.py
+
+Modify all `AutoTokenizer.from_pretrained()` calls to include the new parameter:
+
+```python
+# Line 49-54 in get_stripped_tokenizer()
+tokenizer = transformers.AutoTokenizer.from_pretrained(
+    path.path,
+    revision=path.revision,
+    trust_remote_code=trust_remote_code,
+    use_fast=True,
+    fix_mistral_regex=fix_mistral_regex,  # Add this line
+)
+
+# Line 203-207 in build_tokenizer()
+model_tok = transformers.AutoTokenizer.from_pretrained(
+    model.model.path,
+    revision=model.model.revision,
+    trust_remote_code=trust_remote_code,
+    fix_mistral_regex=fix_mistral_regex,  # Add this line
+)
+
+# Line 155-157 in build_union_tokenizer()
+res = transformers.AutoTokenizer.from_pretrained(
+    p, use_fast=True, trust_remote_code=trust_remote_code,
+    fix_mistral_regex=fix_mistral_regex,  # Add this line
+)
+```
+
+You'll also need to update the function signatures to accept this parameter and pass it through the call chain.
+
+### 3. Update mergekit/merge.py
+
+Modify the tokenizer loading in `_copy_tokenizer()`:
+
+```python
+# Line 271-275
+tokenizer = transformers.AutoTokenizer.from_pretrained(
+    donor_model.model.path,
+    revision=donor_model.model.revision,
+    trust_remote_code=options.trust_remote_code,
+    fix_mistral_regex=options.fix_mistral_regex,  # Add this line
+)
+```
+
+### 4. Update mergekit/scripts/tokensurgeon.py
+
+Modify the tokenizer loading:
+
+```python
+# Line 148-152 in get_stuff()
+tokenizer = transformers.AutoTokenizer.from_pretrained(
+    model.model.path,
+    revision=model.model.revision,
+    trust_remote_code=options.trust_remote_code,
+    fix_mistral_regex=options.fix_mistral_regex,  # Add this line
+)
+```
+
+### 5. Update mergekit/scripts/moe.py
+
+Modify the tokenizer loading:
+
+```python
+# Line 36-38
+tokenizer = transformers.AutoTokenizer.from_pretrained(
+    base_model.model.path, 
+    revision=base_model.model.revision,
+    fix_mistral_regex=merge_options.fix_mistral_regex,  # Add this line
+)
+```
+
+### 6. Update function signatures
+
+You'll need to update the function signatures to pass the `fix_mistral_regex` parameter through the call chain. The key functions to update are:
+- `get_stripped_tokenizer()` in `mergekit/tokenizer/build.py`
+- `build_union_tokenizer()` in `mergekit/tokenizer/build.py`
+- `build_tokenizer()` in `mergekit/tokenizer/build.py`
+- `BuildTokenizer.execute()` in `mergekit/tokenizer/build.py`
+
+### 7. Use the new flag
+
+After making these changes, you can use the new flag in your command:
+
+```bash
+mergekit-yaml C:\mergekit-main\altair7.yaml C:\mergekit-main\merged_model_altair7 --copy-tokenizer --allow-crimes --out-shard-size 5B --trust-remote-code --lazy-unpickle --random-seed 420 --cuda --fix-mistral-regex
+```
+
+## Notes
+
+The `trust_remote_code` parameter follows the same pattern through the codebase - it's passed from CLI options to the tokenizer loading functions [1](#0-0) [2](#0-1) . Adding `fix_mistral_regex` support requires implementing the same pattern throughout all tokenizer loading locations [3](#0-2) [4](#0-3) .
+
+The warnings you're seeing come from the transformers library and indicate that the tokenizer regex pattern may lead to incorrect tokenization [5](#0-4) . While the merge will complete without this fix, enabling it will ensure proper tokenization behavior.
+
+---
+
+> I did all of that and now it says this
+
+```
+C:\mergekit-main>timeout /t 3 /nobreak && mergekit-yaml C:\mergekit-main\altair7.yaml C:\mergekit-main\merged_model_altair7 --copy-tokenizer --allow-crimes --out-shard-size 5B --trust-remote-code --lazy-unpickle --random-seed 420 --cuda --fix-mistral-regex
+
+Waiting for 0 seconds, press CTRL+C to quit ...
+`torch_dtype` is deprecated! Use `dtype` instead!
+Warmup loader cache: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:00<00:00, 1499.99it/s]
+Planning operations
+Executing graph:   0%|                                                                                                                                                                                     | 1/4361 [00:00<14:53,  4.88it/s]
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python311\Scripts\mergekit-yaml.exe\__main__.py", line 5, in <module>
+  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python311\Lib\site-packages\click\core.py", line 1485, in __call__
+    return self.main(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python311\Lib\site-packages\click\core.py", line 1406, in main
+    rv = self.invoke(ctx)
+         ^^^^^^^^^^^^^^^^
+  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python311\Lib\site-packages\click\core.py", line 1269, in invoke
+    return ctx.invoke(self.callback, **ctx.params)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python311\Lib\site-packages\click\core.py", line 824, in invoke
+    return callback(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "C:\mergekit-main\mergekit\options.py", line 169, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "C:\mergekit-main\mergekit\scripts\run_yaml.py", line 30, in main
+    run_merge(
+  File "C:\mergekit-main\mergekit\merge.py", line 85, in run_merge
+    for _task, value in exec.run(quiet=options.quiet):
+  File "C:\mergekit-main\mergekit\graph.py", line 747, in run
+    for handle, value in self._run(quiet=quiet, desc=desc):
+    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "C:\mergekit-main\mergekit\graph.py", line 712, in _run
+    res = task.execute(**cpu_arguments)
+          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "C:\mergekit-main\mergekit\tokenizer\build.py", line 294, in execute
+    return build_tokenizer(
+           ^^^^^^^^^^^^^^^^
+  File "C:\mergekit-main\mergekit\tokenizer\build.py", line 193, in build_tokenizer
+    tokenizer_base = get_stripped_tokenizer(
+                     ^^^^^^^^^^^^^^^^^^^^^^^
+  File "C:\mergekit-main\mergekit\tokenizer\build.py", line 54, in get_stripped_tokenizer
+    fix_mistral_regex=fix_mistral_regex,  # Add this line
+                      ^^^^^^^^^^^^^^^^^
+NameError: name 'fix_mistral_regex' is not defined
+
+C:\mergekit-main>
+```
+
+---
+
+The error occurs because you added `fix_mistral_regex` to the `AutoTokenizer.from_pretrained()` call in `get_stripped_tokenizer()` but didn't update the function signature to accept this parameter. The parameter needs to be passed through the entire call chain.
+
+## Fix the function signatures
+
+### 1. Update `get_stripped_tokenizer()` in mergekit/tokenizer/build.py
+
+```python
+def get_stripped_tokenizer(
+    path: ModelPath, trust_remote_code: bool = False, fix_mistral_regex: bool = False
+) -> transformers.PreTrainedTokenizerFast:
+```
+
+### 2. Update `build_tokenizer()` in mergekit/tokenizer/build.py
+
+```python
+def build_tokenizer(
+    base_model: Optional[ModelReference],
+    referenced_models: List[ModelReference],
+    tokenizer_source: Union[Literal["union"], Literal["base"], ModelReference],
+    trust_remote_code: bool,
+    fix_mistral_regex: bool = False,
+    add_tokens: Optional[List[str]] = None,
+) -> TokenizerInfo:
+```
+
+### 3. Update `BuildTokenizer.execute()` in mergekit/tokenizer/build.py
+
+```python
+class BuildTokenizer(Task[TokenizerInfo]):
+    base_model: Optional[ModelReference]
+    referenced_models: Tuple[ModelReference, ...]
+    tokenizer_source: Union[Literal["union"], Literal["base"], ModelReference]
+    add_tokens: Optional[Tuple[str, ...]]
+    trust_remote_code: bool = False
+    fix_mistral_regex: bool = False
+
+    def execute(self, **_kwargs) -> TokenizerInfo:
+        return build_tokenizer(
+            base_model=self.base_model,
+            referenced_models=self.referenced_models,
+            tokenizer_source=self.tokenizer_source,
+            trust_remote_code=self.trust_remote_code,
+            fix_mistral_regex=self.fix_mistral_regex,
+            add_tokens=self.add_tokens,
+        )
+```
+
+### 4. Update the call in `build_tokenizer()`
+
+```python
+tokenizer_base = get_stripped_tokenizer(
+    base_model.model, trust_remote_code=trust_remote_code, fix_mistral_regex=fix_mistral_regex
+)
+```
+
+### 5. Update the call in `MergePlanner.__init__()` in mergekit/plan.py
+
+```python
+self._tokenizer_task = BuildTokenizer(
+    base_model=config.base_model,
+    referenced_models=tuple(config.referenced_models()),
+    tokenizer_source=tokenizer_source,
+    trust_remote_code=options.trust_remote_code,
+    fix_mistral_regex=options.fix_mistral_regex,
+    add_tokens=tuple(token_cfg.keys()),
+)
+```
+
+### 6. Update `build_union_tokenizer()` in mergekit/tokenizer/build.py
+
+```python
+def build_union_tokenizer(
+    base_tok: transformers.PreTrainedTokenizerBase,
+    tokenizers: Dict[ModelReference, transformers.PreTrainedTokenizerBase],
+    trust_remote_code: bool = False,
+    fix_mistral_regex: bool = False,
+) -> transformers.PreTrainedTokenizerBase:
+```
+
+And update the call within it:
+```python
+res = transformers.AutoTokenizer.from_pretrained(
+    p, use_fast=True, trust_remote_code=trust_remote_code, fix_mistral_regex=fix_mistral_regex
+)
+```
+
+Also update the call to `build_union_tokenizer()` in `build_tokenizer()`:
+```python
+tokenizer_out = build_union_tokenizer(
+    tokenizer_base, tokenizers, trust_remote_code=trust_remote_code, fix_mistral_regex=fix_mistral_regex
+)
+```
+
+## Notes
+
+The error occurs because `get_stripped_tokenizer()` doesn't have `fix_mistral_regex` in its parameter list, so when you try to use it in the function body, Python raises a `NameError` [1](#0-0) . The parameter must be added to each function in the call chain from `BuildTokenizer.execute()` down to the actual `AutoTokenizer.from_pretrained()` calls [2](#0-1) .
+
+---
+
+> I now get this error
+
+```
+C:\mergekit-main>timeout /t 3 /nobreak && mergekit-yaml C:\mergekit-main\altair7.yaml C:\mergekit-main\merged_model_altair7 --copy-tokenizer --allow-crimes --out-shard-size 5B --trust-remote-code --lazy-unpickle --random-seed 420 --cuda --fix-mistral-regex
+
+Waiting for 0 seconds, press CTRL+C to quit ...
+`torch_dtype` is deprecated! Use `dtype` instead!
+Warmup loader cache: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:00<00:00, 1809.80it/s]
+Planning operations
+Executing graph:   0%|                                                                                                                                                                                             | 0/4361 [00:00<?, ?it/s]Loading tokenizers
+Building output tokenizer
+Token '<|im_start|>' added with multiple different settings, using first
+Executing graph:   0%|                                                                                                                                                                                   | 1/4361 [00:04<5:27:21,  4.51s/it]
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python311\Scripts\mergekit-yaml.exe\__main__.py", line 5, in <module>
+  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python311\Lib\site-packages\click\core.py", line 1485, in __call__
+    return self.main(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python311\Lib\site-packages\click\core.py", line 1406, in main
+    rv = self.invoke(ctx)
+         ^^^^^^^^^^^^^^^^
+  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python311\Lib\site-packages\click\core.py", line 1269, in invoke
+    return ctx.invoke(self.callback, **ctx.params)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "C:\Users\Administrator\AppData\Local\Programs\Python\Python311\Lib\site-packages\click\core.py", line 824, in invoke
+    return callback(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "C:\mergekit-main\mergekit\options.py", line 169, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "C:\mergekit-main\mergekit\scripts\run_yaml.py", line 30, in main
+    run_merge(
+  File "C:\mergekit-main\mergekit\merge.py", line 85, in run_merge
+    for _task, value in exec.run(quiet=options.quiet):
+  File "C:\mergekit-main\mergekit\graph.py", line 747, in run
+    for handle, value in self._run(quiet=quiet, desc=desc):
+    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "C:\mergekit-main\mergekit\graph.py", line 712, in _run
+    res = task.execute(**cpu_arguments)
+          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "C:\mergekit-main\mergekit\tokenizer\build.py", line 296, in execute
+    return build_tokenizer(
+           ^^^^^^^^^^^^^^^^
+  File "C:\mergekit-main\mergekit\tokenizer\build.py", line 232, in build_tokenizer
+    tokenizer_out = build_union_tokenizer(
+                    ^^^^^^^^^^^^^^^^^^^^^^
+  File "C:\mergekit-main\mergekit\tokenizer\build.py", line 158, in build_union_tokenizer
+    fix_mistral_regex=fix_mistral_regex,  # Add this line
+                      ^^^^^^^^^^^^^^^^^
+NameError: name 'fix_mistral_regex' is not defined
+
+C:\mergekit-main>
+```
+
+---
+
+The error occurs because you haven't updated the function signature for `build_union_tokenizer()` to accept the `fix_mistral_regex` parameter. Here's the fix:
+
+## Fix for `build_union_tokenizer()`
+
+Update the function signature in `mergekit/tokenizer/build.py`:
+
+```python
+def build_union_tokenizer(
+    base_tok: transformers.PreTrainedTokenizerBase,
+    tokenizers: Dict[ModelReference, transformers.PreTrainedTokenizerBase],
+    trust_remote_code: bool = False,
+    fix_mistral_regex: bool = False,  # Add this parameter
+) -> transformers.PreTrainedTokenizerBase:
+```
+
+And update the `AutoTokenizer.from_pretrained()` call within the same function (around line 155-157):
+
+```python
+res = transformers.AutoTokenizer.from_pretrained(
+    p, 
+    use_fast=True, 
+    trust_remote_code=trust_remote_code, 
+    fix_mistral_regex=fix_mistral_regex  # This line already exists
+)
+```
+
+The call to `build_union_tokenizer()` in `build_tokenizer()` (line 232) should already be passing the parameter correctly based on your previous updates.
+
+## Complete Function Update
+
+Here's the complete updated function for `build_union_tokenizer()`:
+
+```python
+def build_union_tokenizer(
+    base_tok: transformers.PreTrainedTokenizerBase,
+    tokenizers: Dict[ModelReference, transformers.PreTrainedTokenizerBase],
+    trust_remote_code: bool = False,
+    fix_mistral_regex: bool = False,
+) -> transformers.PreTrainedTokenizerBase:
+    # ... rest of function remains the same ...
+```
+
+After this change, your merge should proceed without the `NameError` [1](#0-0) .
+
+## Notes
+
+The error pattern shows that `fix_mistral_regex` needs to be passed through the entire call chain from the CLI options down to each `AutoTokenizer.from_pretrained()` call [2](#0-1) . You've successfully updated most of the chain, but this function signature was the missing piece.
+
+---
+
+**Analyzing the Fix**
+
+I'm currently focused on dissecting the `fix_mistral_regex=True` parameter within `mergekit`. My primary goal is to determine the precise function of this fix in the context of merging Mistral models. I need to understand its impact and how it addresses a potential issue or enhances the merging process.
+
+
+**Investigating the Root**
+
+I'm now diving into the 'why' behind the regex fix, focusing on the author's explanation. The provided text details issues with Mistral tokenizers and chat templates, particularly around premature termination, endless repetition, and hallucinated end-of-sequence tokens. The fix seems to address these problems in a specific Karcher merge, notably resolving issues with chat templates and jailbreaks, while acknowledging some template-specific behavior.
+
+
+**Clarifying the Bugfix**
+
+I'm now zeroing in on the technical implementation. The core of `fix_mistral_regex=True` is overriding a faulty regex pattern in the tokenizer config. This pattern is part of how the tokenizer handles pre-tokenization, and the original pattern was causing issues with specific Mistral models, including those Nemo 12B-based models. This fix is crucial for compatibility because of this bug in the `transformers` library, which causes the tokenizer to misbehave in edge cases.
+
+
+**Connecting the Incompatibilities**
+
+I'm now understanding how different tokenizer configs cause issues during the merge process. Building a "union" tokenizer with conflicting regex patterns leads to misaligned token mappings, especially for special tokens. The `fix_mistral_regex=True` parameter directly addresses this problem. I am able to identify the core issues: early termination or endless repetition, and hallucination of `<|im_end|>`. These are a direct result of the broken regex and tokenizer incompatibility across the various models being merged.
+
+
+**Mapping the Tokenization**
+
+I'm now focusing on how the fix's implementation. It seems by overriding the pre-tokenization regex, mergekit accurately maps overlapping tokens and inserts the ChatML tokens, like `<|im_start|>` and `<|im_end|>`, correctly. This prevents the underlying regex from disrupting token boundaries, ensuring proper handling of different prompt formats during the merge. I'm now documenting how this fixes ChatML/Special Tokens, generation loops, and resolves incompatibilities.
+
+Based on the logs, the Hugging Face issue linked in the warnings, and the author's notes, here is an explanation of what `fix_mistral_regex=True` does and why it solves the merge issues.
+
+### What the fix actually does (Technically)
+Mistral models (specifically those using their newer "Tekken" tokenizer, like Mistral Nemo 12B and Mistral Small 3) rely on a specific Regular Expression (regex) pattern to pre-tokenize text—meaning how it splits sentences into chunks before assigning token IDs. 
+
+Recently, a bug was discovered where the regex pattern saved inside the `tokenizer_config.json` of these models was slightly incorrect. This caused the tokenizer to misinterpret certain character boundaries, spaces, and special tokens. 
+
+When you pass `fix_mistral_regex=True` to `AutoTokenizer.from_pretrained()` in the `transformers` library, it tells the library to **ignore the broken regex pattern in the model's files and dynamically replace it with the corrected pattern**. 
+
+In the context of `mergekit`, if this fix isn't applied, `mergekit` tries to build a "union" tokenizer out of 9 different models using a broken set of rules. This results in a corrupted vocabulary mapping where special tokens get misaligned.
+
+### Why it fixes the issues (According to the Author)
+
+According to the author's notes, applying this fix during the merge process resolved several severe, model-breaking behaviors:
+
+**1. It stops the model from hallucinating `<|im_end|>` as plain text**
+Without the fix, the broken regex was likely causing the tokenizer to fail at recognizing `<|im_end|>` as a single, distinct "Special Token." Instead, the model was treating it as regular text. Because the text generation interface didn't see the actual *End of Sequence ID*, it just printed the literal characters `<|im_end|>` to the screen and kept generating. Fixing the regex ensures the union tokenizer correctly isolates and maps these ChatML special tokens.
+
+**2. It fixes endless repetition and early termination**
+Because the token mappings were corrupted by the bad regex, the merged model's language head was firing the wrong token IDs. It would either output an EOS (End of Sequence) token immediately (terminating early) or completely lose the ability to output an EOS token (repeating endlessly). The fix restores the correct token IDs for stopping generation.
+
+**3. It resolves "Tokenizer Incompatibility" across different fine-tunes**
+The author merged 9 different models. Some of these models (like `Mistral-Nemo-Instruct-2407`) use the default Mistral `[INST]` format, while others (like `Tlacuilo-12B`) were fine-tuned to use `ChatML` (`<|im_start|>`). 
+Normally, merging models with different prompt formats and added tokens causes severe brain damage to the model. By fixing the underlying regex, `mergekit` was able to accurately map the overlapping base tokens and cleanly inject the forced ChatML tokens (as defined in the YAML config), creating a stable hybrid tokenizer.
+
+### The Author's Final Result
+By applying this fix, the author successfully created a 9-model Karcher merge that:
+*   **Works perfectly with ChatML:** It follows instructions, doesn't loop, and stops when it is supposed to.
+*   **Accepts Jailbreaks:** The control vectors from the "heretic/noslop" models successfully carried over.
+*   **Has one remaining quirk:** The author notes that if you try to use the *Mistral Tekken* prompt format instead of *ChatML*, the model terminates early and refuses prompts. This makes sense: the YAML config explicitly forced the ChatML tokens to overwrite the Mistral ones, meaning the model's brain is now permanently wired to expect ChatML.

folder+subfolder_content_combiner_anyfiles.py

@@ -0,0 +1,86 @@
+import os
+import sys
+
+# --- Configuration ---
+OUTPUT_FILENAME = "!combo.txt"
+# Folders to skip to prevent massive/useless data bloat
+BLACKLIST_DIRS = {".git", "__pycache__", ".ipynb_checkpoints", "node_modules", "venv", ".vscode"}
+# --- End Configuration ---
+
+def combine_all_recursive():
+    """
+    Recursively combines all files in the script's directory and all subdirectories
+    into a single output file with relative path tags.
+    """
+    try:
+        # Determine the script's own path
+        if getattr(sys, 'frozen', False):
+            script_full_path = os.path.abspath(sys.executable)
+        else:
+            script_full_path = os.path.abspath(__file__)
+
+        script_dir = os.path.dirname(script_full_path)
+        output_filepath = os.path.join(script_dir, OUTPUT_FILENAME)
+
+        print(f"Scanning recursively in: {script_dir}")
+        print(f"Output file: {output_filepath}")
+        print("-" * 40)
+
+        file_list = []
+
+        # os.walk handles the recursion automatically
+        for root, dirs, files in os.walk(script_dir):
+            # Modify dirs in-place to skip blacklisted folders
+            dirs[:] = [d for d in dirs if d not in BLACKLIST_DIRS]
+            
+            for filename in files:
+                full_path = os.path.join(root, filename)
+                
+                # Exclude the script itself and the output file
+                if (os.path.normpath(full_path) == os.path.normpath(script_full_path) or 
+                    filename == OUTPUT_FILENAME):
+                    continue
+                
+                # Store relative path for sorting and tagging
+                rel_path = os.path.relpath(full_path, script_dir)
+                file_list.append((rel_path, full_path))
+
+        # Sort by relative path so the combo file is organized logically
+        file_list.sort(key=lambda x: x[0])
+
+        if not file_list:
+            print("No files found to combine.")
+            return
+
+        print(f"Found {len(file_list)} files. Starting combination...")
+
+        with open(output_filepath, 'w', encoding='utf-8', errors='replace') as outfile:
+            outfile.write("```\n") # Opening wrapper
+
+            for rel_path, full_path in file_list:
+                print(f"Adding: {rel_path}")
+                try:
+                    outfile.write(f"<{rel_path}>\n")
+                    with open(full_path, 'r', encoding='utf-8', errors='replace') as infile:
+                        outfile.write(infile.read())
+                    outfile.write(f"\n</{rel_path}>\n\n")
+                except Exception as e:
+                    outfile.write(f"ERROR READING FILE {rel_path}: {e}\n")
+                    outfile.write(f"</{rel_path}>\n\n")
+
+            outfile.write("```\n") # Closing wrapper
+
+        print("-" * 40)
+        print(f"SUCCESS! Combined {len(file_list)} files into '{OUTPUT_FILENAME}'")
+
+    except Exception as e:
+        print(f"CRITICAL ERROR: {e}")
+        import traceback
+        traceback.print_exc()
+
+if __name__ == "__main__":
+    combine_all_recursive()
+    try:
+        input("\nProcess complete. Press Enter to exit...")
+    except EOFError:
+        pass

model_tools.md

@@ -8,7 +8,7 @@ pinned: false
 ---
 
 # Model Tools by Naphula
-Tools to enhance LLM quantizations and merging
+Tools to enhance LLM quantizations and merging. Merge and audit large language models with low VRAM.
 
 # [graph_v18.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/graph_v18.py)
 - Merge models in minutes instead of hours on low VRAM. For a 3060/3060 Ti user: This script enables functionality that is otherwise impossible (merging 70B models or large 7B merges with `--cuda`) without OOM. [More details here](https://huggingface.co/spaces/Naphula/model_tools/blob/main/mergekit_low-VRAM-graph_patch.md)
@@ -17,6 +17,14 @@ Tools to enhance LLM quantizations and merging
 # config.py
 - Simply replace line 13 | BEFORE `ScalarOrGradient: TypeAlias = Union[float, List[float]]` → AFTER `ScalarOrGradient: TypeAlias = Union[float, List[float], str, bool]` | to allow for custom filepath strings within parameter settings.
 
+# [enable_fix_mistral_regex_true.md](https://huggingface.co/spaces/Naphula/model_tools/blob/main/enable_fix_mistral_regex_true.md)
+- Merge models with extreme tokenizer incompatibility. Requires modifying the `mergekit.yaml` `tokenizer` section and adding `--fix-mistral-regex` to your merge commands. (Note: Do not use `token_surgeon.py`, `gen_id_patcher.py`, or `vocab_id_patcher.py` with this, they are obsolete now.) Configured for MN 12B by default. Follow the steps in this guide to modify these scripts:
+- `mergekit/merge.py`
+- `mergekit/options.py`
+- `mergekit/scripts/moe.py`
+- `mergekit/scripts/tokensurgeon.py`
+- `mergekit/tokenizer/build.py`
+
 # [audit_della.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/Audits/audit_della.py)
 - Audit the compatibility of donor models for `Della` merges before merging. See: [example chart Asmodeus](https://huggingface.co/spaces/Naphula/model_tools/blob/main/Audits/Asmodeus_Audit.png), [example log Asmodeus](https://huggingface.co/spaces/Naphula/model_tools/blob/main/Audits/Asmodeus_Audit.log), [example chart Slimaki](https://huggingface.co/spaces/Naphula/model_tools/blob/main/Audits/Slimaki_Audit.png), [example log Slimaki](https://huggingface.co/spaces/Naphula/model_tools/blob/main/Audits/Slimaki_Audit.log)
 
@@ -40,13 +48,16 @@ Tools to enhance LLM quantizations and merging
 - Then assign the num_experts_per_tok in config.json (or the config.yaml)
 
 # [tokensurgeon.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/tokensurgeon.py)
-- Uses adaptive VRAM from Grim Jim's `measure.py` like `graph_v18` to prevent OOM. Use recommended [batch file](https://huggingface.co/spaces/Naphula/model_tools/blob/main/fix_tokenizers.bat) here or modify sh. This supposedly avoids 'Potemkin village' fake patches like `gen_id_patcher` and `vocab_id_patcher`.
+- Uses adaptive VRAM from Grim Jim's `measure.py` like `graph_v18` to prevent OOM. Use recommended [batch file](https://huggingface.co/spaces/Naphula/model_tools/blob/main/fix_tokenizers.bat) here or modify sh. This avoids 'Potemkin village' fake patches like `gen_id_patcher` and `vocab_id_patcher`. For this to work properly, you must also run [shield_embeddings.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/shield_embeddings.py) and [shield_norms.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/shield_norms.py) on any merges made from models patched with with tokensurgeon.
 
 # [tokeninspector.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/tokeninspector.py)
 - Audit your tokensurgeon results.
 
+# [arcee_fusion_salience_scanner.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/arcee_fusion_salience_scanner.py)
+- Scan the salience % of your arcee_fusion merges. The default `tukey_fence` value is 1.5 which results in 12.5% salience, but [this can be adjusted (see guide here)](modify_arcee_fusion_tukey_fence_parameter.md).
+
 # [eos_scanner.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/eos_scanner.py)
-- Updated! This tool scans the tokenizer jsons to detect any mismatches with EOS tokens, which cause early termination bugs. You can then use the [gen_id_patcher.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/gen_id_patcher.py) to patch missing `generation_config.json` files for EOS token. See [this post](https://huggingface.co/Naphula/Q0_Bench/discussions/1?not-for-all-audiences=true#6987717c762f0a45f672e250) as well as the [EOS Scanner ReadMe](https://huggingface.co/spaces/Naphula/model_tools/blob/main/eos_scanner_readme.md) for more info.
+- Updated! This tool scans the tokenizer jsons to detect any mismatches with EOS tokens, which cause early termination bugs. You can then use the [gen_id_patcher.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/gen_id_patcher.py) and [vocab_id_patcher.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/vocab_id_patcher.py), or the [chatml_to_mistral.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/chatml_to_mistral.py) to patch missing `generation_config.json` files for EOS token. See [this post](https://huggingface.co/Naphula/Q0_Bench/discussions/1?not-for-all-audiences=true#6987717c762f0a45f672e250) as well as the [EOS Scanner ReadMe](https://huggingface.co/spaces/Naphula/model_tools/blob/main/eos_scanner_readme.md) for more info.
 
 # [weight_counter.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/weight_counter.py)
 - This counts the number of models in a yaml and adds up the total weight values. Useful for large della/ties merges.
@@ -63,6 +74,9 @@ Tools to enhance LLM quantizations and merging
 # [textonly_ripper_v2.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/textonly_ripper_v2.py)
 - Converts a sharded, multimodal (text and vision) model into a text-only version. Readme at [textonly_ripper.md](https://huggingface.co/spaces/Naphula/model_tools/blob/main/textonly_ripper.md)
 
+# [json_reverter.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/json_reverter.py)
+- Revert changes to all JSON files done by `gen_id_patcher.py`, `vocab_id_patcher.py` or other scripts, within a specified root folder. It re-downloads the source files from the HF repo.
+
 # [vocab_resizer.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/vocab_resizer.py)
 - Converts models with larger vocab_sizes to a standard size (default 131072 Mistral 24B) for use with mergekit. Note that `tokenizer.model` must be manually copied into the `/fixed/` folder.
 
@@ -70,11 +84,14 @@ Tools to enhance LLM quantizations and merging
 - This script will load a "fat" 18.9GB model (default Gemma 9B), force it to tie the weights (deduplicating the lm_head), and re-save it. This will drop the file size to ~17.2GB and make it compatible with the others.
 
 # [model_index_json_generator.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/model_index_json_generator.py)
-- Generates a missing `model.safetensors.index.json` file. Useful for cases where safetensors may have been sharded at the wrong size.
+- Generates a missing `model.safetensors.index.json` file. Useful for cases where safetensors may have been sharded at the wrong size. [Single tensor variant here.](https://huggingface.co/spaces/Naphula/model_tools/blob/main/model_index_json_generator_SingleTensor.py)
 
 # [folder_content_combiner_anyfiles.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/folder_content_combiner_anyfiles.py)
 - Combines all files in the script's current directory into a single output file, sorted alphabetically.
 
+# [folder+subfolder_content_combiner_anyfiles.py](https://huggingface.co/spaces/Naphula/model_tools/blob/main/folder+subfolder_content_combiner_anyfiles.py)
+- Combines all files in the script's directory, including all files within subdirectories (excluding blacklisted formats) into a single output file, sorted alphabetically.
+
 # [GGUF Repo Suite](https://huggingface.co/spaces/Naphula/gguf-repo-suite)
 - Create and quantize Hugging Face models
 

modify_arcee_fusion_tukey_fence_parameter.md

@@ -0,0 +1,94 @@
+The default setting is 1.5 which merges to 12% salience. If you want 25% then decrease to 0.75. If you want 33% use 0.4, and 0.0 results in 50% salience.
+
+`arcee_fusion` hardcodes this to 1.5 by default but you can simply edit the python script before merging (easier), or update the code to assign it as a yaml parameter (more complex).
+
+[Here is a scanner to audit Arcee_Fusion merge salience](https://huggingface.co/spaces/Naphula/model_tools/blob/main/arcee_fusion_salience_scanner.py)
+
+---
+
+In the context of merging models using the `mergekit` framework, the provided code showcases a **Dynamic Threshold Fusion** mechanism. Here's an overview of how modifying the **Tukey fence** parameter (from **1.5** to **0.75** or **3.0**) would affect the model merging process.
+
+## Impact of Changing Tukey Fence Parameter
+
+### Current Implementation
+In your code, the dynamic threshold is set as:
+
+```python
+dynamic_threshold = median + 1.0 * iqr  # Tukey Fence
+```
+
+This uses **1.0** as the multiplier instead of **1.5**, but let's assume it was meant to be **1.5** for generalizing the Tukey fence.
+
+### Changes to k Values
+
+#### 1. **If k = 0.75**
+- **Increased Sensitivity**: The dynamic threshold would be lower, leading to more values being included in the fusion mask.
+- **More Features Merged**: This results in a potentially more aggressive merging of parameters, effectively incorporating even slight differences between the model weights.
+- **Use Cases**: Ideal for exploratory processes or applications where capturing subtle variations is critical.
+
+#### Implementation
+You would change the line in `calculate_dynamic_threshold`:
+
+```python
+dynamic_threshold = median + 0.75 * iqr  # Tukey Fence with k = 0.75
+```
+
+---
+
+#### 2. **If k = 3.0**
+- **Decreased Sensitivity**: The dynamic threshold would be much higher, allowing only significant differences to be considered.
+- **Fewer Features Merged**: This could lead to a conservative merging approach, where only clear outliers are merged, possibly overlooking important but smaller differences.
+- **Use Cases**: Useful in high-stakes environments where only distinctly different parameters should contribute to the final model.
+
+#### Implementation
+You would change the same line to:
+
+```python
+dynamic_threshold = median + 3.0 * iqr  # Tukey Fence with k = 3.0
+```
+
+---
+
+## Summary of Effects on Fusion Process
+- **k = 0.75**: This results in a more inclusive fusion mask. More difference is merged, potentially capturing more nuanced information.
+- **k = 3.0**: This leads to a restrictive fusion mask. Smaller differences would be ignored, possibly resulting in a model that retains more characteristics of the base model if the other model has less deviation.
+
+### Final Thought
+The choice of **k** can significantly dictate the behavior and performance of the merged model. Deciding whether to accommodate subtle differences or enforce strict conditions on merging will greatly depend on the specific use case and the importance of the characteristics being learned from either model. If you'd like to delve deeper into any specific aspect or need further assistance, just let me know!
+
+---
+
+The reason you are seeing a consistent **~12%** is not because of a hardcoded number, but because of a specific statistical formula used to identify "outliers" in the weight distribution.
+
+In `arcee_fusion.py`, the density is controlled by the **Tukey's Fence** method. Here are the specific lines of code and the math behind them:
+
+### 1. The Threshold Calculation
+In `arcee_fusion.py`, inside the `DynamicThresholdFusion` class:
+
+```python
+# Line 41-42: Calculate the Interquartile Range (IQR)
+iqr = q3 - q1
+
+# Line 45: The "Magic" Formula
+dynamic_threshold = median + 1.5 * iqr
+```
+
+### 2. The Technical Analysis of "12%"
+The value **`1.5`** is the standard multiplier for **Tukey's Fences**, a statistical method used to find "outliers." 
+
+*   **The Logic:** Arcee Fusion assumes that most weight changes in a fine-tune are "noise" or "jitter." It only wants to keep the "salient" changes—the ones that are statistically significant.
+*   **The Distribution:** Importance scores in neural networks usually follow a **Laplace** or **Heavy-Tailed Gaussian** distribution. 
+*   **The Result:** In these types of distributions, the data points that fall above `Median + 1.5 * IQR` (the "upper fence") typically represent the top **10% to 15%** of the values. 
+
+Your result of **12.1%** means that in every layer, roughly 12% of the weights were "important enough" to cross that statistical fence. The other 88% were considered "noise" and were reverted to the Mistral Base weights.
+
+### 3. Why is it so consistent across layers?
+The scanner shows almost no variation (11.9% to 12.2%). This reveals a technical truth about your fine-tunes: **The "knowledge" is spread evenly.**
+
+If you were merging a model that only learned a specific task (like coding), you would see a "spike" in the middle layers (where logic is processed) and 0% in the early layers. Because your layers are all ~12%, it means the fine-tuned models you are using modified the entire "brain" of the model uniformly.
+
+### 4. How to change this (The "Patch")
+If you want a higher density (e.g., you want 25% or 50% "New Info"), you have to modify the multiplier in `arcee_fusion.py`.
+
+*   **To get MORE new info:** Lower the `1.5` to `0.5`.
+*   **To get LESS new info:** Raise the `1.5` to `3.0` (this is the "Extreme Outlier" threshold).

shield_embeddings.py

@@ -0,0 +1,66 @@
+import os
+import json
+import gc
+import shutil
+from safetensors.torch import load_file, save_file
+import argparse
+
+def get_weight_map(model_path):
+    index_path = os.path.join(model_path, "model.safetensors.index.json")
+    if os.path.exists(index_path):
+        with open(index_path, 'r') as f:
+            return json.load(f)["weight_map"]
+    for f in os.listdir(model_path):
+        if f.endswith(".safetensors"):
+            return {"model.embed_tokens.weight": f, "lm_head.weight": f}
+    return {}
+
+def swap_tensor(base_path, merged_path, tensor_name):
+    base_map = get_weight_map(base_path)
+    merged_map = get_weight_map(merged_path)
+    
+    if tensor_name not in base_map or tensor_name not in merged_map:
+        return
+        
+    base_shard = os.path.join(base_path, base_map[tensor_name])
+    merged_shard = os.path.join(merged_path, merged_map[tensor_name])
+    
+    # Load pristine tensor
+    base_tensors = load_file(base_shard, device="cpu")
+    pristine_tensor = base_tensors[tensor_name].clone()
+    del base_tensors
+    gc.collect()
+
+    # Load merged shards
+    merged_tensors = load_file(merged_shard, device="cpu")
+    merged_tensors[tensor_name] = pristine_tensor
+    
+    # ATOMIC RENAME STRATEGY (The only way to beat Windows 1224)
+    backup_shard = merged_shard + ".old"
+    os.rename(merged_shard, backup_shard) # Move current file to side
+    
+    try:
+        save_file(merged_tensors, merged_shard, metadata={"format": "pt"})
+    except Exception as e:
+        os.rename(backup_shard, merged_shard) # Restore if fail
+        raise e
+    
+    # Cleanup
+    del merged_tensors
+    del pristine_tensor
+    gc.collect()
+    os.remove(backup_shard) # Now delete the old mapped file
+    
+    print(f"  ✅ Successfully shielded {tensor_name}!")
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("base_model")
+    parser.add_argument("merged_model")
+    args = parser.parse_args()
+    
+    swap_tensor(args.base_model, args.merged_model, "model.embed_tokens.weight")
+    swap_tensor(args.base_model, args.merged_model, "lm_head.weight")
+
+if __name__ == "__main__":
+    main()

shield_norms.py

@@ -0,0 +1,113 @@
+# python shield_norms.py "B:\12B\models--p-e-w--Mistral-Nemo-Instruct-2407-heretic-noslop" "C:\Quanter\model_cache\EldritchLabs__Nocturne-Nereid-12B-v1"
+
+import os
+import json
+import gc
+import re
+from safetensors.torch import load_file, save_file
+import argparse
+
+def get_weight_map(model_path):
+    index_path = os.path.join(model_path, "model.safetensors.index.json")
+    if os.path.exists(index_path):
+        with open(index_path, 'r') as f:
+            return json.load(f)["weight_map"]
+    # Fallback for single file models
+    res = {}
+    for f in os.listdir(model_path):
+        if f.endswith(".safetensors"):
+            # This is slow but accurate for single-file models
+            tensors = load_file(os.path.join(model_path, f))
+            for k in tensors.keys():
+                res[k] = f
+    return res
+
+def shield_norms(base_path, merged_path):
+    print("\n[1] Mapping tensors...")
+    base_map = get_weight_map(base_path)
+    merged_map = get_weight_map(merged_path)
+    
+    # Identify all normalization tensors
+    # Catching: model.norm.weight, input_layernorm.weight, post_attention_layernorm.weight
+    norm_pattern = re.compile(r".*norm\.weight$")
+    target_tensors = [t for t in merged_map.keys() if norm_pattern.match(t)]
+    
+    if not target_tensors:
+        print("  [!] No normalization tensors found!")
+        return
+
+    print(f"  -> Found {len(target_tensors)} normalization tensors to shield.")
+
+    # Group by shard to minimize file opening
+    shards_to_process = {}
+    for t in target_tensors:
+        shard = merged_map[t]
+        if shard not in shards_to_process:
+            shards_to_process[shard] = []
+        shards_to_process[shard].append(t)
+
+    print(f"\n[2] Processing {len(shards_to_process)} shards...")
+
+    for shard_name, tensors in shards_to_process.items():
+        merged_shard_path = os.path.join(merged_path, shard_name)
+        backup_shard_path = merged_shard_path + ".old"
+        
+        print(f"  -> Shard: {shard_name}")
+        
+        # 1. Load the merged shard
+        merged_tensors = load_file(merged_shard_path, device="cpu")
+        
+        # 2. Update each target tensor in this shard
+        for t_name in tensors:
+            if t_name in base_map:
+                base_shard_path = os.path.join(base_path, base_map[t_name])
+                base_data = load_file(base_shard_path, device="cpu")
+                
+                print(f"     Injecting pristine: {t_name}")
+                merged_tensors[t_name] = base_data[t_name].clone()
+                
+                del base_data
+            else:
+                print(f"     [!] Warning: {t_name} not found in base model. Skipping.")
+
+        # 3. Atomic Rename Strategy for Windows
+        if os.path.exists(backup_shard_path):
+            os.remove(backup_shard_path)
+            
+        os.rename(merged_shard_path, backup_shard_path)
+        
+        try:
+            save_file(merged_tensors, merged_shard_path, metadata={"format": "pt"})
+            print(f"     ✅ Shard saved successfully.")
+        except Exception as e:
+            print(f"     ❌ Error saving shard: {e}")
+            os.rename(backup_shard_path, merged_shard_path)
+            raise e
+        
+        # 4. Cleanup and release handles
+        del merged_tensors
+        gc.collect()
+        try:
+            os.remove(backup_shard_path)
+        except Exception as e:
+            print(f"     [!] Note: Could not delete .old file immediately (OS lock). It will be orphaned: {e}")
+
+def main():
+    parser = argparse.ArgumentParser(description="Revert all normalization layers to base model values.")
+    parser.add_argument("base_model", help="Path to the pristine base model")
+    parser.add_argument("merged_model", help="Path to the merged model with artifacts")
+    args = parser.parse_args()
+
+    print("="*60)
+    print("🛡️  NORM SHIELD: RE-CALIBRATING OUTPUT MANIFOLD")
+    print("="*60)
+    
+    shield_norms(args.base_model, args.merged_model)
+    
+    print("\n" + "="*60)
+    print("Done! All normalization layers have been reverted to Base.")
+    print("This should eliminate the 'napad' / 'derrotó' vector drift.")
+    print("="*60)
+
+if __name__ == "__main__":
+    main()

vocab_id_patcher.py

@@ -0,0 +1,127 @@
+import json
+import os
+import argparse
+from colorama import init, Fore, Style
+
+init()
+
+# Updated list to include Dolphin, Noromaid, and Hermes
+TARGET_MODELS = [
+    "dreamgen--WizardLM-2-7B",
+    "KoboldAI--Mistral-7B-Erebus-v3",
+    "teknium--Mistral-Trismegistus-7B",
+    "dolphin-2.2.1-mistral-7b",
+    "Noromaid-7B-0.4-DPO",
+    "Hermes-Trismegistus-Mistral-7B"
+]
+
+def load_json(path):
+    if not os.path.exists(path):
+        return {}
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            return json.load(f)
+    except Exception:
+        return {}
+
+def save_json(path, data):
+    with open(path, 'w', encoding='utf-8') as f:
+        json.dump(data, f, indent=2)
+
+def patch_model(model_path):
+    print(f"Processing: {Fore.YELLOW}{os.path.basename(model_path)}{Style.RESET_ALL}")
+    
+    # 1. Fix Tokenizer Config
+    tok_config_path = os.path.join(model_path, "tokenizer_config.json")
+    if os.path.exists(tok_config_path):
+        data = load_json(tok_config_path)
+        
+        # Force EOS to standard Mistral
+        data["eos_token"] = "</s>"
+        
+        # Ensure model_max_length is set
+        if "model_max_length" not in data or data["model_max_length"] > 100000:
+             data["model_max_length"] = 32768
+             
+        save_json(tok_config_path, data)
+        print(f"  -> Updated tokenizer_config.json (eos_token: </s>)")
+    else:
+        print(f"  {Fore.RED}! tokenizer_config.json missing{Style.RESET_ALL}")
+
+    # 2. Fix Special Tokens Map
+    spec_map_path = os.path.join(model_path, "special_tokens_map.json")
+    if os.path.exists(spec_map_path):
+        data = load_json(spec_map_path)
+        data["eos_token"] = "</s>"
+        save_json(spec_map_path, data)
+        print(f"  -> Updated special_tokens_map.json")
+
+    # 3. Attempt to fix Tokenizer JSON (The source of "MISSING" ID)
+    tok_json_path = os.path.join(model_path, "tokenizer.json")
+    if os.path.exists(tok_json_path):
+        data = load_json(tok_json_path)
+        
+        # Check if it has the standard structure
+        if "model" in data and "vocab" in data["model"]:
+            vocab = data["model"]["vocab"]
+            
+            # Ensure </s> exists and is 2
+            if "</s>" not in vocab:
+                print(f"  {Fore.YELLOW}! '</s>' was missing from vocab. Adding it as ID 2.{Style.RESET_ALL}")
+                vocab["</s>"] = 2
+            elif vocab["</s>"] != 2:
+                print(f"  {Fore.RED}! '</s>' is ID {vocab['</s>']}. Forcing to 2.{Style.RESET_ALL}")
+                vocab["</s>"] = 2
+            
+            # Save back
+            data["model"]["vocab"] = vocab
+            save_json(tok_json_path, data)
+            print(f"  -> Patched tokenizer.json vocab")
+        else:
+            print(f"  {Fore.RED}! tokenizer.json has unexpected structure.{Style.RESET_ALL}")
+    else:
+        print(f"  {Fore.RED}! tokenizer.json not found. (This is why Vocab ID is MISSING){Style.RESET_ALL}")
+        print(f"  {Fore.CYAN}* Attempting to create minimal tokenizer.json wrapper...{Style.RESET_ALL}")
+        # Create a dummy wrapper so Mergekit can find the ID
+        dummy_data = {
+            "version": "1.0",
+            "model": {
+                "type": "BPE",
+                "vocab": {"<unk>": 0, "<s>": 1, "</s>": 2}
+            }
+        }
+        save_json(tok_json_path, dummy_data)
+        print(f"  -> Created minimal tokenizer.json")
+
+def main():
+    parser = argparse.ArgumentParser(description="Patch missing Vocab IDs for specific models")
+    parser.add_argument("base_dir", help="Directory containing the model folders (e.g. B:\\7B)")
+    args = parser.parse_args()
+
+    print(f"{Fore.CYAN}--- VOCAB ID PATCHER ---{Style.RESET_ALL}")
+
+    # Walk through the directory to find the specific models
+    found = 0
+    for root, dirs, files in os.walk(args.base_dir):
+        for name in dirs:
+            # Check if this folder matches one of our targets
+            if any(target in name for target in TARGET_MODELS):
+                full_path = os.path.join(root, name)
+                
+                # CHECK FOR 'fixed' SUBDIRECTORY
+                # If the model files are actually inside a 'fixed' subfolder, we must patch that instead.
+                fixed_sub = os.path.join(full_path, "fixed")
+                if os.path.exists(fixed_sub):
+                    print(f"Found target '{name}' -> Detected 'fixed' subdir. Patching inside.")
+                    patch_model(fixed_sub)
+                else:
+                    patch_model(full_path)
+                    
+                found += 1
+
+    if found == 0:
+        print(f"{Fore.RED}No target models found in {args.base_dir}{Style.RESET_ALL}")
+        print(f"Looking for: {TARGET_MODELS}")
+
+if __name__ == "__main__":
+    main()