Add files using upload-large-folder tool

.gitattributes

@@ -1,35 +1,59 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+data/babylm_data/* filter=lfs diff=lfs merge=lfs -text
+data/Perturbed_data/* filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-2080/rng_state_1.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/rng_state_4.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/training_args.bin filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/rng_state_0.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-300/scheduler.pt filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/rng_state_5.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/rng_state_1.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-2080/rng_state_2.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-2080/rng_state_5.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-2080/training_args.bin filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-2080/scheduler.pt filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-2080/rng_state_4.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-2080/rng_state_3.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1650/training_args.bin filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1650/rng_state_6.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1650/scheduler.pt filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1650/rng_state_3.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1950/scheduler.pt filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-600/rng_state_0.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-600/rng_state_5.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-600/scheduler.pt filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-600/rng_state_2.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1050/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-300/rng_state_3.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-600/rng_state_1.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-300/rng_state_4.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-300/training_args.bin filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-600/training_args.bin filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-300/rng_state_2.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-300/rng_state_5.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-300/rng_state_1.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/rng_state_6.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/scheduler.pt filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-450/rng_state_2.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/rng_state_3.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-450/training_args.bin filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/rng_state_2.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1500/rng_state_2.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-450/rng_state_4.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-450/rng_state_5.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-900/rng_state_0.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-900/rng_state_4.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-900/rng_state_6.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-900/rng_state_1.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-450/scheduler.pt filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-900/rng_state_3.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-450/rng_state_1.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-900/rng_state_2.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-450/rng_state_6.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-900/rng_state_5.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1800/rng_state_5.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-900/scheduler.pt filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1800/training_args.bin filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1800/rng_state_1.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1950/rng_state_0.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-450/rng_state_3.pth filter=lfs diff=lfs merge=lfs -text
+train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1500/rng_state_6.pth filter=lfs diff=lfs merge=lfs -text

gpt2_no_positional_encoding_model.py

@@ -0,0 +1,427 @@
+# gpt2_no_positional_encoding_model.py
+# Adapted from Huggingface's transformers library
+
+import torch
+from transformers.models.gpt2.modeling_gpt2 import GPT2Block, GPT2PreTrainedModel
+from transformers.modeling_outputs import CausalLMOutputWithCrossAttentions, BaseModelOutputWithPastAndCrossAttentions
+from transformers.utils.model_parallel_utils import assert_device_map, get_device_map
+from torch import nn
+from torch.nn import CrossEntropyLoss
+from typing import Optional, Tuple, Union
+
+class GPT2NoPositionalEncodingModel(GPT2PreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.embed_dim = config.hidden_size
+
+        self.wte = nn.Embedding(config.vocab_size, self.embed_dim)
+
+        self.drop = nn.Dropout(config.embd_pdrop)
+        self.h = nn.ModuleList([GPT2Block(config, layer_idx=i) for i in range(config.num_hidden_layers)])
+        self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
+
+        # Model parallel
+        self.model_parallel = False
+        self.device_map = None
+        self.gradient_checkpointing = False
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def parallelize(self, device_map=None):
+        # Check validity of device_map
+        self.device_map = (
+            get_device_map(len(self.h), range(torch.cuda.device_count())) if device_map is None else device_map
+        )
+        assert_device_map(self.device_map, len(self.h))
+        self.model_parallel = True
+        self.first_device = "cpu" if "cpu" in self.device_map.keys() else "cuda:" + str(min(self.device_map.keys()))
+        self.last_device = "cuda:" + str(max(self.device_map.keys()))
+        self.wte = self.wte.to(self.first_device)
+        # Load onto devices
+        for k, v in self.device_map.items():
+            for block in v:
+                cuda_device = "cuda:" + str(k)
+                self.h[block] = self.h[block].to(cuda_device)
+        # ln_f to last
+        self.ln_f = self.ln_f.to(self.last_device)
+
+    def deparallelize(self):
+        self.model_parallel = False
+        self.device_map = None
+        self.first_device = "cpu"
+        self.last_device = "cpu"
+        self.wte = self.wte.to("cpu")
+        for index in range(len(self.h)):
+            self.h[index] = self.h[index].to("cpu")
+        self.ln_f = self.ln_f.to("cpu")
+        torch.cuda.empty_cache()
+
+    def get_input_embeddings(self):
+        return self.wte
+
+    def set_input_embeddings(self, new_embeddings):
+        self.wte = new_embeddings
+
+    def _prune_heads(self, heads_to_prune):
+        """
+        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
+        """
+        for layer, heads in heads_to_prune.items():
+            self.h[layer].attn.prune_heads(heads)
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        token_type_ids: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+            input_ids = input_ids.view(-1, input_shape[-1])
+            batch_size = input_ids.shape[0]
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+            batch_size = inputs_embeds.shape[0]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+
+        if token_type_ids is not None:
+            token_type_ids = token_type_ids.view(-1, input_shape[-1])
+
+        if past_key_values is None:
+            past_length = 0
+            past_key_values = tuple([None] * len(self.h))
+        else:
+            past_length = past_key_values[0][0].size(-2)
+        if position_ids is None:
+            position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device)
+            position_ids = position_ids.unsqueeze(0)
+
+        # GPT2Attention mask.
+        if attention_mask is not None:
+            if batch_size <= 0:
+                raise ValueError("batch_size has to be defined and > 0")
+            attention_mask = attention_mask.view(batch_size, -1)
+            # We create a 3D attention mask from a 2D tensor mask.
+            # Sizes are [batch_size, 1, 1, to_seq_length]
+            # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
+            # this attention mask is more simple than the triangular masking of causal attention
+            # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
+            attention_mask = attention_mask[:, None, None, :]
+
+            # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
+            # masked positions, this operation will create a tensor which is 0.0 for
+            # positions we want to attend and the dtype's smallest value for masked positions.
+            # Since we are adding it to the raw scores before the softmax, this is
+            # effectively the same as removing these entirely.
+            attention_mask = attention_mask.to(dtype=self.dtype)  # fp16 compatibility
+            attention_mask = (1.0 - attention_mask) * torch.finfo(self.dtype).min
+
+        # If a 2D or 3D attention mask is provided for the cross-attention
+        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
+        if self.config.add_cross_attention and encoder_hidden_states is not None:
+            encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
+            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
+            if encoder_attention_mask is None:
+                encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
+            encoder_attention_mask = self.invert_attention_mask(encoder_attention_mask)
+        else:
+            encoder_attention_mask = None
+
+        # Prepare head mask if needed
+        # 1.0 in head_mask indicate we keep the head
+        # attention_probs has shape bsz x n_heads x N x N
+        # head_mask has shape n_layer x batch x n_heads x N x N
+        head_mask = self.get_head_mask(head_mask, self.config.n_layer)
+
+        if inputs_embeds is None:
+            inputs_embeds = self.wte(input_ids)
+        hidden_states = inputs_embeds
+
+        if token_type_ids is not None:
+            token_type_embeds = self.wte(token_type_ids)
+            hidden_states = hidden_states + token_type_embeds
+
+        hidden_states = self.drop(hidden_states)
+
+        output_shape = (-1,) + input_shape[1:] + (hidden_states.size(-1),)
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                use_cache = False
+
+        presents = () if use_cache else None
+        all_self_attentions = () if output_attentions else None
+        all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None
+        all_hidden_states = () if output_hidden_states else None
+        for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
+            # Model parallel
+            if self.model_parallel:
+                torch.cuda.set_device(hidden_states.device)
+                # Ensure layer_past is on same device as hidden_states (might not be correct)
+                if layer_past is not None:
+                    layer_past = tuple(past_state.to(hidden_states.device) for past_state in layer_past)
+                # Ensure that attention_mask is always on the same device as hidden_states
+                if attention_mask is not None:
+                    attention_mask = attention_mask.to(hidden_states.device)
+                if isinstance(head_mask, torch.Tensor):
+                    head_mask = head_mask.to(hidden_states.device)
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            if self.gradient_checkpointing and self.training:
+                outputs = self._gradient_checkpointing_func(
+                    block.__call__,
+                    hidden_states,
+                    None,
+                    attention_mask,
+                    head_mask[i],
+                    encoder_hidden_states,
+                    encoder_attention_mask,
+                    use_cache,
+                    output_attentions,
+                )
+            else:
+                outputs = block(
+                    hidden_states,
+                    layer_past=layer_past,
+                    attention_mask=attention_mask,
+                    head_mask=head_mask[i],
+                    encoder_hidden_states=encoder_hidden_states,
+                    encoder_attention_mask=encoder_attention_mask,
+                    use_cache=use_cache,
+                    output_attentions=output_attentions,
+                )
+
+            hidden_states = outputs[0]
+            if use_cache is True:
+                presents = presents + (outputs[1],)
+
+            if output_attentions:
+                all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)
+                if self.config.add_cross_attention:
+                    all_cross_attentions = all_cross_attentions + (outputs[3 if use_cache else 2],)
+
+            # Model Parallel: If it's the last layer for that device, put things on the next device
+            if self.model_parallel:
+                for k, v in self.device_map.items():
+                    if i == v[-1] and "cuda:" + str(k) != self.last_device:
+                        hidden_states = hidden_states.to("cuda:" + str(k + 1))
+
+        hidden_states = self.ln_f(hidden_states)
+
+        hidden_states = hidden_states.view(output_shape)
+        # Add last hidden state
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [hidden_states, presents, all_hidden_states, all_self_attentions, all_cross_attentions]
+                if v is not None
+            )
+
+        return BaseModelOutputWithPastAndCrossAttentions(
+            last_hidden_state=hidden_states,
+            past_key_values=presents,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions,
+            cross_attentions=all_cross_attentions,
+        )
+
+class GPT2NoPositionalEncodingLMHeadModel(GPT2PreTrainedModel):
+    _tied_weights_keys = ["lm_head.weight"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.transformer = GPT2NoPositionalEncodingModel(config)
+        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
+
+        # Model parallel
+        self.model_parallel = False
+        self.device_map = None
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def parallelize(self, device_map=None):
+        self.device_map = (
+            get_device_map(len(self.transformer.h), range(torch.cuda.device_count()))
+            if device_map is None
+            else device_map
+        )
+        assert_device_map(self.device_map, len(self.transformer.h))
+        self.transformer.parallelize(self.device_map)
+        self.lm_head = self.lm_head.to(self.transformer.first_device)
+        self.model_parallel = True
+
+    def deparallelize(self):
+        self.transformer.deparallelize()
+        self.transformer = self.transformer.to("cpu")
+        self.lm_head = self.lm_head.to("cpu")
+        self.model_parallel = False
+        torch.cuda.empty_cache()
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs):
+        token_type_ids = kwargs.get("token_type_ids", None)
+        # Omit tokens covered by past_key_values
+        if past_key_values:
+            past_length = past_key_values[0][0].shape[2]
+
+            # Some generation methods already pass only the last input ID
+            if input_ids.shape[1] > past_length:
+                remove_prefix_length = past_length
+            else:
+                # Default to old behavior: keep only final ID
+                remove_prefix_length = input_ids.shape[1] - 1
+
+            input_ids = input_ids[:, remove_prefix_length:]
+            if token_type_ids is not None:
+                token_type_ids = token_type_ids[:, -input_ids.shape[1] :]
+
+        attention_mask = kwargs.get("attention_mask", None)
+        position_ids = kwargs.get("position_ids", None)
+
+        if attention_mask is not None and position_ids is None:
+            # create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -input_ids.shape[1] :]
+        else:
+            position_ids = None
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "position_ids": position_ids,
+                "attention_mask": attention_mask,
+                "token_type_ids": token_type_ids,
+            }
+        )
+
+        return model_inputs
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        token_type_ids: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
+            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
+            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        transformer_outputs = self.transformer(
+            input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        hidden_states = transformer_outputs[0]
+
+        # Set device for model parallelism
+        if self.model_parallel:
+            torch.cuda.set_device(self.transformer.first_device)
+            hidden_states = hidden_states.to(self.lm_head.weight.device)
+
+        lm_logits = self.lm_head(hidden_states)
+
+        loss = None
+        if labels is not None:
+            # move labels to correct device to enable model parallelism
+            labels = labels.to(lm_logits.device)
+            # Shift so that tokens < n predict n
+            shift_logits = lm_logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+
+        if not return_dict:
+            output = (lm_logits,) + transformer_outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutputWithCrossAttentions(
+            loss=loss,
+            logits=lm_logits,
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+            cross_attentions=transformer_outputs.cross_attentions,
+        )
+
+    @staticmethod
+    def _reorder_cache(
+        past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor
+    ) -> Tuple[Tuple[torch.Tensor]]:
+        """
+        This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or
+        [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
+        beam_idx at every generation step.
+        """
+        return tuple(
+            tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past)
+            for layer_past in past_key_values
+        )

hop_surprisal/hop_surprisal.ipynb

@@ -0,0 +1,223 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# For importing utils\n",
+    "import sys\n",
+    "sys.path.append(\"..\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/nlp/scr/kallini/miniconda3/envs/llmenv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
+    "from matplotlib.patches import Patch\n",
+    "from scipy import stats\n",
+    "from utils import PERTURBATIONS"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_surprisal_differences(perturbation, seed, ckpt, pos_encodings=True):\n",
+    "\n",
+    "    # Load surprisal DataFrame\n",
+    "    surprisals_path = \"hop_surprisal_results/{}_100M{}/randinit_seed{}.csv\"\n",
+    "    nps = \"\" if pos_encodings else \"_no_positional_encodings\"\n",
+    "    surprisal_df = pd.read_csv(surprisals_path.format(perturbation, nps, seed))\n",
+    "    \n",
+    "    # Get summary stats for suprisal differences\n",
+    "    marker_token_surprisals = surprisal_df[f\"Marker Token Surprisals (ckpt {ckpt})\"]\n",
+    "    nomarker_token_surprisals = surprisal_df[f\"No Marker Token Surprisals (ckpt {ckpt})\"]\n",
+    "    differences = nomarker_token_surprisals - marker_token_surprisals\n",
+    "    avg_differences = differences.mean()\n",
+    "\n",
+    "    return avg_differences\n",
+    "\n",
+    "def get_summary_stats(l):\n",
+    "    # Calculate confidence interval using t-distribution\n",
+    "    mean = np.mean(l)\n",
+    "    sem = stats.sem(l)\n",
+    "    ci_lower, ci_upper = stats.t.interval(0.95, df=len(l)-1, loc=mean, scale=sem)\n",
+    "    return mean, (ci_upper - ci_lower) / 2\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def plot_surprisal_differences(ax, seeds, ckpt, colors, hatches, pos_encodings):\n",
+    "\n",
+    "    hop_control_seeds = []\n",
+    "    hop_tokens4_seeds = []\n",
+    "    hop_words4_seeds = []\n",
+    "    for seed in seeds:\n",
+    "        # Get summary stats for each hop model\n",
+    "        avg_hop_control = get_surprisal_differences(\"hop_control\", seed, ckpt, pos_encodings)\n",
+    "        avg_hop_tokens4 = get_surprisal_differences(\"hop_tokens4\", seed, ckpt, pos_encodings)\n",
+    "        avg_hop_words4 = get_surprisal_differences(\"hop_words4\", seed, ckpt, pos_encodings)\n",
+    "\n",
+    "        # Append results\n",
+    "        hop_control_seeds.append(avg_hop_control)\n",
+    "        hop_tokens4_seeds.append(avg_hop_tokens4)\n",
+    "        hop_words4_seeds.append(avg_hop_words4)\n",
+    "\n",
+    "    if len(seeds) > 1:\n",
+    "        # Prepare data to plot\n",
+    "        summary_stats = [\n",
+    "            get_summary_stats(hop_control_seeds),\n",
+    "            get_summary_stats(hop_tokens4_seeds),\n",
+    "            get_summary_stats(hop_words4_seeds)\n",
+    "        ]\n",
+    "    else:\n",
+    "        summary_stats = [\n",
+    "            (hop_control_seeds[0], 0),\n",
+    "            (hop_tokens4_seeds[0], 0),\n",
+    "            (hop_words4_seeds[0], 0),\n",
+    "        ]\n",
+    "\n",
+    "    x = np.arange(3)  # label locations\n",
+    "    width = 0.8  # width of the bars\n",
+    "\n",
+    "    # Iterate over pos / no pos groups\n",
+    "    for i, (avg, err) in enumerate(summary_stats):\n",
+    "        # Iterate over models and plot bars\n",
+    "        color = colors[i]\n",
+    "        hatch = hatches[i]\n",
+    "        ax.bar(x[i], avg, width, yerr=err, label=None,\n",
+    "                color=color, hatch=hatch, edgecolor=\"w\", zorder=2)\n",
+    "\n",
+    "    ax.set_xticks([])\n",
+    "    ax.grid(zorder=0, color=\"lightgray\")\n",
+    "    ax.set_title(f\"{ckpt} Steps\")\n",
+    "\n",
+    "\n",
+    "def plot_surprisal_differences_checkpoints(seeds, checkpoints, pos_encodings=True):\n",
+    "\n",
+    "    # Colors patterns for bars\n",
+    "    color1=PERTURBATIONS[\"hop_control\"][\"color\"]\n",
+    "    color2=PERTURBATIONS[\"hop_tokens4\"][\"color\"]\n",
+    "    color3=PERTURBATIONS[\"hop_words4\"][\"color\"]\n",
+    "    colors = [color1, color2, color3]\n",
+    "\n",
+    "    hatch1 = ''\n",
+    "    hatch2 = '///'\n",
+    "    hatch3 = '..'\n",
+    "    hatches = [hatch1, hatch2, hatch3]\n",
+    "\n",
+    "    # Create a figure with multiple subplots\n",
+    "    fig, axs = plt.subplots(2, 3, figsize=(6, 4), sharey=True)\n",
+    "    axes_flat = axs.flatten()\n",
+    "\n",
+    "    # Call individual plot function with different parameters for each subplot\n",
+    "    for i, checkpoint in enumerate(checkpoints):\n",
+    "        plot_surprisal_differences(\n",
+    "            axes_flat[i], seeds, checkpoint, colors, hatches, pos_encodings)\n",
+    "\n",
+    "    legend_elements = [Patch(facecolor=color1, hatch=hatch1,\n",
+    "                             edgecolor=\"w\", label='NoHop'),\n",
+    "                       Patch(facecolor=color2, hatch=hatch2,\n",
+    "                             edgecolor=\"w\", label='TokenHop'),\n",
+    "                        Patch(facecolor=color3, hatch=hatch3,\n",
+    "                              edgecolor=\"w\", label='WordHop')]\n",
+    "    fig.legend(handles=legend_elements, ncol=3, loc=\"center\",\n",
+    "                bbox_to_anchor=(0.55, 0), frameon=False)\n",
+    "        \n",
+    "    fig.supylabel(\"Surprisal Difference\", fontsize=12, x=0.04)\n",
+    "\n",
+    "    # Adjust layout and show plot\n",
+    "    plt.tight_layout()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkUAAAGhCAYAAABvQ8DIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB/DUlEQVR4nO3dd1hT59sH8O9JgLCXbAVliaMOpG4FcSHuXfeordZaq/VVq60DV13Vqq2L1p9oa62r1Vr33ntWRRTFjVu2MpLn/SMlGjNITgKcJPfnunJJznlynufInfDNmRxjjIEQQgghxMKJSnsAhBBCCCFCQKGIEEIIIQQUigghhBBCAFAoIoQQQggBQKGIEEIIIQQAhSJCCCGEEAAUigghhBBCAFAoIoQQQggBQKGIEEIIIQQAhSJCCCGEEAAUigTl6tWr6NatG4KCgmBvbw8PDw9ERkZi69atatsnJiaiVatWcHR0hLu7O/r27Ytnz56ptJPJZJgzZw4CAwNha2uL6tWrY+3atTqP6+jRo4iNjUXZsmVha2uLgIAAtGvXDr///ruiTU5ODuLi4nDw4EG915uYn/Pnz6N9+/Zwd3eHvb09PvjgAyxatEil3fHjx9GoUSPY29vDx8cHX375JbKyslTa5ebm4uuvv4afnx/s7OxQt25d7NmzR+fxbN26FVFRUfDy8oK9vT2CgoLQvXt37Ny5U9Hm0aNHiIuLw8WLF3mtMzEP586dQ6tWreDs7AwnJye0bNlSY01Q/ZohRgRj27ZtLCYmhsXFxbH4+Hi2YMEC1rhxYwaALV++XKnt/fv3mYeHBwsODmYLFy5kM2bMYG5ubqxGjRosNzdXqe24ceMYAPbpp5+y+Ph41qZNGwaArV27tsgxrV+/nnEcx8LDw9ns2bNZfHw8Gz9+PGvYsCFr0qSJot2zZ88YADZ58mSj/F8Q07Vr1y5mY2PD6taty+bPn8/i4+PZ119/zcaMGaPU7sKFC8zW1paFh4ezpUuXsm+//ZZJJBLWqlUrlWX26NGDWVlZsdGjR7Ply5ez+vXrMysrK3bkyJEixzN37lwGgEVFRbH58+ezZcuWsdGjR7OaNWuy/v37K9qdOXOGAWArV6409L+AmKhz584xW1tbFhoayr7//ns2Z84cVqFCBebs7MyuX7+u1Jbq1zxRKBK4goICVqNGDRYWFqY0fejQoczOzo7dvXtXMW3Pnj0qAerBgwfM2tqaDRs2TDFNJpOxxo0bs3LlyrGCggKt/VepUoVVrVpVJWgxxtiTJ08UP1MoIowxlp6ezry9vVmnTp2YVCrV2jY2Npb5+vqy9PR0xbSff/6ZAWC7du1STDt16hQDwObOnauY9vr1axYcHMzq16+vtY/8/Hzm7OzMWrRooXb+uzVMf1RI69atmZubG3v+/Lli2qNHj5ijoyPr3LmzUluqX/NEocgEtG3blnl7eytN8/LyYt26dVNpW7FiRdasWTPF88WLFzMA7OrVq0rtfv/9dwagyG8qEomEDRgwQGublJQUBkDl8W5ASkxMZF26dGFubm5MIpGwiIgItmXLFqXlrFy5kgFghw4dYoMHD2bu7u7MycmJ9e3bl718+VKp7ZkzZ1jLli1ZmTJlmK2tLatQoQIbOHCg1nGS4rd06VIGgF27do0xxlhWVpbacJSens6srKxUth7l5uYyR0dHNmjQIMW0MWPGMLFYrPTHhzHGvvvuOwaA3bt3T+N4UlNTGQAWFxenddwHDhxQW8Pv/oE5efIki4mJYc7OzszOzo5FRkayo0ePKi1n8uTJDABLTExk3bp1Y05OTszd3Z19+eWX7PXr10ptd+/ezRo2bMhcXFyYg4MDq1ixIhs/frzWcZLi5eTkpPZztU2bNszGxoZlZmYyxqh+GTPf+qVjigQoOzsbz58/x61bt/DDDz9gx44daNasmWL+w4cP8fTpU3z44Ycqr61Tpw4uXLigeH7hwgU4ODigcuXKKu0K52tTvnx57Nu3Dw8ePNDYxtPTE0uXLgUAdOrUCb/++it+/fVXdO7cGYD8WKl69eohMTER48aNw7x58+Dg4ICOHTvir7/+UlneF198gcTERMTFxaFfv35Ys2YNOnbsCMYYAODp06do2bIl7ty5g3HjxuHHH39E7969cfLkSa3rQorf3r174ezsjIcPHyIsLAyOjo5wdnbG0KFD8ebNG0W7f//9FwUFBSo1bGNjg5o1a6rUcMWKFeHs7KzUtrCGtR1D4eXlBTs7O2zduhUvX77U2K5y5cqYOnUqAGDw4MGKGo6MjAQA7N+/H5GRkcjIyMDkyZPx3XffIS0tDU2bNsXp06dVlte9e3e8efMGM2fOROvWrbFo0SIMHjxYMf/q1ato27YtcnNzMXXqVMybNw/t27fHsWPHNI6RFL/c3FzY2dmpTLe3t0deXh6uXLkCgOrXrOu3tFMZUTVkyBBF0heJRKxr165KW0oKN5OuXr1a5bVjxoxhANibN28YY/JvOEFBQSrtsrOzGQA2btw4rWNZsWIFA8BsbGxYdHQ0mzhxIjty5IjKt39tu8+aNWvGqlWrphgTY/JdeA0aNGChoaGKaYVbiiIiIlheXp5i+pw5cxgAxZalv/76iwFgZ86c0Tp2UvKqV6/O7O3tmb29PRs+fDjbtGkTGz58OAPAevTooWi3YcMGBoAdPnxYZRndunVjPj4+iudVq1ZlTZs2VWl39epVBoAtW7ZM65gmTZrEADAHBwcWGxvLZsyYwc6dO6fSTtPuB5lMxkJDQ1lMTAyTyWSK6Tk5OSwwMFBp10bhN+327dsrLePzzz9nANilS5cYY4z98MMPDAB79uyZ1rGTklWtWjVWsWJFpcMKcnNzWUBAAAPANm7cyBij+jXn+qUtRQI0cuRI7NmzB6tWrUJsbCykUiny8vIU81+/fg0AkEgkKq+1tbVVavP69Wud2mny8ccfY+fOnWjSpAmOHj2KadOmoXHjxggNDcXx48eLXJeXL19i//796N69OzIzM/H8+XM8f/4cL168QExMDG7evImHDx8qvWbw4MGwtrZWPB86dCisrKywfft2AICrqysA4J9//kF+fn6RYyAlJysrCzk5OejXrx8WLVqEzp07Y9GiRRgyZAj++OMP3Lx5E0DRNfxuXRpaw1OmTMHvv/+O8PBw7Nq1C99++y0iIiJQq1YtJCYmFrlOFy9exM2bN9GrVy+8ePFCUcPZ2dlo1qwZDh8+DJlMpvSaYcOGKT0fPnw4AKjU8JYtW1ReS0rP559/jhs3bmDQoEG4du0arly5gn79+iE1NRWA8ucqQPVrjvVLoUiAKlWqhObNm6Nfv374559/kJWVhXbt2il2HxVu3s3NzVV5beEuisI2dnZ2OrXTJiYmBrt27UJaWhoOHz6MYcOG4e7du2jbti2ePn2q9bXJyclgjGHixInw9PRUekyePBkAVJYRGhqq9NzR0RG+vr64c+cOACAqKgpdunTBlClT4OHhgQ4dOmDlypVq15OUrMJ66tmzp9L0Xr16AQBOnDih1E5Tbb5bl8ao4Z49e+LIkSN49eoVdu/ejV69euHChQto166d0m49dQqDXP/+/VVq+JdffkFubi7S09OVXvN+DQcHB0MkEilq+KOPPkLDhg3xySefwNvbGz169MD69evN7g+Mqfnss8/wzTff4Pfff0fVqlVRrVo13Lp1C2PHjgUg/ywCqH7NuX6tSnsApGhdu3bFkCFDcOPGDYSFhcHX1xcAFN9e3pWamgp3d3fFNxNfX18cOHAAjDFwHKfUDgD8/Px0Hoe9vT0aN26Mxo0bw8PDA1OmTMGOHTvQv39/ja8pfJOMHj0aMTExatuEhIToPAYA4DgOGzduxMmTJ7F161bs2rULH3/8MebNm4eTJ08qPrhIyfPz88PVq1fh7e2tNN3LywsA8OrVKwAosobfrUtfX1+VrYnvvlafGnZ2dkaLFi3QokULWFtbY9WqVTh16hSioqI0vqawhufOnYuaNWuqbVNUzb373gPkfwgPHz6MAwcOYNu2bdi5cyfWrVuHpk2bYvfu3RCLxTqvEzGuGTNmYPTo0bh69SpcXFxQrVo1fPPNNwCAihUrAqD6Nef6pS1FJqBw82phmi9btiw8PT1x9uxZlbanT59WKvyaNWsiJydHZTPrqVOnFPP5KDzAsPCN/f6bplBQUBAAwNraGs2bN1f7cHJyUnpN4TebQllZWUhNTUWFChWUpterVw8zZszA2bNnsWbNGly9ehV//PEHr/UhxhEREQEAKn8EHj16BEB+UD4AfPDBB7CyslKp4by8PFy8eFGlhm/cuIGMjAyltiVVw8HBwQDkf5A01fC7u3sB1RpOTk6GTCZTqmGRSIRmzZph/vz5uHbtGmbMmIH9+/fjwIEDvNaHGI+bmxsaNWqEatWqAZCfQFCuXDlUqlQJANUvYL71S6FIQNTtisrPz8fq1athZ2eHKlWqKKZ36dIF//zzD+7fv6+Ytm/fPty4cQPdunVTTOvQoQOsra2xZMkSxTTGGJYtW4ayZcuiQYMGWse0b98+tdML9y2HhYUBkG9FAoC0tDSldl5eXmjSpAmWL1+u9luVuitwx8fHKx0rtHTpUhQUFCA2NhaAfGtD4a7EQoUfLLQLrXR1794dALBixQql6b/88gusrKzQpEkTAICLiwuaN2+O3377DZmZmYp2v/76K7KyspRquGvXrpBKpYiPj1dMy83NxcqVK1G3bl34+/trHE9OTo5il937duzYAeBtDTs4OABQreGIiAgEBwfj+++/V3u1YnU1vHjxYqXnP/74IwAoaljdmURUw8K0bt06nDlzBiNHjoRIJP+TSfVrvvVLu88EZMiQIcjIyEBkZCTKli2Lx48fY82aNbh+/TrmzZuntInzm2++wYYNGxAdHY0RI0YgKysLc+fORbVq1TBw4EBFu3LlymHkyJGYO3cu8vPzUbt2bWzevBlHjhzBmjVritzM2aFDBwQGBqJdu3YIDg5GdnY29u7di61bt6J27dpo164dAChC27p161CxYkW4u7vjgw8+wAcffIDFixcrvnV9+umnCAoKwpMnT3DixAk8ePAAly5dUuozLy8PzZo1Q/fu3ZGUlIQlS5agUaNGaN++PQBg1apVWLJkCTp16oTg4GBkZmbi559/hrOzM1q3bm2sXwfhITw8HB9//DH+97//oaCgAFFRUTh48CA2bNiA8ePHK+0qmDFjBho0aICoqCgMHjwYDx48wLx589CyZUu0atVK0a5u3bro1q0bxo8fj6dPnyIkJASrVq3CnTt3VMLX+3JyctCgQQPUq1cPrVq1gr+/P9LS0hTvgY4dOyI8PByA/Bu1q6srli1bBicnJzg4OKBu3boIDAzEL7/8gtjYWFStWhUDBw5E2bJl8fDhQxw4cADOzs4qt+JJSUlB+/bt0apVK5w4cQK//fYbevXqhRo1agAApk6disOHD6NNmzYoX748nj59iiVLlqBcuXJo1KiRsX4dRE+HDx/G1KlT0bJlS5QpUwYnT57EypUr0apVK4wYMUKpLdWvmdZvqZ77RpSsXbuWNW/enHl7ezMrKyvm5ubGmjdvrnKRw0JXrlxhLVu2ZPb29szV1ZX17t2bPX78WKWdVCpl3333HStfvjyzsbFhVatWZb/99pvOY+rRowcLDg5mdnZ2zNbWllWpUoV9++23LCMjQ6nt8ePHWUREBLOxsVE5Pf/WrVusX79+zMfHh1lbW7OyZcuytm3bKk5xZUz14o1ubm7M0dGR9e7dm7148ULR7vz586xnz54sICCASSQS5uXlxdq2bcvOnj2r0zqR4pWXl8fi4uJY+fLlmbW1NQsJCWE//PCD2rZHjhxhDRo0YLa2tszT05MNGzZMpa4Yk18BePTo0czHx4dJJBJWu3ZttnPnziLHkp+fz37++WfWsWNHVr58eSaRSJi9vT0LDw9nc+fOVblS+5YtW1iVKlWYlZWVyunNFy5cYJ07d2ZlypRhEomElS9fnnXv3p3t27dP0abwlOZr166xrl27MicnJ+bm5sa++OILpYvf7du3j3Xo0IH5+fkxGxsb5ufnx3r27Mlu3LhR5DqR4pOcnMxatmzJPDw8mEQiYZUqVWIzZ85Ue0V/xqh+zbF+Ocbe2w9BSClJSEjAwIEDcebMGbUXpiRE6OLi4jBlyhQ8e/YMHh4epT0cQvRC9UvHFBFCCCGEAKBQRAghhBACgEIRIYQQQggAgI4pIoQQQggBbSkihBBCCAFAoYgQQgghBIAFXrxRJpPh0aNHcHJy0nhZdEKMiTGGzMxM+Pn5Ka6IawiqYVKSqH6JqdOnhi0uFD169EjrZdUJKS73799HuXLlDF4O1TApDVS/xNTpUsMWF4oKbz56//59ODs7l/JoiCXIyMiAv7+/yo1v+aIaJiWJ6peYOn1q2OJCUeHmWmdnZ3pDkhJlrF0FVMOkNFD9ElOnSw3TgdaEEEIIIaBQRAghhBACgEIRIYQQQggACkWEEEIIIQAoFBFCCCGEAKBQRAghhBACgEIRIYQQQggACkWEEEIIIQAoFBFCCCGEAKBQRAghhBACgEIRIYQQQggACkWEEEIIIQAEGIoOHz6Mdu3awc/PDxzHYfPmzUrzBwwYAI7jlB6tWrUqncESQgghxGwILhRlZ2ejRo0aWLx4scY2rVq1QmpqquKxdu3aEhwhIYQQQsyRVWkP4H2xsbGIjY3V2kYikcDHx6eERkQIIcQUZWdnw9HREQCQlZUFBweHUh4RETrBhSJdHDx4EF5eXnBzc0PTpk0xffp0lClTRm3b3Nxc5ObmKp5nZGQAAKRSKaRSaYmMl1g2Q+uMapiUJlOu33eXT+8Xy6XP793kQlGrVq3QuXNnBAYG4tatW/jmm28QGxuLEydOQCwWq7SfOXMmpkyZojI9KSlJ8Q2CkOKUlZVl0OuphklpKun6tba2VvtZzkdOTo7i5zt37sDe3t4oy5VKpcjPzzfKskjx06eGOcYYK8axGITjOPz111/o2LGjxja3b99GcHAw9u7di2bNmqnMV/ctxd/fHy9fvoSzs3NxDJsQJRkZGXB3d0d6ejqvmqMaJqWppOuX4ziIRMY53LW4dp/JZDII+E8neY8+NWxyW4reFxQUBA8PDyQnJ6sNRRKJBBKJRGW6WCw22rcRQrQxtM6ohklpKo36Xb58OVJTUw3qFwDy8vIUP8+YMQM2NjYGL9PX1xdDhgwxeDmk5OhTwyYfih48eIAXL17A19e3tIdCiFmhg1RJaUlNTcXdu3cNXs67u7ju3bsHa2trg5dJzJvgQlFWVhaSk5MVz1NSUnDx4kW4u7vD3d0dU6ZMQZcuXeDj44Nbt25h7NixCAkJQUxMTCmOmhBCiNBYW1tjwIABpT0MYkIEF4rOnj2L6OhoxfNRo0YBAPr374+lS5fi8uXLWLVqFdLS0uDn54eWLVti2rRpajfPEkIIIYToSnChqEmTJloPYNu1a1cJjoYQ0yOTyYx2oGpxMYUxEkIsj+BCESHEMCKRyCgHqhbHQaoAHahKCBEuCkWEmCFjHahaeDyGMc4EIoQQoaPt14QQQgghoFBECCGEEAKAQhEhhBBCCAAKRYQQQgghACgUEUIIIYQAoFBECCGEEAKAQhEhhBBCCAAKRYQQQgghACgUEUIIIYQAoFBECCGEEAKAQhEhhBBCCAAKRYQQQgghACgUEUIIIYQAoFBECCGEEAKAQhEhhBBCCAAKRYQQQgghACgUEUIsQHZ2NjiOA8dxyM7OLu3hEEIEikIRIYQQQggoFBFCCCGEAKBQRAghhBACgEIRIYQQQggACkWEEEIIIQAoFBFCCCGEAKBQRAghhBACgEIRIYQQQggACkWEEEKI4NEFSEsGhSJCCCGEEFAoIoQQQggBQKGIECJgTCYt7SEUyRTGSAjRjVVpD4AQQjThRGIU/DkWXKVmEFVuAenhpWBJ+7W+RhTeBeLavSA98ztkFzYBAPJzCxTz8//XC/kS+UcfF9YU4sihkCXugexovPaxBNSCuPlYsPvnId03H5AVgPMIglXnuQauJSFEKAQXig4fPoy5c+fi3LlzSE1NxV9//YWOHTsq5jPGMHnyZPz8889IS0tDw4YNsXTpUoSGhpbeoAkhxUYRiLZOBLv4p9a2osafyQPRgYWQHVn2dkae7O3PT64DNiJwNTvLA9G59ZBtnwqAaR5DaBTEzceAJR+CdOMoQJYPaH0FIcQUCW73WXZ2NmrUqIHFixernT9nzhwsWrQIy5Ytw6lTp+Dg4ICYmBi8efOmhEdKCCkJegWi6BGqgUgNrmZniNtN0z0QdVsElnxYKRARQsyP4LYUxcbGIjY2Vu08xhgWLFiACRMmoEOHDgCA1atXw9vbG5s3b0aPHj1UXpObm4vc3FzF84yMDACAVCqFVErHApDiZ2id6VvDYrHYoP5Kii7/L2KxWL7LzJiBqHp7owcic/4sofpVr6R/5+/2R3+/9KPP/5XgQpE2KSkpePz4MZo3b66Y5uLigrp16+LEiRNqQ9HMmTMxZcoUlelJSUlwdHQs1vESAgBZWVkGvV6fGra1tUVISIhB/ZWUlJQUrVt4C9elyGOI9AhEACBuPcnoW4iKWhdTRvWrXkn/znNychQ/X79+Hfb29iXWt6nTp4ZNKhQ9fvwYAODt7a003dvbWzHvfePHj8eoUaMUzzMyMuDv74+wsDA4OzsX32Ah3xXo4uICAEhPT4eDg0Ox9keEqfCbMV+lWcPFKTAw0OBl6BuIAEB2YRNkB+bAmLvMjLEuQkX1q15J/87fvWBjpUqV6O+JHvSpYZMKRXxIJBJIJBKV6WKxuNg30767/JLojwiTob/30qzh4mTo2PXdZQZ8DwCQ7ZoF2HCa2+oRiDhP+VYNU/49FIXqVz1dxs5kUnAi46xjcf09MeYYhUqf/yuTCkU+Pj4AgCdPnsDX11cx/cmTJ6hZs2YpjYoQUtL0Pqi66TgUhiL5FiL1oUivQORXDeLWk/itALEI8ktKjAF7flt1psgK4majwPnXgnTvHLB757UuS1qrn+Lndy8roY66y1KoHR9dUkKFSYWiwMBA+Pj4YN++fYoQlJGRgVOnTmHo0KGlOzhCSIngdZbZ0dVFLlfvQNRnBdire+B8Kuu9DsRysOe3gcfXlCeKrCHuOh+cfzikG4aD3TykZQkcRK0nQVSp6dtJ/11WQh2Nl6VQJ6yp9vkWSHChKCsrC8nJyYrnKSkpuHjxItzd3REQEICRI0di+vTpCA0NRWBgICZOnAg/Pz+laxkRQswT79Pud83S3pZPIHp2E9K98yAauIbXuhALVRiIQiIh3fClboEoojukG74uetH6vj8iaWPC+wQXis6ePYvo6GjF88ID9Pr374+EhASMHTsW2dnZGDx4MNLS0tCoUSPs3LkTtra2pTVkQkgJEMJ1iJQC0ZpPAfcKPNeGWCS+gWjrRLDLf2tfNJ/3R+IeiKvE8FgR8yW4UNSkSRMwpuUDjOMwdepUTJ06tQRHRQgpTYYGIgcbEfKmVFJta0ggysvR2JYQFYYEouK6cOn5DRSK3iO4K1qXNplMVnSjUmYKYyTEWEThXYS3hYgCEdGHEAPRdtqwoI7gthSVNpFIhOXLlyM1NdXgZeXl5Sl+njFjBmxsbAxepq+vL4YMGWLwcggxFboeNEqBiAiSyEqggYju3KcOhSI1UlNTcffuXYOXk5//9oP23r17sLa2NniZhFga6ZnfKRARkyU/7T6cApGJoFBECBE0bddZASgQEWHj/Gvpftq9lkD07nFxFIiKDx1TRAgxWRSIiNBJ986hLUQmhEIRIcQkCSIQufhqnkcIUMSVqikQCQ2FIkKIyRFEIHKvAKs2qnd/J0Q3FIiEiEIRIcSkCCYQ9UsA8mm3GuFDGIGIC6jFZ/BmjQ60LkbW1tYYMGBAaQ+DELMhqECUm4mCnTNh3XcF/xUiFkgggSg0CuLmY/msgFmjLUWEEJMguEC0eiDwOo33+hBLJKBA1G0R2H1txztZJgpFhBDBE2Qgyn7Of4WIBRJYIEo+DOm++XxWxKxRKCKECBoX1pQCETFxAgxEG0cBsgI+K2PWKBQRQgRNHDmUAhExacIMRJrfH5aMdyjKyMjArFmzEBMTg/DwcJw+fRoA8PLlS8yfPx/JyclGGyQhxHLJEvdQICImS9RoMAUiE8Lr7LMHDx4gKioK9+/fR2hoKK5fv46srCwAgLu7O5YvX467d+9i4cKFRh0sIcTyyI7GgwIRMVWiyi0oEJkQXqFozJgxyMzMxMWLF+Hl5QUvLy+l+R07dsQ///xjlAESQogmFIiI0EkPL6VAZEJ47T7bvXs3vvzyS1SpUgUcx6nMDwoKwv379w0eHCGEaCKIQGTnynv8xDKwpP1a51MgEhZeoej169fw9PTUOD8zM5P3gAghpCiCCEQOHrBqG8d7HQihQCQ8vEJRlSpVcPjwYY3zN2/ejPDwcN6DIoQQTQQTiPqtBKzt+a8IsWhCCEScZwjf4ZstXqFo5MiR+OOPPzB79mykp6cDAGQyGZKTk9G3b1+cOHECX331lVEHSgghggpEEicUbJvMf2WIxRJEIPKrBnHrSXxXwWzxOtC6T58+uHv3LiZMmIBvv/0WANCqVSswxiASifDdd9+hY8eOxhwnIcTCCS4QrR4A2NCWIqIfwQSiPivAXt0D51OZ76qYJd43hP3222/Rt29fbNq0CcnJyZDJZAgODkbnzp0RFBRkzDESQiycIAPRyzuATxW+q0QskKAC0bObkO6dB9HANXxXxyzxDkUAEBAQQLvJCCHFiguoBXHzMcILRKUkOzsbjo6OAICsrCw4ODiU2liI7gQXiNZ8CrhX4Lk25ovXMUXnz5/HkiVLNM5fsmQJLl68yHdMhBCiIG4+lgIRMWmCDETa3h8WjFco+vbbb7F3716N8/fv348JEybwHhQhhBRi989TICImSxTehQKRCeEVis6dO4fGjRtrnN+4cWOcPXuW96AIIaSQdN98CkTEZIlr96JA9J7s7GxwHAeO45CdnV3aw1HCKxRlZmbCykrz4UgikUhxqj4hhBhEVqBxFgUiInTSM79TIDIhvEJRaGgodu/erXH+zp076Qw0QkixEkQgsrblOXpiKWQXNmmdT4FIWHiFokGDBmHbtm0YNWoU0tLSFNPT0tLw1VdfYefOnRg0aJCxxkgIIUoEEYhs7CGOpWMnCX8UiISH1yn5X375JS5evIgFCxZg0aJF8PPzAwA8evQIMpkMffv2pVP1CSHFQjCBqPfP4NwCdB63VAaIeX0NLRlCH5+5EUQgcvE1YA3ME69QxHEcVq5ciX79+mHTpk24ffs2AKBDhw7o0qULmjRpYswxEkIIAIEFIs9QSLdPhVWn2TqNXSwC+h4GXubKnz99o76dvRjwtgNSstTPl77zusbbAbGaPXhetvr1UckF+DWy6HUgxiGIQOReAVZtphiwFubJoIs3RkdHIzo62lhj0UlcXBymTFH+RYaFheH69eslOg5CSMkSXCD6bRCYTKrXOjhZASsbyX9uvQfY/1h5vqMVcLEDUMER+DER+Oq06jJmfQBc+O/n5Awg573/hqY+wJZmhvVBio9gAlG/BCA/B0AZncYtZYCY06lpqTHG1k6DQlFpqVq1qtJ1krSdCUcIMX2CDESP/tX7Nh81ywBW/31oN/RWDSz+DvKwAgAt/NQvo5mvcvukXOX5jbwN74MUD0EFotxMFOycCeu+K3Qa+7uBaHkS8MsN5fmBjsD6/7aRpGQC3Q+qLmNDE6CCk/K0d7d2floRGBzGrw9jbe3klSYYY4iPj8eKFStw+/ZtvHr1SqUNx3EoKNB8Kq0hrKys4OPjUyzLJoQIC+cZAnHsBOEFIh623AOquMp/XnFTdX5iunzrTQs/YPw59cuYdOHtz9czAE6iPH/FTaCpr2F9EOMTXCBaPRBw8tJrHS68ALILgBmXgUfvdXPhpXJdXXip+vqvzgAzI4B/kt9Ou/TqbQ3PuPz2/cG3D0PxCkVjx47F/PnzUbNmTfTp0wdubm7GHpdWN2/ehJ+fH2xtbVG/fn3MnDkTAQHqD3jMzc1Fbu7br1IZGRkAAKlUCqlUddO3WCwunkEbmbqxE2Ey9HdlyTUsFoshbj0J7GmSoAORruvy9A3QZKf2dkXtztr+UPv8hzn8+1C3HlS/6un6OwcEGoiynytCka7rMvi49iBSVO1uvS9/sFz18w2p3UKG1jCvULRq1Sp06dIF69ev5/Nyg9StWxcJCQkICwtDamoqpkyZgsaNG+PKlStwcnJSaT9z5kyVY5AAICkpSXFTxUK2trYICQkptrEbU0pKCt680XAUJRGUrCwNR8zqyFJruHBd2Kt7gg5E+qyL0KlbD6pf9XT9nXNhTSGOHCq8QMRjXUyBoTXMKxS9fv0azZs35/NSg8XGxip+rl69OurWrYvy5ctj/fr1aq+NNH78eIwaNUrxPCMjA/7+/ggLC4Ozs3OJjLk4BAYGlvYQiI4KvxnzZek1LN0xXdCBCCjZ9yMncYA4QfMfVkOoWw+qX/V0/Z0LPRDpsy6mwNAa5hWKmjVrhjNnzmDw4MF8Xm5Urq6uqFixIpKTk9XOl0gkkEgkKtPFYrHJbKZVx5THbmkM/V1ZfA3na9kiKoBABJjP+1HdelD9qqfr2GWJewQdiPRZF2MpzmBvaA3zOnltyZIlOHnyJL777ju8ePGCzyKMJisrC7du3YKvL12EihCLIoRAJKIzX4l2sqPxEHIgIsp4haKwsDDcvn0bEydOhJeXFxwcHODs7Kz0cHFxMfZYAQCjR4/GoUOHcOfOHRw/fhydOnWCWCxGz549i6U/IifkuxoTCySIQGQNcbNRmufrwE4MbIwGzrcDwt3VtxlRBbjWUf6vOrXc5a/fGC1fHp8+etKtKkuFKQciodRuYR/GqmFeX3O6dOkCjiudqzg9ePAAPXv2xIsXL+Dp6YlGjRrh5MmT8PT0LJXxEEJKmFACUdf54Pxr8VwJudblgI7/nTg7thrQ89B73XDAnAj5BenmRMhPR5a9t9FhTDWgurv8EVsO+POu/n2MqGzQahAeBBGI7Fx5j18otVvYxwjV86x44RWKEhISjNM7D3/88Uep9U0IKWVCCkQhkZDunQOrVt/yXBn56c2Z+YCTNXAgVXW+jAFHnwJRPvJ/3/+jAshf162CfDkX1ZwurUsfF18CER68V4PoSRCByMEDVm3jeK+DUGq3sA9j1TDtECeEmAahBaINX4JlPuO5MnK3M4HgTYCbDXArU32bVnuAUGfgpoYTaOJvAPtSgVd5b++rpm8fX5wCTrThtw5EP4IJRP1WAtb2vNdDKLVb2IeDlXFqmPddQu7du4fPPvsMYWFhcHNzw+HDhwEAz58/x5dffokLFy4UsQRCCNGREAPRzUOa2+rhZa7mD3wAyJcB19Lk/2pyK1P9HxVd+yjQsmxiPIIKRBInFGybzH9lIIzaLezDWDXMa0vRtWvX0LhxY8hkMtStWxfJycmKW3p4eHjg6NGjyM7OxooVut1ThRBCNHLxhVXsRLMMRMRyCC4QrR4A2PDfUmSueN/mw9XVFSdPngTHcfDyUr5/Sps2bbBu3TqjDJAQYtms2kwBXr+iQERMliAD0cs7et/Q2BLw2n12+PBhDB06FJ6enmrPQgsICMDDh0XcoIcQQnSRn0OBiJgsLqCWMAMRUYtXKJLJZLC317zZ7dmzZ2qvYEoIIfoq+CfObAKRiAOGhskfYg1XNWnqA0yqAZTV8BFb1l4+v6mP+vliI/RBjEfcfKzZBKKu5c2/dnntPqtVqxa2bduGzz//XGVeQUEB/vjjD9SrV8/gwRFCCF6naZ5nQoEIADoHAF9Xf/t8aZLy/LL2wPYWgJUIaOqr/o7hayKBRt7yA0uDN8nvLP6uwRWBH9/5+OXTBzEedv+8WQQiwDJql9eWovHjx2Pnzp0YOnQorly5AgB48uQJ9u7di5YtWyIxMRHjxo0z6kAJIUSJiQUiYpmk++abRSCyFLy2FMXGxiIhIQEjRoxAfHw8AKBPnz5gjMHZ2RmrV69GZGSkUQdKCCEKgghEHESN9Lsp9p/3gAf//c2Lv6E6/2EO0HoP0NAbWHFT/TJ6HQYGhQLHnqh+035/uXz7IEYkK9A4y9QC0ezL8vo159rlffHGvn37onPnztizZw9u3rwJmUyG4OBgxMTEwMnJSNfbJoSQ9wklELWeBFHlFnoNXcZUdwm8b/9j+UOTRznAtEua50uN0AcpfoIIRNa2eo154135VaY1MYfa1TsU5eTkwN/fH+PGjcOYMWPQsWPHYhgWMQYmk4ITqbnLnoCYwhiJgAgpEEV0h/TwUlhFDeO5MsRSCSIQ2dhDHDuB5xqYL71Dkb29PaysrODg4FAc4yFGxInEKPhzDNjLexA3GwXOvxake+eA3Tuv9XWiRoMhqtwC0sNLwZL2AwDyc99uAs7/Xy/kS+SlIwrvAnHtXpCe+R2yC5u0jyesKcSRQyFL3APZ0XhwHkGw6jzXwLUkFkNogWjrRLDH1wEKRUQPgglEvX8G5xbAcy3MF6/dZ126dMHGjRsxdOhQtdcpIsLBXt6DuNGn4PzDId0wXOddAtKtE8Eu/vl2Vt4711B/ch2wEUHU+DN5IDqwELIjy7SOg6vZWR6Izq2HbPtUAAxq7g9IiHpCDEQX/6SL3xG9CCoQeYZCun0qrDrN5rk25olXKOrRowc+//xzREdH49NPP0WFChVgZ2en0q5WrVoGD5AYRr6FKJzfB74WosafQRw9QvdA1G6aUiAiRGd2rvIPbqEFIiMpIwFctdzw0lr09qaamu4hFewEpOUBLzTcQ6qoPqx43wWT6Epwgei3QWAyKc+1kRNC7Rb2Yawa5hWKmjRpovj5yJEjKvMZY+A4DlKpYf/hxHCcfy3dtxDpGogaDDLpQJSdnQ1HR0cAQFZWFu0KFjirtnGA2MYsA1GQE3CuHeBkDXx+Qv0ZNztbAFE+wKHHQLNdqvMHVwSW1Acy84GIrfI7i+vbx091jbM+RD1BBqJH/xq0pVMotVvYxzkt/0364BWKVq5caZzeSbGT7p1j9A98cdQwkw1ExARZ26MgoY/ZBSIACHeXf+ADQLSv6oe+iAMa/XdryUZe8uey995G0b7yf52sgZruqn9YdOmjprvh60LU4zxDII6dILxAZCCh1G5hH8aqYV6hqH///sbpnRQ77QdV8/vAlx5aDNlZ7cGYAhExloJtk80yEAHA9gfA5ntAkCMwR82wZQwYe07+jTr+huofFUD+ujBn4HYWsOMBvz4WJgKjqhq+PkSVuPUksKdJZhWIAOHUbmEf2x4Yp4Z5X6eoUGpqKp4+fYqQkBDaDWFS9N9lBnwNAJAdXwHYaN6BS4GIGFV6quZ5JhyIAOC1FOh6QHubhdfkD00uvARqbTWsj7W3KRQVF/bqntkFIkA4tVvYR7i7cWqY96FJW7ZsQaVKlVCuXDnUqlULp06dAgA8f/4c4eHh+OuvvwwfHSkmPA6q1vG0Y30CEReg+4H4Ug0H6QmJKYzRrJh4ICKWQbpjutkFInPGa0vR1q1b0blzZ9SvXx+9evVCXFycYp6HhwfKli2LhIQEdOrUyVjjJEbD8yyz7XOKXrI+gSg0CuLmY3UetVgE9D0MXE/X+SUaSd+8/bnxdkCs30Vd1arkAvxKd7YpOQIJRKLwLvzGTyxH/hvN8ygQCQ6vUDR16lRERkbiwIEDePHihVIoAoD69etj+fLlxhgfMSoDTrs/vkL7kvUNRN0Wgd0/Dy5Q99Nerqdrv8S8rtg7p35eegVwEsOXSUqQUALRf9fpIoQXIQQikcFH0JgdXrvPrly5gu7du2uc7+3tjadPn/IeFCkOwrgOkSIQJR+W3z2aEH0IKRBFj4D0zO/81oNYNkEEImuIm43iuQLmi1cosre3R3Z2tsb5t2/fRpkyZXgPihibAAPRxlFa7x6tTVl74GAr+cPPXn2bH+oAVzsC7fzVjEPigI77GK6nMSxorP7kAEP7IMVAaIHowMIib23zPi9bw+uqnb98/g911M+n2hU4oQSirvPB+et3geX4BuZfu7xCUXR0NFatWoWCAtU/ao8fP8bPP/+Mli1bGjw4YhzGCEQONiLkTamEvCmV4PDfmWeGBaJ83uszKBRo5C1/DApVnV/ZBRheGQhzAWZGqF/GrAj5/OGV5e2Low9iREIMREV8YVCnQwDVrkUTUiAKiYR0b9HHir4rvIz51y6vUDRjxgw8ePAAtWvXxvLly8FxHHbt2oUJEyagWrVqYIxh8uTJxh4r4UHUaLAAtxDxD0QAcPQJUCCTP449UZ1/Lxu4kyX/ec8j9cvY/d/0O1ny9sXRBzESa1uzCEQAcPEF1a7FElog2vBlkTcHf58l1C6vo6zCwsJw9OhRjBgxAhMnTgRjDHPnyu923qRJEyxevBgVKlQw5jgJT2pv7qqunYkEIgDY/xgI/m+vxUM1Z7pmFwA1tgABDkCihrPVvjoNxCfJ32TZavbi6dtHOF0RuNiIYyeAcy1n8oEIAM68EF7tkhIgxEB085Det/nosA+4kmbetatTKLp8+TLKly8PF5e327uqVq2KvXv34tWrV0hOToZMJkNQUBA8PT2LZ6SEF+nhpWYViAqpe8O8K7ug6DdNUfON0QcxHOcWAOmvA00+EBWi2rUwLr6wip0ovEDEw9M32mvLHGpXp91n4eHh2LZtm+J506ZNsW/fPgCAm5sbateujbp161IgEiCWtF/rfFMMRMSySLdPNZtARCyPVZspZhGILIVOocjOzg45OW/j28GDB/HkiZodfsSkUCAipoA9S9Y8kwIREbr8HApEJkSn3Wc1atTA/PnzIRaLFbvQzpw5A1tb7ZcC7ty5s+EjJMVCCIGI8wzhO3xCKBARk1DwTxwFIhOiUyhasGABunXrhkGDBgEAOI7DwoULsXDhQo2v4TgOUqnUOKMkRiWIQORXDeLWk/Qadws/zVe07l4BCHQEFl8HstQcwFfdDegUAPx1D7j8SnW+oxUwrBKQkgWsv8OvD1KCBBKIuLCmPAavTAi1G+pswAoQ7V6naZ5n4oFICLVb2IexjjPSKRTVrl0bycnJuHXrFp48eYImTZrgm2++QYsWLYwzClJiBBOI+qwAe3UPnE9lncf+XQRw6hlw6L09t018gN+j5D+XcwCGn3qvPwB7WgJlbIHPKwE+61TXZGYEMLSS/Ocnr/n1QUqIUAJRzc4QRw7lsQJvCaV2l9QzaDUIHyYeiIRSu4V9pOXCKHQKRX///Tc+/PBDhIWFISwsDP3790e7du1Qt67u960ytsWLF2Pu3Ll4/PgxatSogR9//BF16mi4TCYBILBA9OwmpHvnQTRwDd/VUWCah2Y0JdEH0YGQAlG7aZAl7oG4SgyPFZGj2rVQgghEHESNBvNcAfOtXZ1CUadOnfDrr7+iVy/5zQ8PHTqETp06FevAtFm3bh1GjRqFZcuWoW7duliwYAFiYmKQlJQELy+vUhuXkAkuEK35FHCvoNc6fHNO9ZsEIJ/W69DbTazvYwBa7AY6BgCb76lfm/HngAfZ8s24fPsgxUxogejcesjObzAoFAmldj8/+fYbOSlmQglErSdBVJn/3h6h1G5hH9fTjVPDOoUiJycnpKWlKZ7fuXMHWVlZhvfO0/z58/Hpp59i4MCBAIBly5Zh27Zt+N///odx48Yptc3NzUVu7tvtaunp8h2Pr169UnvMk1gshqurq9JrhMTV1RUZGRk6Ha8lFotRYF8WXONYiCO6Q3rgF8gu7wNc1Fw//T9caCTEDT+B7MxmyE78DrhoPhiaK1cD4uiRYNeOQnpoCeBUQXNbjyCIW34Ndv86pHvmAnZlwdmXhZUe63LrKVBdw7H91x8D1wEEWUF9Vb8GNifJf9S0jB3J2udr6yPICsjIgNp1ycjIAAAwnl97qIbLgrkxiKO+AOdVA9It08CePtJax6L6AyAKbQrpjvlgKf9qb1u9Pe/3B1emgl41HGQFSN+rr9KuXQCQ5lD96opX/br892XRzgVWMeOBfCsU/DMRkFprrjdrW4hbjAFn6wvppolg2W80txVZ8Xp/yA6vhji8M+/6FULtFvYR6mycGuaYDq1atmyJq1ev4pNPPoGLiwtGjx6NXr16oVYtzTeT4zgOX331VZED0FdeXh7s7e2xceNGdOzYUTG9f//+SEtLw5YtW5Tax8XFYcqUKUYfByH6un//PsqVK6f366iGiRBQ/RJTp0sN6xSKkpOT0a9fP5w8eVL+Io4rMnEV19lnjx49QtmyZXH8+HHUr19fMX3s2LE4dOgQTp1SPhLr/W8pMpkML1++RJkyZcBxdHgsKX6MMWRmZsLPzw8ikf63G6QaJqWJ6peYOn1qWKfdZyEhITh+/DjevHmDp0+fokKFCliwYAE6dOhglAEXJ4lEAolEojTN1dW1dAZDLNa7t8jRF9UwKW1Uv8TU6VrDet0Q1tbWFgEBAZg8eTKaNm2K8uXL8xqcITw8PCAWi1WuqP3kyRP4+PiU+HgIIYQQYh703xYKYPLkyfjggw+MPRad2NjYICIiQnHvNUC+OXbfvn1Ku9MIIYQQQvSh05aijz/+GBzHIT4+HmKxGB9//HGRr+E4DitWrDB4gOqMGjUK/fv3x4cffog6depgwYIFyM7OVpyNRgghhBCiL51C0f79+yESiSCTySAWi7F///4iD5ArzgPoPvroIzx79gyTJk3C48ePUbNmTezcuRPe3t7F1ichhBBCzJtOZ58RQgghhJg7XscUEUIIIYSYG73OPiuUmZmJo0eP4tatW8jMzISTkxNCQkLQqFEjODo6GnuMhBBCCCHFTq9QJJVKMWHCBPz000/IyclRuoAjx3Gwt7fHiBEjMHXqVF4X+SKEEEIIKS16HVPUo0cPrF+/HlWqVEHPnj3xwQcfwNHREVlZWfj333/x+++/IykpCT179sRvv/1WnOMmhBBCCDEqnUPR3r170bJlSwwbNgwLFy5UuyVIJpNh+PDhWLZsGfbs2YOmTZsafcCEEEIIIcVB51DUr18/nDhxAjdu3NB6ur1MJkNYWBgaNGiAVatWGW2ghBBCCCHFSecDf06fPo3OnTsXef0hkUiEzp07q9yYlRBCCCFEyHQORampqQgJCdGpbUhICFJTU3kPihBCCCGkpOkcirKysuDg4KBTW3t7e2RlZfEeFCGEEEJISdM5FDHGivXWHYQQQgghpUnnA61FIhH8/f3h4uJSZNv09HQ8ePAAUqnU4AESQgghhJQEnS/eGBkZqfOWojJlyiAoKIj3oAghhBBCShrdEJYQQgghBHRDWEIIIYQQABSKCCGEEEIAUCgihBBCCAFAoYgQQgghBACFolKTlZWFyZMno1WrVnB3dwfHcUhISFBpJ5PJkJCQgPbt28Pf3x8ODg744IMPMH36dLx580btslesWIHKlSvD1tYWoaGh+PHHH9W2e/jwIbp37w5XV1c4OzujQ4cOuH37tk7jz8vLw8KFCxEeHg5nZ2e4urqiatWqGDx4MK5fv65od/z4ccTFxSEtLU2n5RLToGv9AsCAAQPAcZzKo1KlSiptZTIZ5syZg8DAQNja2qJ69epYu3at2uUmJiaiVatWcHR0hLu7O/r27Ytnz57pNf4PPvgADg4OKFOmDGrWrIkRI0bg0aNHinbbt29HXFycTsskpuXq1avo1q0bgoKCYG9vDw8PD0RGRmLr1q1q2+tab1TDJo6RUpGSksIAsICAANakSRMGgK1cuVKlXWZmJgPA6tWrx6ZPn87i4+PZwIEDmUgkYk2aNGEymUyp/bJlyxgA1qVLFxYfH8/69u3LALBZs2apLDc0NJR5eXmx2bNns/nz5zN/f39Wrlw59vz58yLH37ZtWyYWi1mfPn3Y4sWL2YIFC9hnn33GypUrp7Qec+fOZQBYSkoKn/8mIlC61i9jjPXv359JJBL266+/Kj3+/vtvlbbjxo1jANinn37K4uPjWZs2bRgAtnbtWqV29+/fZx4eHiw4OJgtXLiQzZgxg7m5ubEaNWqw3NxcrWPPy8tj4eHhzM7Ojn322Wds2bJl7Pvvv2cDBw5kHh4e7MCBA4q2w4YNY/QxaZ62bdvGYmJiWFxcHIuPj2cLFixgjRs3ZgDY8uXLldrqU29Uw6aN/qdKyZs3b1hqaipjjLEzZ85o/KOSm5vLjh07pjJ9ypQpDADbs2ePYlpOTg4rU6YMa9OmjVLb3r17MwcHB/by5UvFtNmzZzMA7PTp04ppiYmJTCwWs/Hjx2sd++nTpxkANmPGDJV5BQUFSqGKQpF50rV+GZOHIgcHhyKX+eDBA2Ztbc2GDRummCaTyVjjxo1ZuXLlWEFBgWL60KFDmZ2dHbt7965i2p49e9T+QXvf+vXrGQC2Zs0alXmvX79m6enpiuf0B8WyFBQUsBo1arCwsDCl6brWG9Ww6dPpf6pChQosMDBQr0dQUFBxj91sFPVHRZ3Lly8zAGzRokWKadu2bWMA2LZt25TaHj9+nAFgv/76q2Ja7dq1We3atVWW27JlSxYcHKy177Vr1zIA7ODBg1rbTZ48mQFQebwbkH799VdWq1YtZmtry9zc3NhHH33E7t27p7ScqKgoVrVqVXb27FlWv359ZmtryypUqMCWLl2q0ueiRYtYlSpVmJ2dHXN1dWURERFqPziI8egaigoKCpQ+rN+3ePFiBoBdvXpVafrvv//OALAjR44opnl5ebFu3bqpLKNixYqsWbNmWsc7c+ZMBoDduXNHa7v+/furrd9CUqmU/fDDD6xKlSpMIpEwLy8vNnjwYKUvH4wxVr58edamTRu2a9cuVqNGDSaRSFjlypXZpk2blNrl5eWxuLg4FhISwiQSCXN3d2cNGzZku3fv1jpOYlxt27Zl3t7eStN0rTeqYdOvYZ2uaB0VFUX3PROYx48fAwA8PDwU0y5cuAAA+PDDD5XaRkREQCQS4cKFC+jTpw9kMhkuX76Mjz/+WGW5derUwe7du5GZmQknJye1fZcvXx4AsGbNGjRs2BBWVurLqHPnzrhx4wbWrl2LH374QTFWT09PAMCMGTMwceJEdO/eHZ988gmePXuGH3/8EZGRkbhw4QJcXV0Vy3r16hVat26N7t27o2fPnli/fj2GDh0KGxsbxXr8/PPP+PLLL9G1a1eMGDECb968weXLl3Hq1Cn06tWryP9TUnxycnLg7OyMnJwcuLm5oWfPnpg9ezYcHR0VbS5cuAAHBwdUrlxZ6bV16tRRzG/UqBEePnyIp0+fqtR5Ydvt27drHUth/a5evRoTJkzQ+Nk2ZMgQPHr0CHv27MGvv/6qdn5CQgIGDhyIL7/8EikpKfjpp59w4cIFHDt2DNbW1oq2N2/exEcffYTPPvsM/fv3x8qVK9GtWzfs3LkTLVq0AADExcVh5syZ+OSTT1CnTh1kZGTg7NmzOH/+vKINMb7s7Gy8fv0a6enp+Pvvv7Fjxw589NFHivn61BvVsBnUcGmnMsJvS1Hz5s2Zs7Mze/XqlWLasGHDmFgsVtve09OT9ejRgzHG2LNnzxgANnXqVJV2hd90rl+/rrFvmUzGoqKiGADm7e3NevbsyRYvXqy0GbiQpt1nd+7cYWKxWGUX3L///susrKyUphf2NW/ePMW03NxcVrNmTebl5cXy8vIYY4x16NCBVa1aVeO4SfEoqn7HjRvHvv76a7Zu3Tq2du1axbfXhg0bsvz8fEW7Nm3aqN3CnJ2dzQCwcePGKfW3evVqlbZjxoxhANibN280jjcnJ4eFhYUxAKx8+fJswIABbMWKFezJkycqbTXtejhy5Ija3Rc7d+5UmV6+fHkGQOlbdXp6OvP19WXh4eGKaTVq1FDZ9U2K35AhQxRbUEQiEevatavSlhJ96o1q2PRrmM4+M0Hfffcd9u7di1mzZiltTXn9+jVsbGzUvsbW1havX79WtAMAiUSitt27bdThOA67du3C9OnT4ebmhrVr12LYsGEoX748PvroI53ONPvzzz8hk8nQvXt3PH/+XPHw8fFBaGgoDhw4oNTeysoKQ4YMUTy3sbHBkCFD8PTpU5w7dw4A4OrqigcPHuDMmTNF9k9KzsyZMzFr1ix0794dPXr0QEJCAmbMmIFjx45h48aNinavX7/WqSYNrV87OzucOnUKY8aMAQAkJCRg0KBB8PX1xfDhw5Gbm1vkOm3YsAEuLi5o0aKFUv1GRETA0dFRpX79/PzQqVMnxXNnZ2f069cPFy5cUGz1dXV1xdWrV3Hz5s0i+yfGM3LkSOzZswerVq1CbGwspFIp8vLyFPP1qTeqYdOvYYNCUX5+Pv79918cPXoUhw8fVnkQ41u3bh0mTJiAQYMGYejQoUrz7OzslN7M73rz5g3s7OwU7QCofeMUnuZf2EYTiUSCb7/9FomJiXj06BHWrl2LevXqYf369fjiiy+KXI+bN2+CMYbQ0FB4enoqPRITE/H06VOl9n5+fnBwcFCaVrFiRQDAnTt3AABff/01HB0dUadOHYSGhmLYsGE4duxYkWMhJe+rr76CSCTC3r17FdPs7Ox0qklj1K+LiwvmzJmDO3fu4M6dO1ixYgXCwsLw008/Ydq0aUWO/+bNm0hPT4eXl5dK/WZlZanUb0hIiMoujvfrd+rUqUhLS0PFihVRrVo1jBkzBpcvXy5yLMQwlSpVQvPmzdGvXz/8888/yMrKQrt27cD+uy2oPvVGNWz6NazTMUXvk8lkGD9+PJYsWYKcnByN7aRSKe+BEVV79uxBv3790KZNGyxbtkxlvq+vL6RSKZ4+fQovLy/F9Ly8PLx48QJ+fn4AAHd3d0gkEqSmpqoso3BaYVtd+Pr6okePHujSpQuqVq2K9evXIyEhQeOxRoC8hjiOw44dOyAWi1Xmv3usia4qV66MpKQk/PPPP9i5cyc2bdqEJUuWYNKkSZgyZYreyyPFx87ODmXKlMHLly8V03x9fXHgwAEwxpQ+fN+vSV9fX6Xp70pNTVXUt67Kly+Pjz/+GJ06dUJQUBDWrFmD6dOna32NTCaDl5cX1qxZo3Z+4XFz+oiMjMStW7ewZcsW7N69G7/88gt++OEHLFu2DJ988oneyyP8dO3aFUOGDMGNGzcQFhamV71RDZt+DfMKRd999x3mzp2LIUOGoFGjRujbty9mz54NV1dXLFmyBBzHYc6cOcYeq0U7deoUOnXqhA8//BDr169XGzhq1qwJADh79ixat26tmH727FnIZDLFfJFIhGrVquHs2bNq+wkKCtJ4kLU21tbWqF69Om7evKnYFabpAMDg4GAwxhAYGKj4tqHNo0ePkJ2drbS16MaNGwCAChUqKKY5ODjgo48+wkcffYS8vDx07twZM2bMwPjx4xWbpUnpy8zMxPPnz5U+eGvWrIlffvkFiYmJqFKlimL6qVOnFPMBoGzZsvD09FRbv6dPn1a005ebmxuCg4Nx5coVxTRt9bt37140bNiwyG/0AJCcnKzyh1Jd/bq7u2PgwIEYOHAgsrKyEBkZibi4OJP5g2IOCndbpaenA9Cv3qiGzaCG+RyIFBwczD766CPGGGPPnz9nHMexffv2McbkB8DWqlWryGvdkLeKOlD12rVrrEyZMqxq1aoqp0q+Kycnh7m7u7O2bdsqTe/Tpw+zt7dnL168UEybNWsWA8DOnDmjmHb9+nUmFovZ119/rXW8N27cUHtQ9atXr5ifnx9zc3NTXI9j6dKlDAC7cOGCUtvk5GQmFotZr169VC5AKZPJlK51pO1Aa09PT8WB1uouOjlmzBgmEolYRkaG1nUi/Gmr39evX6v9vy88mPTPP/9UTLt//77Ga7yULVtW6Rovn332GbOzs1O6fMPevXsZALWXanjXxYsX2bNnz1Sm37lzh9nZ2bHq1asrpn399dcMgNIJDYwxdvDgQQZA7edcfn6+UnttB6nWrFlTMU1d/Xbr1o15eHhoXR/Cj7qDkvPy8litWrWYnZ0dy8zMVEzXtd6ohk2/hnltKXrw4AHGjh0L4O2BYoX7QW1sbNCnTx/Mnz8f3333Hd+sZhF++uknpKWlKS7JvnXrVjx48AAAMHz4cLi4uCAzMxMxMTF49eoVxowZg23btiktIzg4GPXr1wcg3yUxbdo0DBs2DN26dUNMTAyOHDmC3377DTNmzIC7u7vidZ9//jl+/vlntGnTBqNHj4a1tTXmz58Pb29v/N///Z/WcV+6dAm9evVCbGwsGjduDHd3dzx8+BCrVq3Co0ePsGDBAsUusYiICADAt99+ix49esDa2hrt2rVDcHAwpk+fjvHjx+POnTvo2LEjnJyckJKSgr/++guDBw/G6NGjFX36+flh9uzZuHPnDipWrIh169bh4sWLiI+PV5w22rJlS/j4+KBhw4bw9vZGYmIifvrpJ7Rp04bXli+inS71+/jxY4SHh6Nnz56K23rs2rUL27dvR6tWrdChQwfF8sqVK4eRI0di7ty5yM/PR+3atbF582YcOXIEa9asUdrN+s0332DDhg2Ijo7GiBEjkJWVhblz56JatWoYOHCg1nHv2bMHkydPRvv27VGvXj04Ojri9u3b+N///ofc3FylWyIU1u+XX36JmJgYiMVi9OjRA1FRURgyZAhmzpyJixcvomXLlrC2tsbNmzexYcMGLFy4EF27dlUsp2LFihg0aBDOnDkDb29v/O9//8OTJ0+wcuVKRZsqVaqgSZMmiIiIgLu7O86ePYuNGzfqdIwe0d+QIUOQkZGByMhIlC1bFo8fP8aaNWtw/fp1zJs3T2kXvq71RjVsBjXMJ0n5+fkpfWt3cXFhP/74o+L5/PnzdbqCraUrTN/qHoWnsBfeTkHTo3///irLjY+PZ2FhYczGxoYFBwezH374QWVrDGPybzVdu3Zlzs7OzNHRkbVt25bdvHmzyHE/efKEzZo1i0VFRTFfX19mZWXF3NzcWNOmTdnGjRtV2k+bNo2VLVuWiUQildPzN23axBo1asQcHByYg4MDq1SpEhs2bBhLSkpStFF38cby5cuzn376Samf5cuXs8jISFamTBkmkUhYcHAwGzNmjNYLBhL+dKnfV69esT59+rCQkBBmb2/PJBIJq1q1Kvvuu+8UW/jeJZVK2XfffcfKly/PbGxsWNWqVdlvv/2mtv8rV66wli1bMnt7e+bq6sp69+7NHj9+XOS4b9++zSZNmsTq1avHvLy8mJWVFfP09GRt2rRh+/fvV2pbUFDAhg8fzjw9PRnHcSqnNsfHx7OIiAhmZ2fHnJycWLVq1djYsWPZo0ePlP6fCi98V716dSaRSFilSpXYhg0blJY1ffp0VqdOHebq6srs7OxYpUqV2IwZM9T+PxHDrV27ljVv3px5e3srPsOaN2/OtmzZora9rvVGNWzaNcwx9t8h9npo27YtnJycFDe569SpE/7991+sWrUKMpkM/fr1g5+fH535Q4yiSZMmeP78udJ+ckJMRYUKFfDBBx/gn3/+Ke2hEMKLJdUwr1PyBw8ejNzcXMXphDNmzEBaWhoiIyMRFRWFjIwMzJs3z6gDJYQQQggpTryOKWrfvj3at2+veF6lShXcunULBw4cgJWVFRo0aKB0/AohhBBCiNDxCkXquLi4oGPHjsZaHCGEEEJIieJ1TNG9e/dw7949NGrUSDHt0qVLmDdvHnJzc9GzZ08KSIQQQggxKbxCUceOHZGVlaW4RP+TJ09QuXJl5OXlwcnJCU+fPsWGDRvQuXNnow+YEEIIIaQ48DrQ+vTp02jRooXi+erVq/H69WtcunQJDx8+RLNmzfD9998bbZCEEEIIIcWN1zFFL1++VLq31j///IOoqCgEBwcDADp37oxvvvnGOCM0MplMhkePHsHJyUnj5c8JMSbGGDIzM+Hn5weRyKB7MAOgGiYli+qXmDp9aphXKPL09MTdu3cBAGlpaTh58iRmzZqlmF9QUICCggI+iy52jx49gr+/f2kPg1ig+/fvo1y5cgYvh2qYlAaqX2LqdKlhXqGoefPmWLRoEZydnXHw4EHIZDKlA6uvXbsm2KIvvN3D/fv34ezsXMqjIZYgIyMD/v7+RrvVCNUwKUlUv8TU6VPDvELRrFmzcOPGDYwePRo2Njb4/vvvERgYCADIzc3F+vXr0atXLz6LLnaFm2udnZ3pDUlKlLF2FVANk9JA9UtMnS41zCsUeXt749ixY0hPT4ednR1sbGwU82QyGfbt2yfYLUWEEEIIIeoYdPFGFxcXlWl2dnaoUaOGIYslhBBCCClxOoWi1atXAwD69u0LjuMUz4vSr18//iMjhBBCCClBOl28USQSgeM4vH79GjY2NjqdlslxHKRSqVEGaUwZGRlwcXFBeno67c8mJcLYNUc1TEoS1S8xdfrUnE5bilJSUgBAcexQ4XNCCCFEqLKzs+Ho6AgAyMrKgoODQymPiAidTqGofPnyip/z8/ORnp4Od3d3o1yzghBCCCFECPS+PKlIJEJERAT+/PPP4hgPIYQQCyaTyUp7CEUyhTESfvQ++0wsFqN8+fLIzc0tjvEQQgixYCKRCMuXL0dqaqrBy8rLy1P8PGPGDKXLx/Dl6+uLIUOGGLwcIky8TskfPnw4fvrpJwwaNAju7u7GHhMhRADoeAxSWlJTUxW3kjLUgAEDFMskpCi8QpFUKoVEIkFwcDC6du2KChUqwM7OTqkNx3H46quvjDJIQgghhJDixisUjR49WvHzihUr1LahUEQIIYQQU8IrFNEp+YQQQggxN7xC0bun6BNCCCGEmAO9T8l/18uXL7F+/XrMmTMHc+bMwfr16/HixQuDBnT48GG0a9cOfn5+4DgOmzdvVpo/YMAAcByn9GjVqpVBfRaX7OxsxRizs7NLeziEEEII0YL3DWHj4uIwe/ZslVPzbWxsMHbsWEydOpXXcrOzs1GjRg18/PHH6Ny5s9o2rVq1wsqVKxXPJRIJr74IMUcymUynW/GUJlMYIyHE8vAKRdOmTcPUqVPRpk0bfPHFF6hYsSIAICkpCT/99BNmzJgBa2trTJw4Ue9lx8bGIjY2VmsbiUQCHx8fnZaXm5urFNwyMjIAyM+gK+57s727/JLojwiTob93fWtYLBYb5TovxXGNF+DtdV7o/WAaSqN+TQHVr+nQ53fFKxQtW7YM7dq1w5YtW5SmBwYGolWrVmjXrh2WLl3KKxTp4uDBg/Dy8oKbmxuaNm2K6dOno0yZMmrbzpw5E1OmTFGZnpSUpLgGSyFra2sEBwfDyor3BjQl7765xWKx0d7sBQUFuHXrFvLz842yPFK8srKyDHq9PjVsa2uLkJAQo1zn5d36unfvHqytrQ1a3vtSUlLw5s0boy6TGF9p1K8poPo1HfrUMK+//unp6VqP42ndujUOHjzIZ9FFatWqFTp37ozAwEDcunUL33zzDWJjY3HixAm1oWP8+PEYNWqU4nlGRgb8/f0RFham9m65xvqWDRTv1VRN5YODvP1mzJe+NWwqAgMDS3sIRAdUv+pR/ZoOfWqYVyhq2LAhTp06haFDh6qdf+rUKTRs2JDPoovUo0cPxc/VqlVD9erVERwcjIMHD6JZs2Yq7SUSidpjjrRtuTHW1VSL85u2qWxiJob/rvjUsCkw5bFbEqpf9Ux57JZGn98VryMdly1bhhMnTuCrr75CcnIyZDIZZDIZkpOTMXLkSJw8eRLLli3js2i9BQUFwcPDA8nJySXSHyGWwtraGgMGDMCAAQOMvuuMEEKEiNeWourVq0Mmk2HRokVYtGiR4iySwjsHSyQSVK9eXek1HMchPT3dwOGqevDgAV68eAFfX1+jL9tQhX9UCCGEECJ8vEJRly5dwHGcsccCQH5A1LtbfVJSUnDx4kW4u7vD3d0dU6ZMQZcuXeDj44Nbt25h7NixCAkJQUxMTLGMhxBCCCGWgVcoSkhIMPIw3jp79iyio6MVzwsP0Ovfvz+WLl2Ky5cvY9WqVUhLS4Ofnx9atmyJadOm0bWKCCGEEGIQvUNRTk4OGjdujE8//RSfffaZ0QfUpEkTMMY0zt+1a5fR+ySEEEII0ftAa3t7e6SkpBTb7jNCCCGEkNLA6+yzVq1a0RYbQgghhJgVXqFo4sSJuHHjBvr27YujR4/i4cOHePnypcqDEEIIIcRU8DrQumrVqgCAa9eu4ffff9fYju4NQwghhBBTwSsUTZo0iY4pIoQQQohZ4RWK4uLijDwMQgghhJDSxeuYIkIIIYQQc8NrS9HUqVOLbMNxHCZOnMhn8YQQQgghJc7ou884jgNjjEIRIYQQQkwKr91nMplM5VFQUIBbt27hq6++wocffoinT58ae6yEEEIIIcXGaMcUiUQiBAYG4vvvv0doaCiGDx9urEUTQgghhBS7YjnQOjIyEtu3by+ORRNCCCGEFItiCUVnz56FSEQnthFCCCHEdPA60Hr16tVqp6elpeHw4cP4888/8cknnxg0MEIIIYSQksQrFA0YMEDjPA8PD4wbNw6TJk3iOyZCCCGEkBLHKxSlpKSoTOM4Dm5ubnBycjJ4UIQQQgghJY1XKCpfvryxx0EIIYQQUqp4haL3Xb9+HRs2bEBqairCwsIwcOBAODs7G2PRhBBCCCElQudQ9NNPP2HRokU4fvw4PDw8FNO3bt2Kbt26IS8vTzHtxx9/xMmTJ5XaEUIIIYSf7OxsODo6AgCysrLg4OBQyiMyTzqfN//3338jODhYKegUFBTgk08+gVgsxsqVK/Hvv/9i1qxZuHv3LmbMmFEsAyaEEEIIKQ46h6Jr166hXr16StMOHDiAZ8+e4auvvkL//v1RtWpVjB07Ft27d6eLNxJCCCHEpOgcil68eAF/f3+lafv27QPHcejUqZPS9IYNG+LevXvGGSEhhBBCSAnQORR5e3vj8ePHStOOHDkCe3t71KhRQ2m6jY0NbGxsjDNCQgghhJASoHMo+vDDD7Fq1SpkZmYCAK5evYrTp08jJiYGVlbKx2tfv34d5cqVM+5ICSGEEEKKkc6haPLkybh79y5CQ0PRrFkzNGzYEBzHYfz48Spt//rrLzRo0MCoAyWEEEJMCZNJS3sIRWIyWWkPQVB0PiW/WrVq2L9/P2bMmIHbt2+jXr16GD16NCIiIpTaHTx4EPb29ujWrZvRB0sIIYSYCk4kRsGfY8Ce33470c4VVm3jAGt7FGybDKSnal6AtS3EsRPAuQWg4O8Jisn5/+uFfMl7f75FVhA3GwXOvxake+eA3TuvdWyiRoMhqtwCHN28XYleF29s0KABtm3bprVNkyZN8O+//xo0KEIIMSa6xgspLez5beDxNfkTBw9YdZoNiG1QkNAHeHlH8wtt7CHu/TM413KQ/joQ7M6lt/OeXAds3gkzImuIu84H5x8O6YbhYDcPaRkRB1HrSRBVbgHp4aWwihpmyOqZHYqIhBBCSHFz8IBVv5WAxAkFqwfoFog8QyH9bRDYIy0bGgoDUUgkpBu+1C0QRXSHdOtEsKT9PFfGfFEoIjrJzs4Gx3HgOA7Z2dmlPRxCCDEdQgxEF//kuTLmzSj3PiOEEEKIGnau8l1mBgYiBxsR8qZUetuWAlGxoFBECCGEFBOrtnHyY4hoC5FJENzus8OHD6Ndu3bw8/MDx3HYvHmz0nzGGCZNmgRfX1/Y2dmhefPmuHnzZukMlhBCCNHG2p4CkQkRXCjKzs5GjRo1sHjxYrXz58yZg0WLFmHZsmU4deoUHBwcEBMTgzdv3pTwSAkhhBDtCrZNpkBkQgS3+yw2NhaxsbFq5zHGsGDBAkyYMAEdOnQAAKxevRre3t7YvHkzevToUZJDJYQUMyaTghOJS3sYWjGZjK71QjTTdh0iCkSCI7hQpE1KSgoeP36M5s2bK6a5uLigbt26OHHihNpQlJubi9zcXMXzjIwMAIBUKoVUqnq1UbFY2B/AhdSNvaT60/R/R9Qz9P/KkmtYLFZz8Tu8vfCc9PDSIk8rFoV3Aap0UDxXe+G7/3BhTSGOHApZ4h7IjsZrXS4XUAviFl+DE1uZ9fuB6lc9XetXIwEFInOuX0C/9TOpUFR4Q1pvb2+l6epuVlto5syZmDJlisr0pKQkxcXcCtna2iIkJMRIoy1eKSkpJbrLMCcnR/Hz9evXYW9vX2J9m7qsrCyDXm+pNVy4LkoXv3v3wnM6fOCLGn8Gce1ekG6f83bi+xe+K1xyzc7yQHRuPWTbpwJgGpfLhUZB3HwM2L1z4ALrlvj7sSRR/aqna/2qJZBAJArvotO6mDp9atikQhEf48ePx6hRoxTPMzIy4O/vj7CwMDg7O5fiyAwTGBhYov29e22iSpUq0RWB9VD4zZgvquFCen7gN/4M4ugRkB5YCNnxFdqXXLMzxO2m6R6Iui0CSz4M6dGfIfpkXYm/H0sS1a96vH/nQglE/31hAEr+70lJ06eGTSoU+fj4AACePHkCX19fxfQnT56gZs2aal8jkUggkUhUpovFYpPZTKtOSY/93f5M/f+upBn6f0U1DBgUiI4s075kvoFo4yjAK5THupgWql/1eI1dSIEoegSkZ36HuHavEv89lPRtd/RZP5M6OjAwMBA+Pj7Yt2+fYlpGRgZOnTqF+vXrl+LIhMk07tAs/DGS0ifIQCTL57MqxFIJLRAdWAjZhU381sWMCW5LUVZWFpKTkxXPU1JScPHiRbi7uyMgIAAjR47E9OnTERoaisDAQEycOBF+fn7o2LFj6Q1aoDiRGIzJdD9otPlYsPvnId03H5AVKM3Pz337vODvCWCdp4O9ugfpjulAvpZ90S6+sGozBcjPQcE/ccDrtLd9egTBqvNcPqtGLIjioGoKRMRUCTEQHVkG+FThtz5mTHCh6OzZs4iOjlY8L9wX3b9/fyQkJGDs2LHIzs7G4MGDkZaWhkaNGmHnzp2wtbUtrSELmixxD2Qbv4JOB40mH9L8gZ8nU/wobvk12NMkSNd8CuTlqLYt5F4BVrETgdevULB6IJD9XGm25hER8hYFImLSrG2FGYiIWoILRU2aNAFjWj6gOA5Tp07F1KlTS3BUpku+hci4H/js+S1I//qy6EDULwHIzVQbiAjRlfTwUoM/8N+9bxQFIlKSxLETwLmWo0BkIkzqmCJiXPp84HO+VRU/S9cNo0BESkyR1yGiLUREwDi3AApEJoRCkYXSKxD5VYO4x5K3E/Jea16wPoHIzlXvcRPyLgpEROik26dSIDIhFIoskN6BqM8KsOe3il6wPoHIwUN+92hCeBJCIOI8TeNCg6T0sGfJmmdSIBIcCkUWhlcgenZTvstMG30DUb+VgDVdFZvwI4hA5FcN4taT+K4CsXQUiASJQpEF4R2I1nxqvF1mhYFI4iS/ezQhehJMIOqzAuzVPb6rQSyZQAIRF9aUx+DNG4UiC2FYIDLSQdXvBqLVA7TfPZoQNQQViJ7dlF+nixB9CCUQ/XevP11JZUW3KW3GGKPgTsknxmeMQPTuKc0KhgSil3fowmFEL4ILRGs+Bdwr8Fwb/kr6FgnEiIQUiNpNgyxxD8RVYnQauvidTSizLwMb7yrP97IFtjQDrETAhRfA4OOqy4hvAISXAQreCS+NtwPi/y4z2LU88HV1fn1UcgF+jdRpVbSiUGTm5FeqHiOsLUQv7/BdHWKhBBmItL0/CHmf0ALRufWQnd+gcyh614Mc4MJL5Wll3zlENLtAdX7h9PddegVw/90ar56n4X0YikKRmRM3H6v9StX/oUBEhEoU3gXi2r0oEBHTJcRAtH0q4FNZr9WYfVkeVuJvqM57mAO03gM09AZW3FT/+l6HgUGhwOG76ue/u1y+fRiKQpGZY/fPUyAiJo0CETFpIithBiIeN1raeFf71pn9j+UPTR7lANMuASxX/XwpA5YmaR9DUX0YikKRmZPf3JUCETFd0jO/UyAiJkvcbBQ4/3CTD0SWgkKRuZOp2Yn7HwpExBTILmzSOp8CEREyzr8WpBuGUyAyEXRKvoUSRCCytuU5esNkZ2eD4zhwHIfs7OxSGQMxDgpEROike+dQIDIhFIoskCACkY09xLETeK4BIQIJRC6+BqwBsQTs3nktc007EJWRAMFOmudbi4AqrvJ/NQlylC/H0D6sjJRmKBRZGMEEot4/g3ML4LkWxNIJIhC5V4BVmykGrAWxbKYdiIKcgOQuQFJnYHBF9W12tgAud5D/q8nFDvLlBKkJPvr08VNd/ddBHQpFFkRQgcgzFNLtU3muCbFkgglE/RKAfNqtRvgQRiDiAmrxGTwAINwdcLKW/xytZoOpiAMaecl/buQlf67oV+IAcQLD+hQGBwcHOFkDNd0N60Pd6/mgUGQhBBeIfhuk/e7RhKghqECUm4mCf+J0HrvQb5MgpcNNSohAAlFoFMTNx/JZAQDA9gfA5nvA5ZfAnH9V58sYMPYckJQu/1emZihz/pW/fvM9YMcDw/pYmMh7VZTQ2WcWQJCB6NG/dJsPohfBBaLVAwEnL53HLxYBfQ8D19N1fola0jdvf373FgmGMNYtEkhRBBSIui0Cu38eXCC//U6vpUDXA9rbLLwmf2hy4SVQa6tx+gh3B0ZV1d5WFxSKzBznGQJx7AThBSJC9CDIQJT9XK9QBMgDkaG3Jnj3wnfv3iKBCJ3AAlHyYUiP/gzRJ+v4rIzZot1nZk7cehIFImLSuLCmwgxEhOhMgIFo4yit17GzVBSKzBx7dY8CETFp4sihFIiISRNmINL8/rBktPvMzEl3TKdAREyaLHEPBSJiskSNBkNUuQUFIhNBW4rMXf4bzfMoEBETIDsaD3MJRF62wMFW8oefvfo2P9QBrnYE2vmrn9+6rPY+ytob3gcxHnMKRPENDKurdv7y+T/UUT9fCLVLochSCSEQiXTfUCn005kB0xijuTGlQAQAHQKARt7yx6BQ1fmVXYDhlYEwF2BmhPplTA1/+3MlZ9X5g0IN74MYj/TwUrMIRAAQXsawupoVIZ8/vLK8/fuEULu0+8wSCSIQWUPcbJTOQ373dObaZYBF9eTTvzwJnHmh3NZODPzRRP5N448UYN4V5fnvntKcmgN8fEh+6ue79O1j2306pbmkCSIQ2bnqNeaLL4CC/8LzsSeq8+9lA3eygAqOwJ5H6pexL/Xtz/ezofIpfvSJ4X0Q42FJ+7XON5VABBheV7sfyQPNnSx5+/cJoXYpFFkaoQSirvPB+et3NdXC05kvvAS2PZRPe6jhb1vlv4AAByBRzTVh3j2ludbfQI6ad4G+fYQb6WqqRDeCCEQOHrBqG6fXuM+8AII3yX9WV1fZBUCNLZprFwC+PvdOeynAvVe/+x8b3gcpGaYUiACgwz7gShr/uvrqNBCfJA832WpOfBNC7VIosiRCCkQhkZDunQOrVt/yWhVNQaVQdoFubxp1f1SM3QcxLsEEon4rAWsNBz5oYWhdFd4ioTj7IMVPCIGI8wzRa8xP32ivLV3qqqj5pV27dEyRpRBaINrwZRF3jyZElaACkcQJBdsm818ZYrEEEYj8qkHcehLfVTBbFIosgRAD0c1DPFeGWCrBBaLVA4D0VM1tCVFDMIGozwqwV/f4robZolBk7lx8KRARkyfIQKTt/UGIGoIKRM9uyq9jR5RQKDJzVm2mmE0gauGneV73CsDXHwCOGo4Pqu4GTK4h/1cdRyv567tX4N8HKR5cQC2zDkRCqN1QNaf2E+MSXCBa86n269jpQAi1W9iHsWqYPt7NXX6OWQQiAPguAjj1DDj03qmaTXyA36PkP5dzAIafUp7PAdjTEihjC3xeCfBerbrsmRHA0Eryn5+85tcHKR7i5mPBkg+ZZSDSt3Z91qn+eTRG7S6pZ4SVIRoJMhBpe3/oQCi1W9hHWi6MwuS2FMXFxYHjOKVHpUqVSntYglXwT5xZBCJtmPYTcUymD6Ieu3/eLAMRQLVrCUThXcwuEAHmW7smuaWoatWq2Lt3r+K5lZVJrkbJeJ2meZ6JBaJvzql+kwDk03odAgIdgcXXVeczAC12Ax0DgM331H/MjD8HPMgGUrL490GKh3TffLMMRIBwavfzk2+/kRPjEtfuZXaBCBBO7Rb2cT3dODVskmnCysoKPj4+OrXNzc1Fbu7b7WoZGRkAAKlUCqlUqtJeLBYbZ5DFTN3Y36d1XQQUiHRdF21XMF1/R/vrL7+SPwD113nJKgBmX1HzQj36ANSviy7rp43F17BMzVXe/iOUQGTI+1Gf2lXHGLV7U15SVL960PV3Lj3zu+ADEd/6FULtFvZReAFdQ2vYJEPRzZs34efnB1tbW9SvXx8zZ85EQECA2rYzZ87ElClTVKYnJSXB0dFRaZqtrS1CQvS7mFVpSUlJwZs3mg+S07ougghEHESNBhu+LgKjbl2ysrIMWibVsHqCCETWtkZZF6Gg+tWdrr9z2YVNWpdT2oFIn3UxBYbWsMmForp16yIhIQFhYWFITU3FlClT0LhxY1y5cgVOTk4q7cePH49Ro97eYysjIwP+/v4ICwuDs7PpnnIRGBjI74VCCUStJ0FUuQUAA9ZFgNStS+E3Y76ohlUJIhDZ2EMcOwGA+dQw1a/ujPE7F0IgAsynfgHDa9jkQlFsbKzi5+rVq6Nu3booX7481q9fj0GDBqm0l0gkkEgkKtPFYrHJbKZVh9fYhRSIIrpDengprKKGmfTv4X3q1sXQ9aMaViaYQNT7Z3Bu8i3Upvx7eBfVr+4MHbsgApGLLwDzqV/A8Bo2ubPP3ufq6oqKFSsiOTm5tIcibEILRFsnFnn3aG3K2ssfmjhaAZVdtC+jsgvgoOVrgTH6IMYlqEDkGQrp9ql6rwPVLhFEIHKvIL+OnR68bM2/dk0+FGVlZeHWrVvw9fUt7aEIlxAD0cU/ea4M0NQHuNVF/miq5nh7RyvgYgfg347AD3XUL+OHOvL5lzqof4Maow9iXIILRL8NAnum35ex2mWodi2dYAJRvwQgX78z0bY0M//aNblQNHr0aBw6dAh37tzB8ePH0alTJ4jFYvTs2bO0hyZMdq5mFYgAoJE3YCWSPxp6q873dwAq/Hf8pqarYLf8b3oFRyDAoXj6IMYjyECk7f2hQc0yVLuWTFCBKDdTfh07PVhC7ZrcMUUPHjxAz5498eLFC3h6eqJRo0Y4efIkPD09S3togmTVNg4Q25hNIAKAFTeBpr5vf35fYjrwY6L8TTP+nPpljDsnv5rqnkfy9sXRBzEOzjME4tgJJh+IAGDLPaCKq/xnql3LIrhAtHog4OSl1zpceAFkF5h37ZpcKPrjjz9KewimxdoeBQl9zCYQAcDDHKDJTu1tvjqtff7W+/KHsfoovEYGMT5x60lgT5NMPhABwNM3wqtdUvwEGYiyn+sdigYfBy681DzfHGrX5EIR0U/BtslmFYiI5WGv7plFICKWiQtrCnHkUOEFIqKWyR1TRPSUnqp5HgUiYgKkO6ZTICImiwKRaaFQZKkoEBFTka/5SrsUiIjQyRL3UCAyIRSKLJFAApEovAu/8RMCCCMQiegIBKKd7Gg8KBCZDgpFlkYogajxZxDX7qXX0Od8CNipuTCpnRjYGA2cb6f5gOcRVYBrHeX/qlPLXf76jdH8+yAlSBCByBriZqM0z9eBUGq3ZxCv4RMDmXIgEkrtFvZhrBqmUGRJhBSIokdAeuZ3vYYf7QvEllOd3roc0DEAqO4OjK2mpj8OmBMBVHSR/yviVNuMqSZ/fccAfn1wapZJiolQAlHX+eD8a/FcCTkh1K6IA0ZUNmg1CA+CCER2rrzHL5TaLezDWDVMochSCC0QHVhY5N2j35edD1xUczrohZdA5n+fJwfUHFcuY8DRp/Kfjz6VP39f4esyefbBNG8dJ8YkpEAUEgnp3jk8V0ROCLUrY+pfR4qPIAKRg4f8OnY8CaV2C/swVg3TDnFLIMRAdGQZ4KNhm6oG7fcDtzNVp9/OBII3AW42wC018wGg1R4g1Bm4qeFmyfE3gH2pwKs84GUuvz5IMRNaINrwJVjmM54rIyeU2v3iFHCiDb91IPoRTCDqtxKw1nKTsSIIpXYL+3CwMk4N05Yic2dtK8xAxENGnuZ5L3O1h5V8GXAtTf6vJrcy1b8xde2DFCMhBiKt7w/dCaF2C7QsmxiPoAKRxEl+HTsDCKF2C/swVg3TliIzJ46dAM61nMkHImLBXHxhFTvRLAMRsRyCC0SrBwA2/LcUmSvaUmTmOLcACkTEpFm1mUKBiJg0QQYibe8PC0ZbisycdPtUCkTEtOXnUCAiJosLqAVx8zEUiEwEbSkyc+xZsuaZJhqIxBwwNEz+EGs4Fb6pDzCpBlBWw9bhsvby+U19iq8PYhwF/8SZTSASUe1aHHHzsWYTiLqWN//apS1FlspEAxEADK4I/Fjv7fOlScrzy9oD21sAViKgqa/6uy6viQQaecsPzgveJL87syF90F3Hi9HrNM3zTCgQAUDnAODr6m+fC6F2i7orOTEMu3/eLAIRYBm1S1uKLJFAAhEX1pTH4An5j4kFImKZpPvmm0UgshS0pcjSCCUQ1ewMceRQHisgv7aFup8LPcwBWu8BGnoDK26qX0avw8CgUODYE9VvK3z68LbVbx2IgQQRiDiIGg3Wa9h/3gMe/FdvQqldUsxkBRpnmVogmn1ZXr/mXLsUiiyJkAJRu2mQJe6BuEqM3qshZaqbVd+3/7H8ocmjHGDaJeP1QaGoBAklELWeBFHlFnoNXSbA2iWlQxCByFq/D66Nd+VXmdbEHGqXdp9ZCqEFonPr/7t7NCF6EFIgiugO6eGlPFeEWDJBBCIbe4hjJ/BcA/NFocgSCDEQbZ/KY0WIRRNaINo6ESxpP8+VIZZKMIGo98/g3AJ4roX5olBk7kRWAg1EdAdVogchBqIi3h+EvE9QgcgzFFL6cqqCQpGZEzcbZTaByMVG87wyEiDYSfN8axFQxVX+rybBTvLl8O2DFBM7V7MOREKoXSv6S1DsBBeIfhuk/Tp2OhBC7Rb2YawapreCmeP8a5lFIAKALU2BIDVvjiAnILkLkNRZfp0LdXa2AC53kP+rzuCK8tcnd+HfBykeVm3jzDYQCaV2f6rLb/xEN4IMRNreHzoQSu0W9mGsGqZQZOake+eYRSACAAdroKa76vRwd8DJWv5ztK+aMXNAIy/5z4285M/fV/g6J559cBquvkqMwNreLAMRIIzaFXHqX0eMg/MMMbtABAindgv7MFYNUygyc+zeeS1zTScQAcCBVGDHA9Xp2x8Am+8Bl18Cc9S812UMGHsOSEqX/ytTM5Q5/8pfv/kevz4YHSJVbAq2TTbLQAQIo3ZlDFiYaPi6EPXErSeZXSAChFO7hX0Yq4bpOkUWy7QCEQCMPQu8lqpOfy0Fuh7Q/tqF1+QPTS68BGpt1Txflz5IMUlP1TzPhAMRIJzaXXsbGFVVexvCD3t1z+wCESCc2i3sI9zdODVMW4oskjACERdQi8/gCZEz8UBELIN0x3SzC0TmjEKRxRFIIAqNgrj5WD4rQIhgApEovAu/8RPLkf9G8zwKRIJDociiCCgQdVsEdl/b8U6EaCCUQNT4M4hr9+K3DoQIIRCJ6Aia91EoshgCC0TJh+V3jyZEH0IKRNEjID3zO7/1IJZNEIHIGuJmo3iugPmiUGQRBBiINo7SevdobcraAwdbyR9+9urb/FAHuNoRaOevfn47f/n8H+oUXx/EyIQWiA4shOzCJr1WwcuWatfiCSUQdZ0Pzl+/4zrjG5h/7VIosgDCDET5fFYFADAoFGjkLX8MClWdX9kFGF4ZCHMBZkaoX8asCPn84ZXl7YujD2JEQgxERbw/1OkQQLVr0YQUiEIiId07R6/hh5cx/9o12VC0ePFiVKhQAba2tqhbty5Onz5d2kMSJFGjwWYViADg6BOgQCZ/HHuiOv9eNnAnS/7znkfql7H7v+l3suTti6MPYiTWtmYRiADg4guqXYsltEC04csirmOnyhJq1ySPslq3bh1GjRqFZcuWoW7duliwYAFiYmKQlJQELy+v0h6eoIgqtzCrQAQA+x8Dwf/ttXio5kzX7AKgxhYgwAFITFe/jK9OA/FJ8jdZtpq9ePr2EU5XBC424tgJ4FzLmXwgAoAzL4RXu6QECDEQ3TwE+FTRazU67AOupJl37ZrklqL58+fj008/xcCBA1GlShUsW7YM9vb2+N///lfaQxMc6eGlZhWICj3MUf+mKZRdUPSbJjFd/RvTmH0Qw3FuAWYRiApR7VoYF19hBiIenr4x/9o1uS1FeXl5OHfuHMaPH6+YJhKJ0Lx5c5w4cUKlfW5uLnJzcxXP09Pl/5uvXr2CVKp6eWSxWAxXV1el1wiJq6srMjIy1I79fWKxGAX3k8Bc1Oyc/Y+oenuII7pDeuAXyC7vA7S05UIjIW74CWRnNkN24nfAJURz23I1II4eCXbtKKSHlgBOFVTb2JeFlR7rEmQFSG2LbFoqgqyAjAyoXZeMjAwAAON5LxBLr2Hp4WWQZb/RXJsiK4ijvgDnVQPSLdPAnj7SWsei+gMgCm0K6Y75YCn/am9bxPvDXGqY6ld3+tYvi/oaePUCBbtmAlZugIub+sZ2LrCKGQ/kW6Hgn4mA1FpzbVrbQtxiDDhbX0g3TQTj+f4wl/oFjFjDzMQ8fPiQAWDHjx9Xmj5mzBhWp04dlfaTJ09mkG/OoAc9SvVx//59XjVPNUwPITyofulh6g9daphjzLRuZfno0SOULVsWx48fR/369RXTx44di0OHDuHUqVNK7d//liKTyfDy5UuUKVMGHN3anJQAxhgyMzPh5+cHkUj/PdZUw6Q0Uf0SU6dPDZvc7jMPDw+IxWI8eaJ8aPqTJ0/g4+Oj0l4ikUAikShNc3V1Lc4hEqLCxcWF92uphklpo/olpk7XGja5A61tbGwQERGBffv2KabJZDLs27dPacsRIYQQQog+TG5LEQCMGjUK/fv3x4cffog6depgwYIFyM7OxsCBA0t7aIQQQggxUSYZij766CM8e/YMkyZNwuPHj1GzZk3s3LkT3t7epT00QgghhJgokzvQmhBCCCGkOJjcMUWEEEIIIcWBQhEhhBBCCCgUEUIIIYQAoFBECCGEEAKAQhEhhBBCCAAKRYQQQgghACgUEUIIIYQAoFBECCGEEAKAQhEhhBBCCAAKRYQQQgghACgUEULMyJ07d8BxHC5evFjaQyFmqEmTJhg5cmRpD4MUIwpFRG8ymUzQ/Q0YMAAcx2HWrFlK0zdv3gyO43RejqYPwISEBLi6uuo1JqFhJfw7ZDKpzm05jtP6iIuLK76B6kBb8DL1P5rSki0LvftbtmwZnJycUFBQoJiWlZUFa2trNGnSRKntwYMHwXEcbt26ZYSRqmfOnxGWyqq0B0BMj0gkwvLly5Gamlrsffn6+mLIkCF6v87W1hazZ8/GkCFD4ObmVgwjM22cSISCbVPBHl7S3MjOFVZt4wBrexRsmwyka/l9W9tCHDsBnFsApNungj1LftuXRxCsOs/VeWzv1tW6deswadIkJCUlKaY5OjrqvCyiH7EI6HsYuJ6uPL2SCzCpBvAgB5h4Ach9L+NKxMC0cKCcPTDlEpD03usBoGcQ0CUA2HQPWHtbvsxfI/UbX3R0NLKysnD27FnUq1cPAHDkyBH4+Pjg1KlTePPmDWxtbQEABw4cQEBAAIKDg/XqgzEGqVQKKyv682iJaEsR4SU1NRV3794t9gff4NW8eXP4+Phg5syZGtts2rQJVatWhUQiQYUKFTBv3jy+/x1YunQpgoODYWNjg7CwMPz6669K8zmOw9KlSxEbGws7OzsEBQVh48aNvPszBvbwEvD4mvpH5lNYtRoPiG1QkNAHSNqnue3LOxA3/z9wruUg/XUg2L9/K81nz2/rNS4fHx/Fw8XFBRzHKZ57eXlh/vz5KFeuHCQSCWrWrImdO3dqXJZUKsXHH3+MSpUq4d69ewCALVu2oFatWrC1tUVQUBCmTJmitOWB4zj88ssv6NSpE+zt7REaGoq///6b1//xq1ev0K9fP7i5ucHe3h6xsbG4efOmYn7hFoXNmzcjNDQUtra2iImJwf3793n1ZwxJGcCFl8qP9gFAqAsQ7Qv42avOL2svnxfqAnQIUJ1/6RUwojJQ3kn+76VXqsFLF2FhYfD19cXBgwcV0w4ePIgOHTogMDAQJ0+eVJoeHR2N3NxcfPnll/Dy8oKtrS0aNWqEM2fOKLXjOA47duxAREQEJBIJjh49iuzsbPTr1w+Ojo7w9fU16PMBMM3PCEtEoYiYJbFYjO+++w4//vgjHjx4oDL/3Llz6N69O3r06IF///0XcXFxmDhxIhISEvTu66+//sKIESPwf//3f7hy5QqGDBmCgQMH4sCBA0rtJk6ciC5duuDSpUvo3bs3evTogcTERL6rWHwcPGDVbyUgcULB6gHAyzua29rYQ9z7Z3CeoZD+Ngjs0b/FOrSFCxdi3rx5+P7773H58mXExMSgffv2SkGjUG5uLrp164aLFy/iyJEjCAgIwJEjR9CvXz+MGDEC165dw/Lly5GQkIAZM2YovXbKlCno3r07Ll++jNatW6N37954+fKl3uMdMGAAzp49i7///hsnTpwAYwytW7dGfn6+ok1OTg5mzJiB1atX49ixY0hLS0OPHj30/88xEsZUpx3477tJZj5wUc1/w4WX8nnvtn2XjAFHn8p/PvpU/pyv6OhopffWgQMH0KRJE0RFRSmmv379GqdOnUJ0dDTGjh2LTZs2YdWqVTh//jxCQkIQExOj8vscN24cZs2ahcTERFSvXh1jxozBoUOHsGXLFuzevRsHDx7E+fPneY3Z7D4jzBjHmLq3ACHaxcXF4e7du8XeT/ny5fU+hmTAgAFIS0vD5s2bUb9+fVSpUgUrVqzA5s2b0alTJzDG0Lt3bzx79gy7d+9WvG7s2LHYtm0brl69CkB+vMDx48dhY2OjtPyCggLY2toiLS0NANCwYUNUrVoV8fHxijbdu3dHdnY2tm3bBkD+LfCzzz7D0qVLFW3q1auHWrVqYcmSJXqtn7Hkx3eRb815V3EEIp8qsB68idcYExISMHLkSMX/ddmyZTFs2DB88803ijZ16tRB7dq1sXjxYty5cweBgYE4cuQI4uLikJubi3/++QcuLi4A5FsQmzVrhvHjxyte/9tvv2Hs2LF49OgRAPnvasKECZg2bRoAIDs7G46OjtixYwdatWql6MPOzg4ikfL3ytevX2P48OFYsGABbt68iYoVK+LYsWNo0KABAODFixfw9/fHqlWr0K1bNyQkJGDgwIE4efIk6tatCwC4fv06KleujFOnTqFOnTq8/t8MUXurPOS8L9gJeJUHvMxV/zp3CeBmA9zKVD/fWgSEOgM3M4B8GRDuDpxpp//4fvnlF0VNvH79Gu7u7nj06BH27t2LZcuW4dChQ9i/fz+aNWuGO3fuIDQ0FAkJCejVqxcAID8/HxUqVMDIkSMxZswYxRalzZs3o0OHDgDkxymVKVMGv/32G7p16wYAePnyJcqVK4fBgwdjwYIFAMz/M8IS0ZYiYtZmz56NVatWqXzbSkxMRMOGDZWmNWzYEDdv3oRU+vaAid69e+PixYtKj6lTp+q0rPf7rF+/vspzQX0LLK4tRCLjHJuRkZGBR48e6fR/3bNnT2RnZ2P37t2KQAQAly5dwtSpU+Ho6Kh4fPrpp0hNTUVOTo6iXfXq1RU/Ozg4wNnZGU+fPlXqY926dSq18eGHHyrmJyYmwsrKShF2AKBMmTIICwtTGq+VlRVq166teF6pUiW4uroKqzYgDzuaAhEgn6cpEAHyIHQtTf6vIZo0aYLs7GycOXMGR44cQcWKFeHp6YmoqCjFcUUHDx5EUFAQ0tPTkZ+fr1Qz1tbWqFOnjsr/77u/u1u3biEvL0/pd+fu7o6wsDCV8VjUZ4QFoCPJiFmLjIxETEwMxo8fjwEDBuj9ehcXF4SEhChN8/LyMtLoBKTYApE1xM1GGXu0RWrdujV+++03nDhxAk2bNlVMz8rKwpQpU9C5c2eV1xQeoAvI/3C+i+M4lbMg/f39VWrDzs7OGMMnWoSEhKBcuXI4cOAAXr16haioKACAn58f/P39cfz4cRw4cEDp964LBwcHXuOxmM8IC0FbiojZmzVrFrZu3YoTJ04oplWuXBnHjh1Tanfs2DFUrFgRYrFYr+VrWlaVKlWUpr17EGjh88qVK+vVV7EozkDUdT44/1pGGaazszP8/Px0+r8eOnQoZs2ahfbt2+PQoUOK6bVq1UJSUhJCQkJUHu/vCjNU5cqVUVBQgFOnTimmvXjxAklJSUrjLSgowNmzZxXPk5KSkJaWJozaEKjo6GgcPHgQBw8eVDoVPzIyEjt27MDp06cRHR2tOLD53ZrJz8/HmTNnVGrmXcHBwbC2tlb63b169Qo3btzgNV6T/4ywILSliPDi6+trMv1Uq1YNvXv3xqJFixTT/u///g+1a9fGtGnT8NFHH+HEiRP46aefeO27HzNmDLp3747w8HA0b94cW7duxZ9//om9e/cqtduwYQM+/PBDNGrUCGvWrMHp06exYsUKg9ePL84jCOz90+5t7AEfDX8s3j/tXibV3FZkBXGzUeD8a0F2+leIG3xslDGPGTMGkydPRnBwMGrWrImVK1fi4sWLWLNmjUrb4cOHQyqVom3bttixYwcaNWqESZMmoW3btggICEDXrl0hEolw6dIlXLlyBdOnTzfKGAuFhoaiQ4cO+PTTT7F8+XI4OTlh3LhxKFu2rOLYFUC+VWr48OFYtGgRrKys8MUXX6BevXqlcjwRID9VXuj9REdHY9iwYcjPz1dsKQKAqKgofPHFF8jLy0N0dDQcHBwwdOhQjBkzBu7u7ggICMCcOXOQk5ODQYMGaVy+o6MjBg0ahDFjxqBMmTLw8vLCt99+yzs4m+pnhEVihOhJKpUKur/+/fuzDh06KE1LSUlhNjY27N2S37hxI6tSpQqztrZmAQEBbO7cuUqviYqKYiNGjFBZ/sqVK5mLi4vStCVLlrCgoCBmbW3NKlasyFavXq00HwBbvHgxa9GiBZNIJKxChQps3bp1eq2XMcmkBSbR3/v/11KplMXFxbGyZcsya2trVqNGDbZjxw7F/JSUFAaAXbhwQTFt3rx5zMnJiR07dowxxtjOnTtZgwYNmJ2dHXN2dmZ16tRh8fHxivYA2F9//aU0DhcXF7Zy5UqNfRR6v2ZevnzJ+vbty1xcXJidnR2LiYlhN27cUFm/TZs2saCgICaRSFjz5s3Z3bt39f/PMoKCkn1rswIZv9cV/g4qVaqkNP3OnTsMAAsLC1NMe/36NRs+fDjz8PBgEomENWzYkJ0+fVox/8CBAwwAe/XqldKyMjMzWZ8+fZi9vT3z9vZmc+bMUfn9mvNnhKWis88IKQEcx+Gvv/5Cx44dS3soREDeP7uOWC76jBAGOqaIEEIIIQQUigghhBBCANDFGwkhhBBCANCWIkIIIYQQABSKCCGEEEIAUCgihBBCCAFAoYgQQgghBACFIkIIIYQQABSKCCGEEEIAUCgihBBCCAFAoYgQQgghBACFIkIIIYQQABSKCCGEEEIAUCgihBBCCAFAoYgQQgghBACFIkIIIYQQABSKCCGEEEIAAP8P7w/uDoRpkpEAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 600x400 with 6 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "CHECKPOINTS = [300, 600, 900, 1200, 1500, 3000]\n",
+    "SEEDS = [0, 14, 41, 53, 96]\n",
+    "\n",
+    "plot_surprisal_differences_checkpoints(seeds=SEEDS, checkpoints=CHECKPOINTS)\n",
+    "plt.savefig(f\"figures/hop_surprisals.pdf\", format=\"pdf\", bbox_inches=\"tight\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkUAAAGhCAYAAABvQ8DIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB+I0lEQVR4nO3dd3wT9f8H8FeS7t0CXdACHZQhoyB7b8qQjYACIgoiIspPEJRREARBEFGm8mUo8mUpfBFB9pKNILIpe5RNNxSafH5/xEZCRpNrSi/J6/l45NHm7nP3+Vz7TvvK3eVOIYQQICIiInJyysIeABEREZEcMBQRERERgaGIiIiICABDEREREREAhiIiIiIiAAxFRERERAAYioiIiIgAMBQRERERAWAoIiIiIgLAUEREREQEgKFIVk6ePImuXbsiKioKXl5eKFq0KBo0aIB169YZbX/69Gm0atUKPj4+CAoKQq9evXD37l2DdhqNBlOmTEHp0qXh4eGBSpUqYdmyZRaPa8+ePUhISEDx4sXh4eGByMhItGvXDj/99JOuTVZWFhITE7Fjxw6rt5scz59//olXXnkFQUFB8PLywksvvYSZM2catNu7dy/q1asHLy8vhIaG4v3330dGRoZBu+zsbHz88ccIDw+Hp6cnatasic2bN1s8nnXr1qFhw4YIDg6Gl5cXoqKi0K1bN2zcuFHX5ubNm0hMTMSxY8ckbTM5hiNHjqBVq1bw8/ODr68vWrRoYbImWL8OSJBsrF+/XrRs2VIkJiaK+fPnixkzZoj69esLAGLevHl6ba9duyaKFi0qoqOjxddffy0mTpwoAgMDReXKlUV2drZe2xEjRggA4u233xbz588Xbdq0EQDEsmXL8hzTihUrhEKhEPHx8eKLL74Q8+fPFyNHjhR169YVjRo10rW7e/euACDGjh1rk58F2a/ff/9duLm5iZo1a4rp06eL+fPni48//lgMGzZMr93Ro0eFh4eHiI+PF3PmzBGffvqpcHd3F61atTJYZ/fu3YWLi4v46KOPxLx580Tt2rWFi4uL2L17d57jmTp1qgAgGjZsKKZPny7mzp0rPvroI1GlShXRp08fXbtDhw4JAGLhwoX5/RGQnTpy5Ijw8PAQsbGx4ssvvxRTpkwRpUqVEn5+fuLMmTN6bVm/jomhSOZycnJE5cqVRVxcnN70gQMHCk9PT3HlyhXdtM2bNxsEqOvXrwtXV1cxaNAg3TSNRiPq168vSpQoIXJycsz2X758eVGhQgWDoCWEELdv39Z9z1BEQgiRmpoqQkJCRMeOHYVarTbbNiEhQYSFhYnU1FTdtO+++04AEL///rtu2oEDBwQAMXXqVN20R48eiejoaFG7dm2zfTx9+lT4+fmJ5s2bG53/bA3znwq1bt1aBAYGinv37umm3bx5U/j4+IhOnTrptWX9OiaGIjvQtm1bERISojctODhYdO3a1aBtmTJlRNOmTXXPZ82aJQCIkydP6rX76aefBIA836m4u7uLN954w2ybS5cuCQAGj2cD0unTp0Xnzp1FYGCgcHd3F9WqVRNr167VW8/ChQsFALFz507Rv39/ERQUJHx9fUWvXr3EgwcP9NoeOnRItGjRQhQpUkR4eHiIUqVKib59+5odJxW8OXPmCADi1KlTQgghMjIyjIaj1NRU4eLiYrD3KDs7W/j4+Ih+/frppg0bNkyoVCq9fz5CCPH5558LAOLq1asmx5OcnCwAiMTERLPj3r59u9EafvYfzP79+0XLli2Fn5+f8PT0FA0aNBB79uzRW8/YsWMFAHH69GnRtWtX4evrK4KCgsT7778vHj16pNd206ZNom7dusLf3194e3uLMmXKiJEjR5odJxUsX19fo39X27RpI9zc3ER6eroQgvUrhOPWL88pkqHMzEzcu3cPFy5cwFdffYUNGzagadOmuvk3btzAnTt38PLLLxssW6NGDRw9elT3/OjRo/D29ka5cuUM2uXON6dkyZLYunUrrl+/brJNsWLFMGfOHABAx44d8cMPP+CHH35Ap06dAGjPlapVqxZOnz6NESNGYNq0afD29kaHDh3wyy+/GKzvvffew+nTp5GYmIjevXtj6dKl6NChA4QQAIA7d+6gRYsWuHz5MkaMGIFvvvkGr732Gvbv3292W6jgbdmyBX5+frhx4wbi4uLg4+MDPz8/DBw4EI8fP9a1+/vvv5GTk2NQw25ubqhSpYpBDZcpUwZ+fn56bXNr2Nw5FMHBwfD09MS6devw4MEDk+3KlSuH8ePHAwD69++vq+EGDRoAALZt24YGDRogLS0NY8eOxeeff46UlBQ0adIEBw8eNFhft27d8PjxY0yaNAmtW7fGzJkz0b9/f938kydPom3btsjOzsb48eMxbdo0vPLKK/jjjz9MjpEKXnZ2Njw9PQ2me3l54cmTJzhx4gQA1q9D129hpzIyNGDAAF3SVyqVokuXLnp7SnJ3ky5ZssRg2WHDhgkA4vHjx0II7TucqKgog3aZmZkCgBgxYoTZsSxYsEAAEG5ubqJx48Zi9OjRYvfu3Qbv/s0dPmvatKmoWLGibkxCaA/h1alTR8TGxuqm5e4pqlatmnjy5Ilu+pQpUwQA3Z6lX375RQAQhw4dMjt2evEqVaokvLy8hJeXlxg8eLBYvXq1GDx4sAAgunfvrmu3cuVKAUDs2rXLYB1du3YVoaGhuucVKlQQTZo0MWh38uRJAUDMnTvX7JjGjBkjAAhvb2+RkJAgJk6cKI4cOWLQztThB41GI2JjY0XLli2FRqPRTc/KyhKlS5fWO7SR+077lVde0VvHu+++KwCIv/76SwghxFdffSUAiLt375odO71YFStWFGXKlNE7rSA7O1tERkYKAGLVqlVCCNavI9cv9xTJ0AcffIDNmzdj8eLFSEhIgFqtxpMnT3TzHz16BABwd3c3WNbDw0OvzaNHjyxqZ8qbb76JjRs3olGjRtizZw8+++wz1K9fH7Gxsdi7d2+e2/LgwQNs27YN3bp1Q3p6Ou7du4d79+7h/v37aNmyJc6fP48bN27oLdO/f3+4urrqng8cOBAuLi747bffAAABAQEAgF9//RVPnz7Ncwz04mRkZCArKwu9e/fGzJkz0alTJ8ycORMDBgzAf//7X5w/fx5A3jX8bF3mt4bHjRuHn376CfHx8fj999/x6aefolq1aqhatSpOnz6d5zYdO3YM58+fR8+ePXH//n1dDWdmZqJp06bYtWsXNBqN3jKDBg3Sez548GAAMKjhtWvXGixLhefdd9/FuXPn0K9fP5w6dQonTpxA7969kZycDED/7yrA+nXE+mUokqGyZcuiWbNm6N27N3799VdkZGSgXbt2usNHubt3s7OzDZbNPUSR28bT09Oidua0bNkSv//+O1JSUrBr1y4MGjQIV65cQdu2bXHnzh2zyyYlJUEIgdGjR6NYsWJ6j7FjxwKAwTpiY2P1nvv4+CAsLAyXL18GADRs2BCdO3fGuHHjULRoUbRv3x4LFy40up30YuXWU48ePfSm9+zZEwCwb98+vXamavPZurRFDffo0QO7d+/Gw4cPsWnTJvTs2RNHjx5Fu3bt9A7rGZMb5Pr06WNQw99//z2ys7ORmpqqt8zzNRwdHQ2lUqmr4VdffRV169bFW2+9hZCQEHTv3h0rVqxwuH8w9uadd97BJ598gp9++gkVKlRAxYoVceHCBQwfPhyA9m8RwPp15Pp1KewBUN66dOmCAQMG4Ny5c4iLi0NYWBgA6N69PCs5ORlBQUG6dyZhYWHYvn07hBBQKBR67QAgPDzc4nF4eXmhfv36qF+/PooWLYpx48Zhw4YN6NOnj8llcl8kH330EVq2bGm0TUxMjMVjAACFQoFVq1Zh//79WLduHX7//Xe8+eabmDZtGvbv36/7w0UvXnh4OE6ePImQkBC96cHBwQCAhw8fAkCeNfxsXYaFhRnsTXx2WWtq2M/PD82bN0fz5s3h6uqKxYsX48CBA2jYsKHJZXJreOrUqahSpYrRNnnV3LOvPUD7j3DXrl3Yvn071q9fj40bN2L58uVo0qQJNm3aBJVKZfE2kW1NnDgRH330EU6ePAl/f39UrFgRn3zyCQCgTJkyAFi/jly/3FNkB3J3r+am+eLFi6NYsWI4fPiwQduDBw/qFX6VKlWQlZVlsJv1wIEDuvlS5J5gmPvCfv5FkysqKgoA4OrqimbNmhl9+Pr66i2T+84mV0ZGBpKTk1GqVCm96bVq1cLEiRNx+PBhLF26FCdPnsR///tfSdtDtlGtWjUAMPgncPPmTQDak/IB4KWXXoKLi4tBDT958gTHjh0zqOFz584hLS1Nr+2LquHo6GgA2n9Ipmr42cO9gGENJyUlQaPR6NWwUqlE06ZNMX36dJw6dQoTJ07Etm3bsH37dknbQ7YTGBiIevXqoWLFigC0HyAoUaIEypYtC4D1Czhu/TIUyYixQ1FPnz7FkiVL4OnpifLly+umd+7cGb/++iuuXbumm7Z161acO3cOXbt21U1r3749XF1dMXv2bN00IQTmzp2L4sWLo06dOmbHtHXrVqPTc48tx8XFAdDuRQKAlJQUvXbBwcFo1KgR5s2bZ/RdlbErcM+fP1/vXKE5c+YgJycHCQkJALR7G3IPJebK/cPCQ2iFq1u3bgCABQsW6E3//vvv4eLigkaNGgEA/P390axZM/z4449IT0/Xtfvhhx+QkZGhV8NdunSBWq3G/PnzddOys7OxcOFC1KxZExERESbHk5WVpTtk97wNGzYA+LeGvb29ARjWcLVq1RAdHY0vv/zS6NWKjdXwrFmz9J5/8803AKCrYWOfJGINy9Py5ctx6NAhfPDBB1Aqtf8yWb+OW788fCYjAwYMQFpaGho0aIDixYvj1q1bWLp0Kc6cOYNp06bp7eL85JNPsHLlSjRu3BhDhgxBRkYGpk6diooVK6Jv3766diVKlMAHH3yAqVOn4unTp6hevTrWrFmD3bt3Y+nSpXnu5mzfvj1Kly6Ndu3aITo6GpmZmdiyZQvWrVuH6tWro127dgCgC23Lly9HmTJlEBQUhJdeegkvvfQSZs2apXvX9fbbbyMqKgq3b9/Gvn37cP36dfz11196fT558gRNmzZFt27dcPbsWcyePRv16tXDK6+8AgBYvHgxZs+ejY4dOyI6Ohrp6en47rvv4Ofnh9atW9vq10ESxMfH480338R//vMf5OTkoGHDhtixYwdWrlyJkSNH6h0qmDhxIurUqYOGDRuif//+uH79OqZNm4YWLVqgVatWunY1a9ZE165dMXLkSNy5cwcxMTFYvHgxLl++bBC+npeVlYU6deqgVq1aaNWqFSIiIpCSkqJ7DXTo0AHx8fEAtO+oAwICMHfuXPj6+sLb2xs1a9ZE6dKl8f333yMhIQEVKlRA3759Ubx4cdy4cQPbt2+Hn5+fwa14Ll26hFdeeQWtWrXCvn378OOPP6Jnz56oXLkyAGD8+PHYtWsX2rRpg5IlS+LOnTuYPXs2SpQogXr16tnq10FW2rVrF8aPH48WLVqgSJEi2L9/PxYuXIhWrVphyJAhem1Zvw5av4X62TfSs2zZMtGsWTMREhIiXFxcRGBgoGjWrJnBRQ5znThxQrRo0UJ4eXmJgIAA8dprr4lbt24ZtFOr1eLzzz8XJUuWFG5ubqJChQrixx9/tHhM3bt3F9HR0cLT01N4eHiI8uXLi08//VSkpaXptd27d6+oVq2acHNzM/h4/oULF0Tv3r1FaGiocHV1FcWLFxdt27bVfcRVCMOLNwYGBgofHx/x2muvifv37+va/fnnn6JHjx4iMjJSuLu7i+DgYNG2bVtx+PBhi7aJCtaTJ09EYmKiKFmypHB1dRUxMTHiq6++Mtp29+7dok6dOsLDw0MUK1ZMDBo0yKCuhNBeAfijjz4SoaGhwt3dXVSvXl1s3Lgxz7E8ffpUfPfdd6JDhw6iZMmSwt3dXXh5eYn4+HgxdepUgyu1r127VpQvX164uLgYfLz56NGjolOnTqJIkSLC3d1dlCxZUnTr1k1s3bpV1yb3I82nTp0SXbp0Eb6+viIwMFC89957ehe/27p1q2jfvr0IDw8Xbm5uIjw8XPTo0UOcO3cuz22igpOUlCRatGghihYtKtzd3UXZsmXFpEmTjF7RXwjWryPWr0KI545DEBWSRYsWoW/fvjh06JDRC1MSyV1iYiLGjRuHu3fvomjRooU9HCKrsH55ThERERERAIYiIiIiIgAMRUREREQAAJ5TRERERATuKSIiIiICwFBEREREBMAJL96o0Whw8+ZN+Pr6mrwsOpEtCSGQnp6O8PBw3RVx84M1TC8S65fsnTU17HSh6ObNm2Yvq05UUK5du4YSJUrkez2sYSoMrF+yd5bUsNOFotybj167dg1+fn6FPBpyBmlpaYiIiDC48a1UrGF6kVi/ZO+sqWGnC0W5u2v9/Pz4gqQXylaHCljDVBhYv2TvLKlhnmhNREREBIYiIiIiIgAMRUREREQAGIqIiIiIADAUEREREQFgKCIiIiICwFBEREREBIChiIiIiAgAQxERERERAIYiIiIiIgAyDEW7du1Cu3btEB4eDoVCgTVr1ujNf+ONN6BQKPQerVq1KpzBEhERkcOQXSjKzMxE5cqVMWvWLJNtWrVqheTkZN1j2bJlL3CERERE5Ihkd0PYhIQEJCQkmG3j7u6O0NDQFzQiIiIicgayC0WW2LFjB4KDgxEYGIgmTZpgwoQJKFKkiNG22dnZyM7O1j1PS0sDAKjVaqjV6hcyXnJu+a0z1jAVphddvwqFAkql7A5i6NFoNBBCFPYwyELW1LDdhaJWrVqhU6dOKF26NC5cuIBPPvkECQkJ2LdvH1QqlUH7SZMmYdy4cQbTz549Cx8fnxcxZHJyGRkZ+VqeNUyF6UXWr4eHB2JiYjBv3jwkJyfnq9+CEhYWhgEDBiApKQmPHz8u7OGQBaypYYWQcdxVKBT45Zdf0KFDB5NtLl68iOjoaGzZsgVNmzY1mG/sXUpERAQePHgAPz+/ghg2kZ60tDQEBQUhNTVVUs2xhqkwvej6ValUSExMxJUrV/I17oJSsmRJJCYmci+tHbGmhu1uT9HzoqKiULRoUSQlJRkNRe7u7nB3dzeYrlKpjO5ZIrK1/NYZa5gKE+vXOHseu7Ox5ncl7wO3Frh+/Tru37+PsLCwwh4KERER2THZ7SnKyMhAUlKS7vmlS5dw7NgxBAUFISgoCOPGjUPnzp0RGhqKCxcuYPjw4YiJiUHLli0LcdRERERk72QXig4fPozGjRvrng8dOhQA0KdPH8yZMwfHjx/H4sWLkZKSgvDwcLRo0QKfffaZ0d2zRERERJaSXShq1KiR2Y86/v777y9wNEREROQs7P6cIiIiIiJbYCgiIiIiAkMREREREQCGIiIiIiIADEVEREREABiKiIiIiAAwFBEREREBYCgiIiIiAsBQRERERASAoYiIiIgIAEMREREREQCGIiIiIiIADEVEREREABiKiIiIiAAwFBEREREBYCgiIiIiAsBQRERERASAoYiIiIgIAEMRkcPRaDSFPYQ82cMYicj5uBT2AIjItpRKJebNm4fk5OTCHopRYWFhGDBgQGEPg4jIAEMRkQNKTk7GlStXCnsYRER2hYfPiIiIiMBQRERERASAoYiIiIgIAEMREREREQCGIiIiIiIADEVEREREABiKiIiIiAAwFBEREREBYCgiIiIiAsBQRERERASAoYiIiIgIgAxD0a5du9CuXTuEh4dDoVBgzZo1evOFEBgzZgzCwsLg6emJZs2a4fz584UzWCIiInIYsgtFmZmZqFy5MmbNmmV0/pQpUzBz5kzMnTsXBw4cgLe3N1q2bInHjx+/4JESERGRI3Ep7AE8LyEhAQkJCUbnCSEwY8YMjBo1Cu3btwcALFmyBCEhIVizZg26d+/+IodKREREDkR2ocicS5cu4datW2jWrJlumr+/P2rWrIl9+/YZDUXZ2dnIzs7WPU9LSwMAqNVqqNXqgh80Ob381pm1NaxSqfLV34vC1599YP0ax/q1H9b8ruwqFN26dQsAEBISojc9JCREN+95kyZNwrhx4wymnz17Fj4+PrYfJNFzMjIy8rW8NTXs4eGBmJiYfPX3oly6dImHve0A69c41q/9sKaG7SoUSTFy5EgMHTpU9zwtLQ0RERGIi4uDn59fIY6MnEXuO2OpHLWGS5cuXdhDIAuwfo1j/doPa2rYrkJRaGgoAOD27dsICwvTTb99+zaqVKlidBl3d3e4u7sbTFepVHazm5bsW37rzFFr2J7H7kxYv8bZ89idjTW/K9l9+syc0qVLIzQ0FFu3btVNS0tLw4EDB1C7du1CHBkRERHZO9ntKcrIyEBSUpLu+aVLl3Ds2DEEBQUhMjISH3zwASZMmIDY2FiULl0ao0ePRnh4ODp06FB4gyYiIiK7J7tQdPjwYTRu3Fj3PPdYdJ8+fbBo0SIMHz4cmZmZ6N+/P1JSUlCvXj1s3LgRHh4ehTVkIiIicgCyC0WNGjWCEMLkfIVCgfHjx2P8+PEvcFRERETk6OzqnCIiIiKigsJQRERERASGIiIiIiIADEVEREREABiKiIiIiAAwFBEREREBYCgiIiIiAsBQRERERASAoYiIiIgIAEMREREREQCGIiIiIiIADEVEREREABiKiIiIKA9qTWGPIG+2GKNL/lfhWDQaDZRKeWdFexgjEZGzExo1FEpVYQ/DLEvHqFICvXYBZ1JfwKAkKOsP/NAg/+thKHqOUqnEvHnzkJycXNhDMSosLAwDBgwo7GEQEVEeFEoVcn4eDkXZplCWaw71rjkQZ7eZXUYZ3xmq6j2hPvQTNEdXm19/XBOoGgyE5vRmaPbMN982sipUzYZDXPsT6q3TAU0OFEWj4NJpqsXbcyYVOPrA4uZ2SXIoSktLw+zZs7F9+3bcuXMH8+bNQ40aNfDgwQMsWrQIr7zyCmJiYmw51hcmOTkZV65cKexhEBGRndMFonWjIY79bLatsv472kC0/Wtods81v94qnbSB6MgKaH4bD0CYbhvbEKpmwyCSdkK9aiigeQqYXcJ5SToGc/36dcTHx2PMmDG4fv06jh8/joyMDABAUFAQ5s2bh2+++camAyUiIrI3VgWixkMsD0TtPrM8EHWdCZG0Sy8QSTG/DrCjFRDuZXz+VzWAkx2AdhHG57eL0M7/qobx+cW9tOvPTx/5JSkUDRs2DOnp6Th27Bh27twJIfR/IR06dMCWLVtsMkAiIiJ7pd41xyECEQDEFwHqhQD9Yg3nlfMHBpcD4vyBSdWMLz+5mnb+4HLa9s/rF6tdf376yC9JoWjTpk14//33Ub58eSgUCoP5UVFRuHbtWr4HR0REZM/yPIfITgIRAORotI8/bhvOu5oJXNYeMMLmm8aX3/TP9MsZ2vbP23M7/33kl6Rzih49eoRixYqZnJ+eni55QERERM7AngIRALTfCpxIAW5kGc7LzAEqrwUivYHTJj6h9uFBYP5ZbbjJzDGcv+0WEP3PueVS+8gvSXuKypcvj127dpmcv2bNGsTHx0seFBERkSOTQyBSFLPuw1B3HhsPK7kyc/IOK6dTjQeiXDey8t9HfkgKRR988AH++9//4osvvkBqqnZ0Go0GSUlJ6NWrF/bt24cPP/zQpgMlIiJyBLIIROEVoWo9RuomOCxJh89ef/11XLlyBaNGjcKnn34KAGjVqhWEEFAqlfj888/RoUMHW46TiIjI7skmEL2+AOLhVShCy0ndFIck+TpFn376KXr16oXVq1cjKSkJGo0G0dHR6NSpE6Kiomw5RiIiIrsnq0B09zzUW6ZB2Xep1M1xSPm6onVkZCQPkxEREeVBdoFo6dtAUCmJW6PVrRRQ2geYdQbIMHKeUKVAoGMk8MtV4PhDw/k+LsCgssClDGDF5fz1YavzjCSFoj///BP79+/Hu+++a3T+7NmzUadOHVSpUiU/YyMiIrJ7sgxET8yczWyBRqHATw2135fwBgYfeK4/AJtbAEU8gHfLAqHLDbdkUjVgYFnt97cfATuf+xi+NX2kZOdrc3QknWj96aefmr0447Zt2zBq1CjJgyIiInIEyvjODheIAEC8gHuEvIg+nidpT9GRI0cwcuRIk/Pr16+PSZMmSR4UERGRI7DqXmZ2EogA7V6dnjv/PbT1PAGg+SagQySw5qrxrRl5BLieqT189vxeImv7OJP6716l/JAUitLT0+HiYnpRpVKp+6g+ERGRs1If+snhAlEuU+cB5Tr+0Pi5RLkycoAvTtimj/gg8+0sJenwWWxsLDZt2mRy/saNG/kJNCIicnqao6vNzrfXQOSoJIWifv36Yf369Rg6dChSUlJ001NSUvDhhx9i48aN6Nevn63GSERE5HAYiORH0uGz999/H8eOHcOMGTMwc+ZMhIeHAwBu3rwJjUaDXr168aP6REREJsgiEPmH5WMLHJOkPUUKhQILFy7E1q1b8c477+Cll17CSy+9hIEDB2Lbtm1YvHgxFAqFrccKAEhMTIRCodB7lC1btkD6IiIisjVZBKKgUnBpM86qcQd7AMW9TM/3cQHK+ZtfRzl/wNvM7pjiXvnvIz/ydfHGxo0bo3HjxrYai8UqVKigd0kAcyd9ExERyYVsAlHvRcDTLABFLB772qbar603a+9o/ywfF+BYe6CUD/DNaeDDg4bLf1UDGFwOuJyhvdv98zeGbRIK/NY8f33kl6Q9RYXNxcUFoaGhukfRokULe0hERERmySoQZacj59dEq8bvotQ+6oYYzovw1oYVAGgebnz5Fv9ML+UDRHobzq8Xkv8+8kvSLhYhBObPn48FCxbg4sWLePjQ8DN3CoUCOTlGrsltA+fPn0d4eDg8PDxQu3ZtTJo0CZGRkUbbZmdnIzv730tdpqWlAQDUajXUarVBe5VKVSBjtjVjYyd5yu/vijVMhYn1a5wlP5dnt0V2gWhJX8A32KptOXpfu3dnwXnD+adTtXtvmodrrz9kzIgj2qtYb75p/LYcC84DTcL+/V5KH8a2xZoalhSKhg8fjunTp6NKlSp4/fXXERgYKGU1ktSsWROLFi1CXFwckpOTMW7cONSvXx8nTpyAr6+vQftJkyZh3DjD46Znz56Fj4+P3jQPDw/ExMQU2Nht6dKlS3j8+HFhD4MskJGRka/lWcNUmFi/xuVVv89uiywDUeY9XSiydFv67wWOPjDdRV6Hs9Zd0z5MuZEFNNpofh159WFsW6ypYUmhaPHixejcuTNWrFghZfF8SUhI0H1fqVIl1KxZEyVLlsSKFSuMXgZg5MiRGDp0qO55WloaIiIiEBcXBz8/vxcy5oJQunTpwh4CWSj3nbFUrGEqTKxf4yytX0VcE6gaDJRfIJKwLfbA2LZYU8OSQtGjR4/QrFkzKYvaXEBAAMqUKYOkpCSj893d3eHu7m4wXaVS2c1uWmPseezOJr+/K9YwFSbWr3GWjl3ugciabbEHxrbFmu2TdKJ106ZNcejQISmL2lxGRgYuXLiAsDBeb4GIiORFc3qzrAMR6ZMUimbPno39+/fj888/x/379209JrM++ugj7Ny5E5cvX8bevXvRsWNHqFQq9OjR44WOg4iIKC+aPfPBQGQ/JIWiuLg4XLx4EaNHj0ZwcDC8vb3h5+en9/D3L5irK12/fh09evRAXFwcunXrhiJFimD//v0oVqxYgfRHRIVHaOT/CTWh0RT2EMhO2XMg8lQBqxoDf7YzfTPWIeWBUx20X42pGqRdflVj7fry00cPG91uVdI5RZ07dy6wK1bn5b///W+h9EtEL55CqULOz8Mg7l38d6J/mPZKvE+ztNdZeZRiegWeAXBpmwi4eiFn/VggNdl0W1cPqBJGQREYCfVv4yHuGj9PEQCgdIGq6VAoIqtBoeLFY8l6sghEngGSx9+6BNDhnyvhDK8I9NipP1+pAKZUA1RK7ddvTgOa53aYDasIVArSPhJKAD9fkd7HEMMPn0si6dW8aNEi2/RORJQHce8icOuU9klQKbgkjAYePcz7D753Ubh0/AJQuSFn0evAg8um27p5QfXad1AElID6h74QN/823VbpClWX6VBExEO9+Qu4tPpU0naR85JFIPIuqn3DINHRB0D6U8DXFdhu5L2GRgB77gANQ7Vfnw9EgHa5rqW06zlm5KP+1vRx7AFQzQbXceZbHCKyD9b+we+9EHD3Rc6SNywLRMViof6xn2WBKKYB1Cvfh0i/K3FjyFnJJhD1Xgi4mrnJWB4upgPRq4FAN+BCuvE2rTYDsX7AeROfiJ9/DtiaDDx8AjzINpxvTR/eLsC+NtK25VmSb/Nx9epVvPPOO4iLi0NgYCB27doFALh37x7ef/99HD16NP+jIyIC5BmIzu803dYItcxPPZL7+ByBrAKRu6/2kHI+PMg2HVYA4KkGOJWi/WrKhXTjgcjaPnJsVL+S9hSdOnUK9evXh0ajQc2aNZGUlKS7pUfRokWxZ88eZGZmYsGCBbYZJRE5L/8w7SEzOw5EgPa8h167gDNGbm9Q2Mr6Az80KOxRODbZBaIlbwBu0vcUOSrJt/kICAjA/v37oVAoEBwcrDe/TZs2WL58uU0GSETOzaXNOMvPIZJpIMp1JtX8bRLIMckyED24DISa+FiYE5N0+GzXrl0YOHAgihUrZvRTaJGRkbhx40a+B0dEhKdZDhGIyDkpIqvKMxCRUZJCkUajgZeX6d1ud+/eNXpZdyIia+X8mugwgUipAAbGaR8qE1c1aRIKjKkMFDfxJ7a4l3Z+k1Dj81U26INsR9VsuMMEoi4lHb92JR0+q1q1KtavX493333XYF5OTg7++9//olatWvkeHBGR2esQ2VEgAoBOkcDHlf59Pues/vziXsBvzQEXJdAkzPgdw5c2AOqFaE8sjV6tvbP4s/qXAb555s+vlD7IdsS1Px0iEAHOUbuS9hSNHDkSGzduxMCBA3HixAkAwO3bt7Flyxa0aNECp0+fxogRI2w6UCIiPXYWiMg5qbdOd4hA5Cwk7SlKSEjAokWLMGTIEMyfPx8A8Prrr0MIAT8/PyxZsgQNGvCjDERUQGQRiBRQ1utv1bB/vgpc/+d/3vxzhvNvZAGtNwN1Q4AF542vo+cuoF8s8Mdtw3faz69Xah9kQ5ock7PsLRB9cVxbv45cu5Iv3tirVy906tQJmzdvxvnz56HRaBAdHY2WLVvC19dG19smInqeXAJR6zFQlmtu1dA1wvCQwPO23dI+TLmZBXz2l+n5ahv0QQVPFoHI1cOqMa+6Yv7Tk45Qu1aHoqysLERERGDEiBEYNmwYOnToUADDIiIyQk6BqFo3qHfNgUvDQRI3hpyVLAKRmxdUCaMkboHjsvqcIi8vL7i4uMDb27sgxkNEZJzcAtG60RBnt0ncGHJWsglEr30HRWCkxK1wXJJOtO7cuTNWrVoFIYzc4Y1kQ2jUhT2EPNnDGEkG5BiIjv0scWPIWckqEBWLhfq38RK3xHFJOqeoe/fuePfdd9G4cWO8/fbbKFWqFDw9PQ3aVa1aNd8DJOkUShU0N09AERwH9ZYpEFf/NNteWa8/lOWaQ71rTp7vgJXxnaGq3hPqQz9Bc3S1+XHENYGqwUBoTm+GZs/8f6cXjYJLp6mWbxA5J88A7d3uHTQQFXEHAszc8NJV+e9NNU3dQyraF0h5Atw3cQ+pvPpwkXwXTLKU7ALRj/3y/aZUDrWb24etalhSKGrUqJHu+927dxvMF0JAoVBAreZegMKmCI6DeuVgi08ateQPvrL+O9pAtP1raHbPNd9/lU7aQHRkBTS/jQfw795F7mckS7i0TQRUbg4ZiKJ8gSPtAF9X4N19xj9xs7E50DAU2HkLaPq74fz+ZYDZtYH0p0C1ddo7i1vbx7c1bbM9ZJwsA9HNv/N1mw+51G5uH0fM/JisISkULVy40Da9U4FTb5li0z/4yvrvQNV4iOWBqN1nRgMRkcVcvZCz6HWHC0QAEB+k/YMPAI3DDP/oKxVAvX9uLVkvWPtc89zLqHGY9quvK1AlyPAfiyV9VAnK/7aQcYpiMVAljJJfIMonudRubh+2qmFJoahPnz626Z0KnPlDZgxEJH8568c6ZCACgN+uA2uuAlE+wBQjw9YIYPgR7Tvq+ecM/6kA2uXi/ICLGcCG69L6+Po0MLRC/reHDKlaj4G4c9ahAhEgn9rN7WP9ddvUsOTrFOVKTk7GnTt3EBMTw0+k2RUGIrITqcmm59lxIAKAR2qgy3bzbb4+pX2YcvQBUHVd/vpYdpGhqKCIh1cdLhAB8qnd3D7ig2xTw5JPTVq7di3Kli2LEiVKoGrVqjhw4AAA4N69e4iPj8cvv/yS/9FRAZFHIFJEWn4ivtrESXpyYg9jdCh2HojIOag3THC4QOTIJO0pWrduHTp16oTatWujZ8+eSExM1M0rWrQoihcvjkWLFqFjx462GifZjEwCUWxDqJoNt3jUKiXQaxdwJtXiRV6osv7AD7yzzYsjk0CkjO8sbfzkPJ4+Nj2PgUh2JIWi8ePHo0GDBti+fTvu37+vF4oAoHbt2pg3b54txkc2JaNA1HUmxLU/oSht+cdezqSav8Q8OQm5BKJ/PoVJJIkcApEy32fQOBxJh89OnDiBbt26mZwfEhKCO3fuSB4UFQSZBaKkXdq7RxNZQ06BqPEQqA/9JG07yLnJIhC5QtV0qMQNcFySQpGXlxcyMzNNzr948SKKFCkieVBkazIMRKuGmr17tDnFvYAdrbSPcC/jbb6qAZzsALSLMD6/XYR2/lc1Cq4PsjG5BaLtX+d54dLnBXuwdp2eXAJRl+lQRFh3geX5dRy/diWFosaNG2Px4sXIyTH8p3br1i189913aNGiRb4HR7Yhz0D0VMqmAAD6xQL1QrSPfrGG88v5A4PLAXH+wKRqxtcxuZp2/uBy2vYF0QfZkBwDUR6vD2PaR7J2nZqcAlFMA6i3TLFq+PFFHL92JYWiiRMn4vr166hevTrmzZsHhUKB33//HaNGjULFihUhhMDYsWNtPVaSQFmvv0MFIgDYcxvI0Wgff9w2nH81E7icof1+803j69j0z/TLGdr2BdEH2Yirh0MEIgA4dp+167TkFohWvp/nrZ+e5wy1K+ksq7i4OOzZswdDhgzB6NGjIYTA1Knae1g1atQIs2bNQqlSpWw5TpLIqlt32EEgAoBtt4Dof45a3DDySdfMHKDyWiDSGzht4tNqHx4E5p/VvsgyjRzFs7aPeF4RuMCoEkZBEVDC7gMRABy6L7/apRdAjoHo/E6rb/PRfitwIsWxa9eiUHT8+HGULFkS/v7/7u+qUKECtmzZgocPHyIpKQkajQZRUVEoVqxYwYyUJFHvmuNQgSiXsRfMszJz8n7R5DXfFn1Q/ikCI6H+oa/dB6JcrF0n4x8Gl4TR8gtEEtx5bL62HKF2LTp8Fh8fj/Xr1+ueN2nSBFu3bgUABAYGonr16qhZsyYDkQzlebd7OwxE5FzUv413mEBEzselzTiHCETOwqJQ5Onpiaysf+Pbjh07cPu2kQN+ZFcYiMgeiLtJpmcyEJHcPc1iILIjFh0+q1y5MqZPnw6VSqU7hHbo0CF4eHiYXa5Tp075HyEVCDkEIkWxGKnDJ2IgIruQ82siA5EdsSgUzZgxA127dkW/fv0AAAqFAl9//TW+/vprk8soFAqo1WrbjJJsShaBKLwiVK3HWDXu5uGmr2jdrRRQ2geYdQbIMHICX6VAoGMk8MtV4PhDw/k+LsCgssClDGDFZWl90Askk0CkiGsiYfD65FC7sX752AAy71GK6Xl2HojkULu5fdjqPCOLQlH16tWRlJSECxcu4Pbt22jUqBE++eQTNG/e3DajoBdGNoHo9QUQD69CEVrO4rF/Xg04cBfY+dyR20ahwE8Ntd+X8AYGH3iuPwCbWwBFPIB3ywKhyw23ZFI1YGBZ7fe3H0nrg14QuQSiKp2gajBQwgb8Sy61O7tWvjaDpLDzQCSX2s3tIyXbBhsFC0PR//73P7z88suIi4tDXFwc+vTpg3bt2qFmTcvvW2Vrs2bNwtSpU3Hr1i1UrlwZ33zzDWrUMHGZTAIgs0B09zzUW6ZB2Xep1M3REaaHZjMvog+ygJwCUbvPoDm9GaryLSVsiBZr10nJIhApoKzXX+IGOG7tWhSKOnbsiB9++AE9e2pvfrhz50507NixQAdmzvLlyzF06FDMnTsXNWvWxIwZM9CyZUucPXsWwcHBhTYuOZNdIFr6NhBUyqpt+OSI4TsJQDut585/d7E+TwBovgnoEAmsuWp8a0YeAa5nanfjSu2DCpjcAtGRFdD8uTJfoUgutfvu/n/fkVMBk0sgaj0GynLSj/bIpXZz+ziTapsatigU+fr6IiUlRff88uXLyMjIyH/vEk2fPh1vv/02+vbtCwCYO3cu1q9fj//85z8YMWKEXtvs7GxkZ/+7Xy01VXvg8eHDh0bPeVKpVAgICNBbRk4CAgKQlpZm0flaKpUKOV7FoaifAFW1blBv/x6a41sBfyPXT/+HIrYBVHXfgubQGmj2/QT4mz4ZWlGiMlSNP4A4tQfqnbMB31Km2xaNgqrFxxDXzkC9eSrgWRwKr+JwsWJbLtwBKpk4t//MLeAMgCgXGK/qR8Cas9pvTa1jQ5L5+eb6iHIB0tJgdFvS0tIAAELi2x7WcHGIQAFVw/egCK4M9drPIO7cNFvHytpvQBnbBOoN0yEu/W2+baVXJL8+FEVKWVXDUS6A+rn6KuzaBQB1FuvXUpLq1/+fN4ue/nBpORJ46oKcX0cDalfT9ebqAVXzYVB4hEG9ejRE5mPTbZUukl4fml1LoIrvJLl+5VC7uX3E+tmmhhXCglYtWrTAyZMn8dZbb8Hf3x8fffQRevbsiapVTd9MTqFQ4MMPP8xzANZ68uQJvLy8sGrVKnTo0EE3vU+fPkhJScHatWv12icmJmLcuHE2HweRta5du4YSJUpYvRxrmOSA9Uv2zpIatigUJSUloXfv3ti/f792IYUiz8RVUJ8+u3nzJooXL469e/eidu3auunDhw/Hzp07ceCA/plYz79L0Wg0ePDgAYoUKQKFgqfHUsETQiA9PR3h4eFQKq2/3SBrmAoT65fsnTU1bNHhs5iYGOzduxePHz/GnTt3UKpUKcyYMQPt27e3yYALkru7O9zd3fWmBQQEFM5gyGk9e4sca7GGqbCxfsneWVrDVt0Q1sPDA5GRkRg7diyaNGmCkiVLShpcfhQtWhQqlcrgitq3b99GaGjoCx8PEREROQbr94UCGDt2LF566SVbj8Uibm5uqFatmu7ea4B2d+zWrVv1DqcRERERWcOiPUVvvvkmFAoF5s+fD5VKhTfffDPPZRQKBRYsWJDvARozdOhQ9OnTBy+//DJq1KiBGTNmIDMzU/dpNCIiIiJrWRSKtm3bBqVSCY1GA5VKhW3btuV5glxBnkD36quv4u7duxgzZgxu3bqFKlWqYOPGjQgJCSmwPomIiMixWfTpMyIiIiJHJ+mcIiIiIiJHY9Wnz3Klp6djz549uHDhAtLT0+Hr64uYmBjUq1cPPj4+th4jERERUYGzKhSp1WqMGjUK3377LbKysvQu4KhQKODl5YUhQ4Zg/Pjxki7yRURERFRYrDqnqHv37lixYgXKly+PHj164KWXXoKPjw8yMjLw999/46effsLZs2fRo0cP/PjjjwU5biIiIiKbsjgUbdmyBS1atMCgQYPw9ddfG90TpNFoMHjwYMydOxebN29GkyZNbD5gIiIiooJgcSjq3bs39u3bh3Pnzpn9uL1Go0FcXBzq1KmDxYsX22ygRERERAXJ4hN/Dh48iE6dOuV5/SGlUolOnToZ3JiViIiISM4sDkXJycmIiYmxqG1MTAySk5MlD4qIiIjoRbM4FGVkZMDb29uitl5eXsjIyJA8KCIiIqIXzeJQJIQo0Ft3EBERERUmi0+0ViqViIiIgL+/f55tU1NTcf36dajV6nwPkIiIiOhFsPjijQ0aNLB4T1GRIkUQFRUleVBERERELxpvCEtEREQE3hCWiIiICABDEREREREAhiIiIiIiAAxFRERERAAYigpNRkYGxo4di1atWiEoKAgKhQKLFi0yaKfRaLBo0SK88soriIiIgLe3N1566SVMmDABjx8/NrruBQsWoFy5cvDw8EBsbCy++eYbo+1u3LiBbt26ISAgAH5+fmjfvj0uXrxo0fifPHmCr7/+GvHx8fDz80NAQAAqVKiA/v3748yZM7p2e/fuRWJiIlJSUixaL9kHS+sXAN544w0oFAqDR9myZQ3aajQaTJkyBaVLl4aHhwcqVaqEZcuWGV3v6dOn0apVK/j4+CAoKAi9evXC3bt3rRr/Sy+9BG9vbxQpUgRVqlTBkCFDcPPmTV273377DYmJiRatk+zLyZMn0bVrV0RFRcHLywtFixZFgwYNsG7dOqPtLa031rCdE1QoLl26JACIyMhI0ahRIwFALFy40KBdenq6ACBq1aolJkyYIObPny/69u0rlEqlaNSokdBoNHrt586dKwCIzp07i/nz54tevXoJAGLy5MkG642NjRXBwcHiiy++ENOnTxcRERGiRIkS4t69e3mOv23btkKlUonXX39dzJo1S8yYMUO88847okSJEnrbMXXqVAFAXLp0ScqPiWTK0voVQog+ffoId3d38cMPP+g9/ve//xm0HTFihAAg3n77bTF//nzRpk0bAUAsW7ZMr921a9dE0aJFRXR0tPj666/FxIkTRWBgoKhcubLIzs42O/YnT56I+Ph44enpKd555x0xd+5c8eWXX4q+ffuKokWLiu3bt+vaDho0SPDPpGNav369aNmypUhMTBTz588XM2bMEPXr1xcAxLx58/TaWlNvrGH7xp9UIXn8+LFITk4WQghx6NAhk/9UsrOzxR9//GEwfdy4cQKA2Lx5s25aVlaWKFKkiGjTpo1e29dee014e3uLBw8e6KZ98cUXAoA4ePCgbtrp06eFSqUSI0eONDv2gwcPCgBi4sSJBvNycnL0QhVDkWOytH6F0IYib2/vPNd5/fp14erqKgYNGqSbptFoRP369UWJEiVETk6ObvrAgQOFp6enuHLlim7a5s2bjf5De96KFSsEALF06VKDeY8ePRKpqam65/yH4lxycnJE5cqVRVxcnN50S+uNNWz/LPpJlSpVSpQuXdqqR1RUVEGP3WHk9U/FmOPHjwsAYubMmbpp69evFwDE+vXr9dru3btXABA//PCDblr16tVF9erVDdbbokULER0dbbbvZcuWCQBix44dZtuNHTtWADB4PBuQfvjhB1G1alXh4eEhAgMDxauvviquXr2qt56GDRuKChUqiMOHD4vatWsLDw8PUapUKTFnzhyDPmfOnCnKly8vPD09RUBAgKhWrZrRPxxkO5aGopycHL0/1s+bNWuWACBOnjypN/2nn34SAMTu3bt104KDg0XXrl0N1lGmTBnRtGlTs+OdNGmSACAuX75stl2fPn2M1m8utVotvvrqK1G+fHnh7u4ugoODRf/+/fXefAghRMmSJUWbNm3E77//LipXrizc3d1FuXLlxOrVq/XaPXnyRCQmJoqYmBjh7u4ugoKCRN26dcWmTZvMjpNsq23btiIkJERvmqX1xhq2/xq26IrWDRs25H3PZObWrVsAgKJFi+qmHT16FADw8ssv67WtVq0alEoljh49itdffx0ajQbHjx/Hm2++abDeGjVqYNOmTUhPT4evr6/RvkuWLAkAWLp0KerWrQsXF+Nl1KlTJ5w7dw7Lli3DV199pRtrsWLFAAATJ07E6NGj0a1bN7z11lu4e/cuvvnmGzRo0ABHjx5FQECAbl0PHz5E69at0a1bN/To0QMrVqzAwIED4ebmptuO7777Du+//z66dOmCIUOG4PHjxzh+/DgOHDiAnj175vkzpYKTlZUFPz8/ZGVlITAwED169MAXX3wBHx8fXZujR4/C29sb5cqV01u2Ro0auvn16tXDjRs3cOfOHYM6z23722+/mR1Lbv0uWbIEo0aNMvm3bcCAAbh58yY2b96MH374wej8RYsWoW/fvnj//fdx6dIlfPvttzh69Cj++OMPuLq66tqeP38er776Kt555x306dMHCxcuRNeuXbFx40Y0b94cAJCYmIhJkybhrbfeQo0aNZCWlobDhw/jzz//1LUh28vMzMSjR4+QmpqK//3vf9iwYQNeffVV3Xxr6o017AA1XNipjKTtKWrWrJnw8/MTDx8+1E0bNGiQUKlURtsXK1ZMdO/eXQghxN27dwUAMX78eIN2ue90zpw5Y7JvjUYjGjZsKACIkJAQ0aNHDzFr1iy93cC5TB0+u3z5slCpVAaH4P7++2/h4uKiNz23r2nTpummZWdniypVqojg4GDx5MkTIYQQ7du3FxUqVDA5bioYedXviBEjxMcffyyWL18uli1bpnv3WrduXfH06VNduzZt2hjdw5yZmSkAiBEjRuj1t2TJEoO2w4YNEwDE48ePTY43KytLxMXFCQCiZMmS4o033hALFiwQt2/fNmhr6tDD7t27jR6+2Lhxo8H0kiVLCgB676pTU1NFWFiYiI+P102rXLmywaFvKngDBgzQ7UFRKpWiS5cuentKrKk31rD91zA/fWaHPv/8c2zZsgWTJ0/W25vy6NEjuLm5GV3Gw8MDjx490rUDAHd3d6Ptnm1jjEKhwO+//44JEyYgMDAQy5Ytw6BBg1CyZEm8+uqrFn3S7Oeff4ZGo0G3bt1w79493SM0NBSxsbHYvn27XnsXFxcMGDBA99zNzQ0DBgzAnTt3cOTIEQBAQEAArl+/jkOHDuXZP704kyZNwuTJk9GtWzd0794dixYtwsSJE/HHH39g1apVunaPHj2yqCbzW7+enp44cOAAhg0bBgBYtGgR+vXrh7CwMAwePBjZ2dl5btPKlSvh7++P5s2b69VvtWrV4OPjY1C/4eHh6Nixo+65n58fevfujaNHj+r2+gYEBODkyZM4f/58nv2T7XzwwQfYvHkzFi9ejISEBKjVajx58kQ335p6Yw3bfw3nKxQ9ffoUf//9N/bs2YNdu3YZPMj2li9fjlGjRqFfv34YOHCg3jxPT0+9F/OzHj9+DE9PT107AEZfOLkf889tY4q7uzs+/fRTnD59Gjdv3sSyZctQq1YtrFixAu+9916e23H+/HkIIRAbG4tixYrpPU6fPo07d+7otQ8PD4e3t7fetDJlygAALl++DAD4+OOP4ePjgxo1aiA2NhaDBg3CH3/8kedY6MX78MMPoVQqsWXLFt00T09Pi2rSFvXr7++PKVOm4PLly7h8+TIWLFiAuLg4fPvtt/jss8/yHP/58+eRmpqK4OBgg/rNyMgwqN+YmBiDQxzP1+/48eORkpKCMmXKoGLFihg2bBiOHz+e51gof8qWLYtmzZqhd+/e+PXXX5GRkYF27dpB/HNbUGvqjTVs/zVs0TlFz9NoNBg5ciRmz56NrKwsk+3UarXkgZGhzZs3o3fv3mjTpg3mzp1rMD8sLAxqtRp37txBcHCwbvqTJ09w//59hIeHAwCCgoLg7u6O5ORkg3XkTstta4mwsDB0794dnTt3RoUKFbBixQosWrTI5LlGgLaGFAoFNmzYAJVKZTD/2XNNLFWuXDmcPXsWv/76KzZu3IjVq1dj9uzZGDNmDMaNG2f1+qjgeHp6okiRInjw4IFuWlhYGLZv3w4hhN4f3+drMiwsTG/6s5KTk3X1bamSJUvizTffRMeOHREVFYWlS5diwoQJZpfRaDQIDg7G0qVLjc7PPW/OGg0aNMCFCxewdu1abNq0Cd9//z2++uorzJ07F2+99ZbV6yNpunTpggEDBuDcuXOIi4uzqt5Yw/Zfw5JC0eeff46pU6diwIABqFevHnr16oUvvvgCAQEBmD17NhQKBaZMmWLrsTq1AwcOoGPHjnj55ZexYsUKo4GjSpUqAIDDhw+jdevWuumHDx+GRqPRzVcqlahYsSIOHz5stJ+oqCiTJ1mb4+rqikqVKuH8+fO6Q2GmTgCMjo6GEAKlS5fWvdsw5+bNm8jMzNTbW3Tu3DkAQKlSpXTTvL298eqrr+LVV1/FkydP0KlTJ0ycOBEjR47U7Zamwpeeno579+7p/eGtUqUKvv/+e5w+fRrly5fXTT9w4IBuPgAUL14cxYoVM1q/Bw8e1LWzVmBgIKKjo3HixAndNHP1u2XLFtStWzfPd/QAkJSUZPCP0lj9BgUFoW/fvujbty8yMjLQoEEDJCYm2s0/FEeQe9gqNTUVgHX1xhp2gBqWciJSdHS0ePXVV4UQQty7d08oFAqxdetWIYT2BNiqVavmea0b+ldeJ6qeOnVKFClSRFSoUMHgo5LPysrKEkFBQaJt27Z6019//XXh5eUl7t+/r5s2efJkAUAcOnRIN+3MmTNCpVKJjz/+2Ox4z507Z/Sk6ocPH4rw8HARGBioux7HnDlzBABx9OhRvbZJSUlCpVKJnj17GlyAUqPR6F3ryNyJ1sWKFdOdaG3sopPDhg0TSqVSpKWlmd0mks5c/T569Mjozz73ZNKff/5ZN+3atWsmr/FSvHhxvWu8vPPOO8LT01Pv8g1btmwRAIxequFZx44dE3fv3jWYfvnyZeHp6SkqVaqkm/bxxx8LAHofaBBCiB07dggARv/OPX36VK+9uZNUq1SpoptmrH67du0qihYtanZ7SBpjJyU/efJEVK1aVXh6eor09HTddEvrjTVs/zUsaU/R9evXMXz4cAD/niiWexzUzc0Nr7/+OqZPn47PP/9calZzCt9++y1SUlJ0l2Rft24drl+/DgAYPHgw/P39kZ6ejpYtW+Lhw4cYNmwY1q9fr7eO6Oho1K5dG4D2kMRnn32GQYMGoWvXrmjZsiV2796NH3/8ERMnTkRQUJBuuXfffRffffcd2rRpg48++giurq6YPn06QkJC8H//939mx/3XX3+hZ8+eSEhIQP369REUFIQbN25g8eLFuHnzJmbMmKE7JFatWjUAwKefforu3bvD1dUV7dq1Q3R0NCZMmICRI0fi8uXL6NChA3x9fXHp0iX88ssv6N+/Pz766CNdn+Hh4fjiiy9w+fJllClTBsuXL8exY8cwf/583cdGW7RogdDQUNStWxchISE4ffo0vv32W7Rp00bSni8yz5L6vXXrFuLj49GjRw/dbT1+//13/Pbbb2jVqhXat2+vW1+JEiXwwQcfYOrUqXj69CmqV6+ONWvWYPfu3Vi6dKneYdZPPvkEK1euROPGjTFkyBBkZGRg6tSpqFixIvr27Wt23Js3b8bYsWPxyiuvoFatWvDx8cHFixfxn//8B9nZ2Xq3RMit3/fffx8tW7aESqVC9+7d0bBhQwwYMACTJk3CsWPH0KJFC7i6uuL8+fNYuXIlvv76a3Tp0kW3njJlyqBfv344dOgQQkJC8J///Ae3b9/GwoULdW3Kly+PRo0aoVq1aggKCsLhw4exatUqi87RI+sNGDAAaWlpaNCgAYoXL45bt25h6dKlOHPmDKZNm6Z3CN/SemMNO0ANS0lS4eHheu/a/f39xTfffKN7Pn36dIuuYOvsctO3sUfuR9hzb6dg6tGnTx+D9c6fP1/ExcUJNzc3ER0dLb766iuDvTFCaN/VdOnSRfj5+QkfHx/Rtm1bcf78+TzHffv2bTF58mTRsGFDERYWJlxcXERgYKBo0qSJWLVqlUH7zz77TBQvXlwolUqDj+evXr1a1KtXT3h7ewtvb29RtmxZMWjQIHH27FldG2MXbyxZsqT49ttv9fqZN2+eaNCggShSpIhwd3cX0dHRYtiwYWYvGEjSWVK/Dx8+FK+//rqIiYkRXl5ewt3dXVSoUEF8/vnnuj18z1Kr1eLzzz8XJUuWFG5ubqJChQrixx9/NNr/iRMnRIsWLYSXl5cICAgQr732mrh161ae47548aIYM2aMqFWrlggODhYuLi6iWLFiok2bNmLbtm16bXNycsTgwYNFsWLFhEKhMPho8/z580W1atWEp6en8PX1FRUrVhTDhw8XN2/e1Ps55V74rlKlSsLd3V2ULVtWrFy5Um9dEyZMEDVq1BABAQHC09NTlC1bVkycONHoz4nyb9myZaJZs2YiJCRE9zesWbNmYu3atUbbW1pvrGH7rmGFEP+cYm+Ftm3bwtfXV3eTu44dO+Lvv//G4sWLodFo0Lt3b4SHh/OTP2QTjRo1wr179/SOkxPZi1KlSuGll17Cr7/+WthDIZLEmWpY0kfy+/fvj+zsbN3HCSdOnIiUlBQ0aNAADRs2RFpaGqZNm2bTgRIREREVJEnnFL3yyit45ZVXdM/Lly+PCxcuYPv27XBxcUGdOnX0zl8hIiIikjtJocgYf39/dOjQwVarIyIiInqhJJ1TdPXqVVy9ehX16tXTTfvrr78wbdo0ZGdno0ePHgxIREREZFckhaIOHTogIyNDd4n+27dvo1y5cnjy5Al8fX1x584drFy5Ep06dbL5gImIiIgKgqQTrQ8ePIjmzZvrni9ZsgSPHj3CX3/9hRs3bqBp06b48ssvbTZIIiIiooIm6ZyiBw8e6N1b69dff0XDhg0RHR0NAOjUqRM++eQT24zQxjQaDW7evAlfX1+Tlz8nsiUhBNLT0xEeHg6lMl/3YAbAGqYXi/VL9s6aGpYUiooVK4YrV64AAFJSUrB//35MnjxZNz8nJwc5OTlSVl3gbt68iYiIiMIeBjmha9euoUSJEvleD2uYCgPrl+ydJTUsKRQ1a9YMM2fOhJ+fH3bs2AGNRqN3YvWpU6dkW/S5t3u4du0a/Pz8Cnk05AzS0tIQERFhs1uNsIbpRWL9kr2zpoYlhaLJkyfj3Llz+Oijj+Dm5oYvv/wSpUuXBgBkZ2djxYoV6Nmzp5RVF7jc3bV+fn58QdILZatDBaxhKgysX7J3ltSwpFAUEhKCP/74A6mpqfD09ISbm5tunkajwdatW2W7p4iIiIjImHxdvNHf399gmqenJypXrpyf1RIRERG9cBaFoiVLlgAAevXqBYVCoXuel969e0sfGREREdELZNHFG5VKJRQKBR49egQ3NzeLPpapUCigVqttMkhbSktLg7+/P1JTU3k8m14IW9cca5heJNYv2Ttras6iPUWXLl0CAN25Q7nPiYiIbEmj0djkekgFyR7GSNJYFIpKliyp+/7p06dITU1FUFCQTa5ZQURElEupVGLevHlITk4u7KEYFRYWhgEDBhT2MKiAWH2itVKpRLVq1TBt2jS8//77BTEmIiJyYsnJyboLBBO9SFbv/1OpVChZsiSys7MLYjxEREREhULSQdHBgwdj/vz5ePDgga3HQ0RERFQoJF2nSK1Ww93dHdHR0ejSpQtKlSoFT09PvTYKhQIffvihTQZJREREVNAkhaKPPvpI9/2CBQuMtmEoIiIiInsi6fDZpUuX8nxcvHhR0oB27dqFdu3aITw8HAqFAmvWrNGb/8Ybb0ChUOg9WrVqJakvIkek0WgKewh5socxEpHzkbSn6NmP6NtaZmYmKleujDfffBOdOnUy2qZVq1ZYuHCh7rm7u3uBjYfI3vAjzURE0uTr3mcPHjzAli1bcPnyZQBAqVKl0LRpUxQpUkTyOhMSEpCQkGC2jbu7O0JDQyX3YY49XJTLHsZIhYsfaSYisp7kUJSYmIgvvvjC4KP5bm5uGD58OMaPH5/vwZmyY8cOBAcHIzAwEE2aNMGECRNMBrHs7Gy9MaalpQHQnixu7DYkKpXKLt5ly/EWKmRcfn9XUmrYHrCG7QPr1zjWr/2w5nclKRR99tlnGD9+PNq0aYP33nsPZcqUAQCcPXsW3377LSZOnAhXV1eMHj1ayurNatWqFTp16oTSpUvjwoUL+OSTT5CQkIB9+/YZfTFNmjQJ48aNM5h+9uxZ+Pj46E3z8PBATEyMXbzLvnTpEh4/flzYwyALZGRk5Gt5KTVsD1jD9oH1axzr135YU8MW3RD2ecWLF8fLL7+MtWvXGp3frl07HDlyBDdv3rR21fqDUyjwyy+/oEOHDibbXLx4EdHR0diyZQuaNm1qMN/Yu5SIiAg8ePDA6I3hVCoVEhMTZRuKSpYsicTERL5LsSNpaWkICgqSfANM1jAVJtavPtav/bGmhiXtKUpNTTX7ia/WrVtjx44dUlZttaioKBQtWhRJSUlGQ5G7u7vRE7FVKpXd7KY1xp7H7mzy+7tiDVNhYv0aZ89jdzbW/K4kna1bt25dHDhwwOT8AwcOoG7dulJWbbXr16/j/v37CAsLeyH9ERERkWOSFIrmzp2Lffv24cMPP0RSUhI0Gg00Gg2SkpLwwQcfYP/+/Zg7d66kAWVkZODYsWM4duwYAO1x22PHjuHq1avIyMjAsGHDsH//fly+fBlbt25F+/btERMTg5YtW0rqj4iIiAiQePisUqVK0Gg0mDlzJmbOnKn7eHjuBdnc3d1RqVIlvWUUCgVSU1PzXPfhw4fRuHFj3fOhQ4cCAPr06YM5c+bg+PHjWLx4MVJSUhAeHo4WLVrgs88+47WKiIiIKF8khaLOnTtDoVDYeiwAgEaNGsHcud+///57gfRLREREzk1SKFq0aJGNh0FERERUuKw+pygrKwvVqlWTfM4QERERkRxZHYq8vLxw6dKlAjt8RkRERFQYJH36rFWrVjy3h4iIiByKpFA0evRonDt3Dr169cKePXtw48YNPHjwwOBBREREZC8knWhdoUIFAMCpU6fw008/mWzHy6ATERGRvZAUisaMGcNzioiIiMihSApFiYmJNh4GERGRYxEaNRRKed8jzR7G+CJJCkVERERknkKpQs7PwwCFEqrWYyAeXoV6wwTg6WPTC/mHwaXNOOBpFnJ+TQQepZhu6xkAl7aJgKsXctaPBVKTTbd19YAqYRQUgZFQ/zYe4m4SFEWj4NJpqsStc0ySQtH48ePzbKNQKDB69GgpqyciInIMCiVUCaMg7pyFeunbwJMs022DSsElYTTw6CFylvQFMu+ZbutdFC4dvwBUbshZ9Drw4LLptm5eUL32HRQBJaD+oS/Ezb8BAKbvHeG8bH74TKFQQAjBUERERE5P1XqM5YGo9yIgO92yQNR7IeDui5wlb1gWiIrFQv1jP10gIuMkfSRfo9EYPHJycnDhwgV8+OGHePnll3Hnzh1bj5WIiMiuiIdXGYjsiKRQZHRFSiVKly6NL7/8ErGxsRg8eLCtVk1ERGSX1BsmMBDZEZuFomc1aNAAv/32W0GsmoiIyH6YO6magUh2CiQUHT58GEplgayaiIjI/skhECn5AfTnSfqJLFmyxOj0lJQU7Nq1Cz///DPeeuutfA2MiIjIIckiELlC1XSoxA1wXJJC0RtvvGFyXtGiRTFixAiMGTNG6piIiIgck1wCUZfpUERUlbgRjktSKLp06ZLBNIVCgcDAQPj6+uZ7UERERA5HToEopgHUW6bApdWnEjfGMUkKRSVLlrT1OIiIiByX3ALRyvch0u9aPHy1AFQyv+WpWgOo8nk6s03Osjpz5gxWrlyJ5ORkxMXFoW/fvvDz87PFqomIiOybHAPR+Z1AaHmLN0GlAIYcAJLSgDsmPlDnqQJCPYFLGabXU9oHuPUIeKQ2Pj/YQ/vV2j7K+gM/NDC/DZawOBR9++23mDlzJvbu3YuiRYvqpq9btw5du3bFkydPdNO++eYb7N+/X68dERGR0/EP0966Q26BSIK9d4CjDyQtqpPf5QuaxTua/ve//yE6Olov6OTk5OCtt96CSqXCwoUL8ffff2Py5Mm4cuUKJk6cWCADJiIishcubcY5RCByFhaHolOnTqFWrVp607Zv3467d+/iww8/RJ8+fVChQgUMHz4c3bp148UbiSjfhMbEPnYZsYcxUiF6msVAZEcsPnx2//59RERE6E3bunUrFAoFOnbsqDe9bt26+Pnnn20zQiJyWgqlCjk/D4O4d9FwptIFqqZDoYioCvWWKRBX/zS7LmW9/lCWaw71rjkQZ7eZbxvfGarqPaE+9BM0R1ebHl/RKLh0mmrRtpBzyvk1kYHIjlgcikJCQnDr1i29abt374aXlxcqV66sN93NzQ1ubm62GSEROTVx7yJw65T+RN11VuKhXjk4jz/4Cihbj9EGonWjIY6Zf8OmrP+ONhBt/xqa3XPNDy6uiWUbQc7rUYrpeXYeiLqV0p44PesMkJFjOL9SINAxEvjlKnD8oeF8HxdgUFntSdMrLuevj9Op+diQZ1gcil5++WUsXrwYgwcPhq+vL06ePImDBw+iffv2cHHRX82ZM2dQokQJ24yQiOhZVv3B/ycQVetmeSBqPMSiQKSo0gmqBgMlbAAR7D4QNQoFfmqo/b6ENzD4gP58BYDNLYAiHsC7ZYHQ5YB4bh2TqgEDy2q/v/0I2Hlbeh8p2TbYKFhxTtHYsWNx5coVxMbGomnTpqhbty4UCgVGjhxp0PaXX35BnTp1bDNCIqJccgpE7T6D5vRmCRtBTk8WgUgBZb3+EjcAEM8nnALwIvp4nsV7iipWrIht27Zh4sSJuHjxImrVqoWPPvoI1apV02u3Y8cOeHl5oWvXrjYfLBE5MbkFoiMroPlzJVTlW0rYGHJacglE/xxSlmrnbaDnzn8PbT1PAGi+CegQCay5ariXCABGHgGuZ2oPnz2/l8jaPs6k/rtXKT+sunhjnTp1sH79erNtGjVqhL//NvOLIyKylhwD0W/jgdByEjaGnJacAlG1blDvmgOXhoMkbozp84ByHX9o/FyiXBk5wBcnbNNHfJD5dpbK5wWxiYgKmNJFnoHI6HtfIhPkFojWjc7zU5jOiKGIiGRN1XQoAxHZNzkGojxeH86KoYiIZE0RUZWBiOyXZwADkR2RXSjatWsX2rVrh/DwcCgUCqxZs0ZvvhACY8aMQVhYGDw9PdGsWTOcP3++cAZLRAVOvWWKQwQitUbyoi+E3Mdnr1zaJjpMIAr2AIp7mZ7v4wKU8ze/jnL+gLeZs5mLe+W/j/yw6kTrFyEzMxOVK1fGm2++iU6dOhnMnzJlCmbOnInFixejdOnSGD16NFq2bIlTp07Bw8OjEEZMRAXJ/JWq7SMQAYBKCUw+Dnz0kvb5+/uBQ/f123ipgGWNgHAv4L+XgGlGTkL9v5eA7qWBm1lA9x2GdxuvXgSYWcu6Pmx1h3EywtULOYtet/tABABrm2q/tt4MbNO/ljN8XIBj7YFSPsA3p4EPDxou/1UNYHA54HIGUHktkPncxRibhAK/Nc9fH/klu1CUkJCAhIQEo/OEEJgxYwZGjRqF9u3bAwCWLFmCkJAQrFmzBt27d3+RQyWiQmU/gShXkDvg8s/++RAv4OhzO7nL+WvDCgBUCTJ+R/HcT9mEewEPnxheybddRP77INvJWT/WIQIR8G9d1Q0xDCwR3tqwAgDNw40v3+Kf6aV8gEhvw9qtF5L/PvJLdqHInEuXLuHWrVto1qyZbpq/vz9q1qyJffv2GQ1F2dnZyM7+91KXaWlpAAC1Wg212vBGjiqVqgBGbnvGxk7ylN/flTPXsOltkUcgUkRWBWD5tqy9CpQP0D5fYOSo/+lU7Tvg5uHaa7gYM+KI9krAm28av7XBgvNAkzBpfRjbDtavcRbXb2qy6QYyCUSWbsvR+9q9O3KsXXPbYk0N21Uoyr33WkhIiN50Y/dlyzVp0iSMGzfOYPrZs2fh4+OjN83DwwMxMTE2Gm3BunTpEh4/flzYwyALZGRk5Gt5Z61h09sik0AU2xCqZsOt2pY7j4FGG80OIc9DAuuuaR+m3MiS3oex7WD9Gie9fv8hk0Bkzbb032t+z2Jh1m6u/NawXYUiKUaOHImhQ4fqnqelpSEiIgJxcXHw8/MrxJHlT+nSpQt7CGSh3HfGUrGGnyWjQNR1JsS1P6EoXdNhXo/GtoP1a1y+fucyCUTK+M4AHOv/SX5r2K5CUWhoKADg9u3bCAsL002/ffs2qlSpYnQZd3d3uLu7G0xXqVR2s5vWGEvGLjRqKJTy3kZ7GGN+5bfOnLmG9cksECXtgnrPd1C+tdyufw/PMrYdrF/jJI9dLoGo/jtQVe8JwH4OWVoivzVsV6GodOnSCA0NxdatW3UhKC0tDQcOHMDAgbxb9fMUShVyfh4GFCkFVYOB0JzeDM2e+eaXiawKVbPhENf+hHrrdECTY7ptsRioWo+BeHgV6g0TgKdmDuf5h8GlzTjgaRZyfk0EHqVAUTQKLp2mStw6ci4yDESrhgLBsVI2hpyVnAJR4yFQH/pJF4xIS3ahKCMjA0lJSbrnly5dwrFjxxAUFITIyEh88MEHmDBhAmJjY3UfyQ8PD0eHDh0Kb9BylhuILD5HYhhE0k7tH3zNU9NtwytClTAK4s5ZqJe+DTzJMj2GoFJwSRgNPHqInCV9gcx7gNmREOmTZSAy8/ogMiC3QLT9a2jO72Ioeo7sLt54+PBhxMfHIz4+HgAwdOhQxMfHY8yYMQCA4cOHY/Dgwejfvz+qV6+OjIwMbNy4kdcoMsGqQGThH3xFeEWoXl8Acfe8ZYGo9yIgO10vEBFZSlmvv8MGIk8VsKox8Gc70ze0HFIeONVB+9WYqkHa5Vc11q5PSh89oiQNnywlx0CUx+sjL3Kp3dw+bFXDsgtFjRo1ghDC4LFo0SIAgEKhwPjx43Hr1i08fvwYW7ZsQZkyZQp30DKmOb2ZgYjsmrJcc4cMRADQugTQIRKoFAQMr2g4X6kAplQDyvhrvyoVhm2GVdQu3yESSCghrY8h5fK/LWSCq4fDBSJAPrWb24etalh2oYhsS3sOEQMR2S/1rjkOGYgA7ceb0/9Z5XYjl7PRCGDPHe33e+5onz8vd7n0p8AxIx+XtqQPY8uRbagSRjlcIALkU7u5fdiqhmV3ThG9OAxEZA/E2W1m59trIAKAi+lA9Gog0A24kG68TavNQKwfcN7Ep4rnnwO2Jmuvbv0g23C+JX28dwDY10baNpB5isBIqH/o61CBCJBP7eb24e1imxpmKHJSsghEngGSx08E2HcgyvUg2/g/hFxPNcCpFPPrMPUPw9I+cngz2AKj/m28wwWiXHKo3dw+TJ1zZC0ePnNCsghE3kW1d48mkkgOgUhRzD6uvkyFR9xNMj3TjgORo2IocjKyCUS9FwKuXtI3hJyaLAJReEWoWo+Rugnk7BiIZImhyInIKhC5+2rvHk1kJdkEotcXQDy8KnUzyJnJJBAp4ppIGLxjYyhyErILREveMH/3aCIjZBWI7p7XXsndmvErgIFx2ofKyEeUAaBJKDCmMlDcxI7U4l7a+U1Cjc9X2aAPKkByCURVOkHVwLo7QXQp6fi1yxOtnYAsA9GDy0CoiSt6ERkhu0C09G0gqJRV29ApEvi40r/P55zVn1/cC/itOeCiBJqEGb9j+NIGQL0Q7cnR0au1dxZ/Vv8ywDe18tcHFRA5BaJ2n0FzejNU5VtaPHxnqF3uKXJwisiq8gxERFaQZSAy9/ogep7cAtGRFXneC9MZcU+Rg1M1G275vcxkHIjUGkAl8whvD2O0R8r4zlBV7+kQgejnq8D1fxadf85w/o0soPVmoG4IsOC88XX03AX0iwX+uG34Tvv59Urtg2xMjoHot/FAqHWXgf7iuLZ+Hbl2GYocnLj2p90HIkAbNnrtAs6kSl5FgSrrD/zQoLBH4ZgcJRAB2ivwPn9I4HnbbmkfptzMAj77y/R8tQ36IBtSusgzEEm4JfeqK9qrTJviCLXLUOTg1Fun230gynUm1fwLkhyT+tBPDhGIyDmpmg6FIiLe7gORs2AocnSaHJOz7CkQkfPSHF1tdj4DEcmZIqIq1CsHMxDZCZ4B4aRkEYhcPSSOnkiLgYjkTr1lCgORHWEockKyCERuXlAljLJq3P5upucVcQeifU3Pd1UC5QO0X02J9tWuR2of9GLJIhD5h+VjC7TkULsu/E9QYMTVP83Mte9AJIfaze3DVjXMl4KTkU0geu07KAIjrRr72iZAlJEXR5QvkNQZONtJe50LYzY2B4631341pn8Z7fJJnaX3QS+OLAJRUCm4tBmXj62QT+1+W1Pa+Ck/7DsQyaV2c/uwVQ0zFDkRWQWiYrFQ/zbeqvF7uwJVjNwJOT4I8HXVft/YyBt3pQKoF6z9vl6w9vnzcpfzldiHwsTVV8n2ZBOIei8CnubvsJocalepML4cFSR5BCJFZFUpgwcgn9rN7cNWNcxQ5CRkF4h+7Gf+7tFGbE8GNlw3nP7bdWDNVeD4A2DK34bzNQIYfgQ4m6r9qjHyN2LK39rl11yV1ofgofoXQlaBKDsdOb8mSt4WQB61qxHA16fztRlkFZkEotiGUDUbLmUDAMindnP7sFUN89NnTkCWgejm31bf5mP4YeCR2nD6IzXQZbv5Zb8+pX2YcvQBUHWd6fmW9EEFS3aBaElfwDdY+gZBPrW77CIwtIL5NmQLMgpEXWdCXPsTitLSjjvJpXZz+4gPsk0Nc0+Rg1MUi5FnICKygiwDkbnXB5EBmQWipF3a69iRHoYiB6dqPYaBiOyaIq4JAxHZORkGolVDzV7HzlkxFDk48fAqAxHZNVWDgQxEZNfkGYhMvz6cGc8pcnDqDRMYiMiuaU5vZiAiu6Ws1x/Kcs0ZiOwE9xQ5uqePTc+z00BU3AvY0Ur7CPcy3uarGsDJDkC7COPz20Vo539Vo+D6INvQ7JkPRwlEwR6sXWfjSIFofh3Hr12GImclh0CklLajsl8sUC9E++gXazi/nD8wuBwQ5w9MqmZ8HZOraecPLqdtXxB9UMGzp0AEAO0jWbvORr1rjkMEIgCIL+L4tctQ5IxkEYhcoWo6VNLw99wGcjTaxx+3DedfzQQuZ2i/33zT+Do2/TP9coa2fUH0QQVLFoHIM8CqMR+7z9p1NuLsNrPz7SUQAc5RuzynyNnIJRB1mQ5FhLSrqW67BUT/c+P0G0b+t2XmAJXXApHewOlU4+v48CAw/6z2RZZp5AMY1vYRzysCv1CyCETeReHSNtGqcR+6L7/apcJjT4EIANpvBU6kOHbtMhQ5EzkFopgGUG+ZApdWn0raFGMvmGdl5uT9oslrvi36INuTTSDqvRBwNXHigxmsXQLkEYgUxWKsGvOdx+ZryxFql4fPnIXcAtHK9/O4ezSRIVkFIndf5KwfK31jyGnJIhCFV4Sq9Ripm+CwGIqcgRwD0fmdEjeGnJXsAtGSN4DUZMnbQ85JNoHo9QUQD69K3QyHxVDk6PzDGIjI7skyEJl7fRAZIatAdPe89jp2pIehyMG5tBnnMIGoebjped1KAR+/BPiYOEuuUiAwtrL2qzE+Ltrlu5WS3gcVDEVkVYcORHKo3Vg/KwZMksguEC192/x17Cwgh9rN7cNWNWx3f94TExMxbtw4vWlxcXE4c+ZMIY1I5p5mOUQgAoDPqwEH7gI7n/uoZqNQ4KeG2u9LeAODD+jPVwDY3AIo4gG8WxYIXW74J2ZSNWBgWe33tx9J64MKhqrZcIiknQ4ZiORSu7Nr2WBjyCRZBiJzrw8LyKV2c/tIyc7X5ujYXSgCgAoVKmDLli265y4udrkZL0TOr4kOEYjMEab/ZthVH2ScuPanQwYigLXrDJTxnaGq3tOhAhHguLVrl2nCxcUFoaGhhT0M+/AoxfQ8OwtEnxwxfCcBaKf13AmU9gFmGdlhKAA03wR0iATWXDX+Z2bkEeB6JnApQ3ofVDDUW6c7ZCAC5FO77+7/9x052ZYjBiJAPrWb28eZVNvUsF2GovPnzyM8PBweHh6oXbs2Jk2ahMjISKNts7OzkZ397361tLQ0AIBarYZarTZor1KpCmbQNmZs7M8zuy0yCkSWbou5K5iuuGx++eMPtQ9TMnKAL06YX0defQDGt8WS7TPH6WtYY+Qqb/+QSyDKz+tRDrV7XltSrF8rWPo7Vx/6SfaBSGr9yqF2c/vIvYBufmvY7kJRzZo1sWjRIsTFxSE5ORnjxo1D/fr1ceLECfj6+hq0nzRpksE5SABw9uxZ+Pj46E3z8PBATIx1F7MqLJcuXcLjx6ZPkjO7LbIIRAoo6/XP/7bIjLFtycjIyNc6WcPGySIQuXrYZFvkgvVrOUt/55qjq82up7ADkTXbYg/yW8N2F4oSEhJ031eqVAk1a9ZEyZIlsWLFCvTr18+g/ciRIzF06L/32EpLS0NERATi4uLg52e/H7koXbq0tAXlEohaj4GyXHMA+dgWGTK2LbnvjKViDRuSRSBy84IqYRQAx6lh1q/lbPE7l0MgAhynfoH817DdhaLnBQQEoEyZMkhKSjI6393dHe7u7gbTVSqV3eymNUbS2OUUiKp1g3rXHLg0HGTXv4fnGduW/G4fa1ifbALRa99BEag9bG/Pv4dnsX4tl9+xyyIQ+YcBcJz6BfJfw3Z/naKMjAxcuHABYWFhhT0UeZNbIFo3Os+7R5tT3Ev7MMXHBSjnb34d5fwBbzNvC2zRB9mWrAJRsViofxtv9TawdkkWgSiolPY6dlYI9nD82rW7UPTRRx9h586duHz5Mvbu3YuOHTtCpVKhR48ehT00+ZJjIDr2s8SNAZqEAhc6ax9NjHwI0ccFONYe+LsD8FUN4+v4qoZ2/l/tjb9AbdEH2ZbsAtGP/SDuGt9DbUr1IqxdZyebQNR7EfDUuk+irW3q+LVrd6Ho+vXr6NGjB+Li4tCtWzcUKVIE+/fvR7FixQp7aPLkGeBQgQgA6oUALkrto26I4fwIb6DUP+dvmroKdot/ppfyASK9C6YPsh1ZBiJzrw8TqhRh7TozWQWi7HTtdeys4Ay1a3fnFP33v/8t7CHYFZe2iYDKzWECEQAsOA80Cfv3++edTgW+Oa190Yw8YnwdI45or6a6+aa2fUH0QbahKBYDVcIouw9EALD2KlA+QPs9a9e5yC4QLekL+AZbtQ1H7wOZOY5du3YXishKrl7IWfS6wwQiALiRBTTaaL7NhwfNz193TfuwVR+518gg21O1HgNx56zdByIAuPNYfrVLBU+WgSjzntWhqP9e4OgD0/MdoXYZihxczvqxDhWIyPmIh1cdIhCRc1LENYGqwUD5BSIyyu7OKSIrpSabnsdARHZAvWECAxHZLQYi+8JQ5KwYiMhePDV9pV0GIpI7zenNDER2hKHIGckkECnjO0sbPxEgj0Ck5BkIZJ5mz3wwENkPhiJnI5dAVP8dqKr3tGroU14GPI1cmNRTBaxqDPzZzvQJz0PKA6c6aL8aUzVIu/yqxtL7oBdIFoHIFaqmQ03Pt4BcardHlKThUz7ZcyCSS+3m9mGrGmYociZyCkSNh0B96Cerht84DEgoYTi9dQmgQyRQKQgYXtFIfwpgSjWgjL/2q1Jh2GZYRe3yHSKl9aEwsk4qIHIJRF2mQxFRVeJGaMmhdpUKYEi5fG0GSSCLQOQZIHn8cqnd3D5sVcMMRc5CboFo+9d53j36eZlPgWNGPg569AGQ/s/fk+1GzivXCGDPHe33e+5onz8vd7l0iX0I03vHyZbkFIhiGkC9ZYrEDdGSQ+1qhPHlqODIIhB5F9Vex04iudRubh+2qmEeEHcGcgxEu+cCoSb2qZrwyjbgYrrh9IvpQPRqINANuGBkPgC02gzE+gHnTdwsef45YGsy8PAJ8CBbWh9UwOQWiFa+D5F+V+LGaMmldt87AOxrI20byDqyCUS9FwKuZm4ylge51G5uH94utqlh7ilydK4e8gxEEqQ9MT3vQbb5sPJUA5xK0X415UK68RempX1QAZJjIDL7+rCcHGo3x8y6yXZkFYjcfbXXscsHOdRubh+2qmHuKXJwqoRRUASUsPtARE7MPwwuCaMdMhCR85BdIFryBuAmfU+Ro+KeIgenCIxkICK75tJmHAMR2TVZBiJzrw8nxj1FDk7923gGIrJvT7MYiMhuKSKrQtVsGAORneCeIgcn7iaZnmmngUilAAbGaR8qEx+FbxIKjKkMFDexd7i4l3Z+k9CC64NsI+fXRIcJRErWrtNRNRvuMIGoS0nHr13uKXJWdhqIAKB/GeCbWv8+n3NWf35xL+C35oCLEmgSZvyuy0sbAPVCtCfnRa/W3p05P33wruMF6FGK6Xl2FIgAoFMk8HGlf5/LoXbzuis55Y+49qdDBCLAOWqXe4qckUwCkSKuiYTBE/3DzgIROSf11ukOEYicBfcUORu5BKIqnaBqMFDCBmivbWHs+1w3soDWm4G6IcCC88bX0XMX0C8W+OO24bsVKX2EeFi3DZRPsghECijr9bdq2D9fBa7/U29yqV0qYJock7PsLRB9cVxbv45cuwxFzkROgajdZ9Cc3gxV+ZZWb4ZaGO5Wfd62W9qHKTezgM/+sl0fDEUvkFwCUesxUJZrbtXQNTKsXSocsghErtb94Vp1RXuVaVMcoXZ5+MxZyC0QHVnxz92jiawgp0BUrRvUu+ZI3BByZrIIRG5eUCWMkrgFjouhyBnIMRD9Nl7ChpBTk1sgWjca4uw2iRtDzko2gei176AIjJS4FY6LocjRKV1kGoh4B1WyghwDUR6vD6LnySoQFYuFmm9ODTAUOThV06EOE4j83UzPK+IORPuanu+qBMoHaL+aEu2rXY/UPqiAeAY4dCCSQ+268D9BgZNdIPqxn/nr2FlADrWb24etapgvBQeniKjqEIEIANY2AaKMvDiifIGkzsDZTtrrXBizsTlwvL32qzH9y2iXT+osvQ8qGC5tEx02EMmldr+tKW38ZBlZBiJzrw8LyKV2c/uwVQ0zFDk49ZYpDhGIAMDbFagSZDg9PgjwddV+3zjMyJgVQL1g7ff1grXPn5e7nK/EPhQmrr5KNuDq5ZCBCJBH7SoVxpcj21AUi3G4QATIp3Zz+7BVDTMUOThx9U8zc+0nEAHA9mRgw3XD6b9dB9ZcBY4/AKYYea1rBDD8CHA2VftVY2QoU/7WLr/mqrQ+BE+RKjA568c6ZCAC5FG7GgF8fTr/20LGqVqPcbhABMindnP7sFUN8zpFTsu+AhEADD8MPFIbTn+kBrpsN7/s16e0D1OOPgCqrjM935I+qICkJpueZ8eBCJBP7S67CAytYL4NSSMeXnW4QATIp3Zz+4gPsk0Nc0+RU5JHIFJEVpUyeCItOw9E5BzUGyY4XCByZAxFTkcmgSi2IVTNhkvZACLZBCJlfGdp4yfn8fSx6XkMRLLDUORUZBSIus6EuGbufCciE+QSiOq/A1X1ntK2gUgOgUjJM2iex1DkNGQWiJJ2ae8eTWQNOQWixkOgPvSTtO0g5yaLQOQKVdOhEjfAcTEUOQUZBqJVQ83ePdqc4l7AjlbaR7iX8TZf1QBOdgDaRRif3y5CO/+rGgXXB9mY3ALR9q+hObraqk0I9mDtOj25BKIu06GIsO68zvl1HL92GYqcgDwD0VMpmwIA6BcL1AvRPvrFGs4v5w8MLgfE+QOTqhlfx+Rq2vmDy2nbF0QfZENyDER5vD6MaR/J2nVqcgpEMQ2g3jLFquHHF3H82rXbUDRr1iyUKlUKHh4eqFmzJg4ePFjYQ5IlZb3+DhWIAGDPbSBHo338cdtw/tVM4HKG9vvNN42vY9M/0y9naNsXRB9kI64eDhGIAODYfdau05JbIFr5fh7XsTPkDLVrl2dZLV++HEOHDsXcuXNRs2ZNzJgxAy1btsTZs2cRHBxc2MOTFWW55g4ViABg2y0g+p+jFjeMfNI1MweovBaI9AZOpxpfx4cHgflntS+yTCNH8aztI55XBC4wqoRRUASUsPtABACH7suvdukFkGMgOr8TCC1v1Wa03wqcSHHs2rXLPUXTp0/H22+/jb59+6J8+fKYO3cuvLy88J///KewhyY76l1zHCoQ5bqRZfxFkyszJ+8XzelU4y9MW/ZB+acIjHSIQJSLtetk/MPkGYgkuPPY8WvX7vYUPXnyBEeOHMHIkSN105RKJZo1a4Z9+/YZtM/OzkZ2drbueWqq9qf58OFDqNWGl0dWqVQICAjQW0ZOAgICkJaWZnTsz1OpVMi5dhbC38jB2X8oK70CVbVuUG//HprjWwEzbRWxDaCq+xY0h9ZAs+8nwD/GdNsSlaFq/AHEqT1Q75wN+JYybONVHC5WbEuUC6D2yLNpoYhyAdLSYHRb0tLSAABC4r1AnL2G1bvmQpP52HRtKl2gavgeFMGVoV77GcSdm2brWFn7DShjm0C9YTrEpb/Nt83j9eEoNcz6tZy19Ssafgw8vI+c3ycBLoGAf6Dxxp7+cGk5EnjqgpxfRwNqV9O16eoBVfNhUHiEQb16NITE14ej1C9gwxoWdubGjRsCgNi7d6/e9GHDhokaNWoYtB87dqyAdncGH3wU6uPatWuSap41zIccHqxfPuz9YUkNK4Swr1tZ3rx5E8WLF8fevXtRu3Zt3fThw4dj586dOHDggF7759+laDQaPHjwAEWKFIGCtzanF0AIgfT0dISHh0OptP6INWuYChPrl+ydNTVsd4fPihYtCpVKhdu39U9Nv337NkJDQw3au7u7w93dXW9aQEBAQQ6RyIC/v7/kZVnDVNhYv2TvLK1huzvR2s3NDdWqVcPWrVt10zQaDbZu3aq354iIiIjIGna3pwgAhg4dij59+uDll19GjRo1MGPGDGRmZqJv376FPTQiIiKyU3YZil599VXcvXsXY8aMwa1bt1ClShVs3LgRISEhhT00IiIislN2d6I1ERERUUGwu3OKiIiIiAoCQxERERERGIqIiIiIADAUEREREQFgKCIiIiICwFBEREREBIChiIiIiAgAQxERERERAIYiIiIiIgAMRUREREQAGIqIyIFcvnwZCoUCx44dK+yhkANq1KgRPvjgg8IeBhUghiKymkajkXV/b7zxBhQKBSZPnqw3fc2aNVAoFBavx9QfwEWLFiEgIMCqMcmNeMG/Q6FRW9xWoVCYfSQmJhbcQC1gLnjZ+z9N9YstC6v7mzt3Lnx9fZGTk6OblpGRAVdXVzRq1Eiv7Y4dO6BQKHDhwgUbjNQ4R/4b4axcCnsAZH+USiXmzZuH5OTkAu8rLCwMAwYMsHo5Dw8PfPHFFxgwYAACAwMLYGT2TaFUImf9eIgbf5lu5BkAl7aJgKsXctaPBVLN/L5dPaBKGAVFYCTUv42HuJv0b19Fo+DSaarFY3u2rpYvX44xY8bg7Nmzumk+Pj4Wr4uso1ICvXYBZ1L1p5f1B8ZUBq5nAaOPAtnPZVx3FfBZPFDCCxj3F3D2ueUBoEcU0DkSWH0VWHZRu84fGlg3vsaNGyMjIwOHDx9GrVq1AAC7d+9GaGgoDhw4gMePH8PDwwMAsH37dkRGRiI6OtqqPoQQUKvVcHHhv0dnxD1FJElycjKuXLlS4A+pwatZs2YIDQ3FpEmTTLZZvXo1KlSoAHd3d5QqVQrTpk2T+uPAnDlzEB0dDTc3N8TFxeGHH37Qm69QKDBnzhwkJCTA09MTUVFRWLVqleT+bEHc+Au4dcr4I/0OXFqNBFRuyFn0OnB2q+m2Dy5D1ez/oAgoAfUPfSH+/p/efHHvolXjCg0N1T38/f2hUCh0z4ODgzF9+nSUKFEC7u7uqFKlCjZu3GhyXWq1Gm+++SbKli2Lq1evAgDWrl2LqlWrwsPDA1FRURg3bpzengeFQoHvv/8eHTt2hJeXF2JjY/G///1P0s/44cOH6N27NwIDA+Hl5YWEhAScP39eNz93j8KaNWsQGxsLDw8PtGzZEteuXZPUny2cTQOOPtB/vBIJxPoDjcOAcC/D+cW9tPNi/YH2kYbz/3oIDCkHlPTVfv3roWHwskRcXBzCwsKwY8cO3bQdO3agffv2KF26NPbv3683vXHjxsjOzsb777+P4OBgeHh4oF69ejh06JBeO4VCgQ0bNqBatWpwd3fHnj17kJmZid69e8PHxwdhYWH5+vsA2OffCGfEUEQOSaVS4fPPP8c333yD69evG8w/cuQIunXrhu7du+Pvv/9GYmIiRo8ejUWLFlnd1y+//IIhQ4bg//7v/3DixAkMGDAAffv2xfbt2/XajR49Gp07d8Zff/2F1157Dd27d8fp06elbmLB8S4Kl94LAXdf5Cx5A3hw2XRbNy+oXvsOimKxUP/YD+Lm3wU6tK+//hrTpk3Dl19+iePHj6Nly5Z45ZVX9IJGruzsbHTt2hXHjh3D7t27ERkZid27d6N3794YMmQITp06hXnz5mHRokWYOHGi3rLjxo1Dt27dcPz4cbRu3RqvvfYaHjx4YPV433jjDRw+fBj/+9//sG/fPggh0Lp1azx9+lTXJisrCxMnTsSSJUvwxx9/ICUlBd27d7f+h2MjQhhO2/7Pe5P0p8AxIz+Gow+0855t+yyNAPbc0X6/5472uVSNGzfWe21t374djRo1QsOGDXXTHz16hAMHDqBx48YYPnw4Vq9ejcWLF+PPP/9ETEwMWrZsafD7HDFiBCZPnozTp0+jUqVKGDZsGHbu3Im1a9di06ZN2LFjB/78809JY3a4vxEOTCGEsZcAkXmJiYm4cuVKgfdTsmRJq88heeONN5CSkoI1a9agdu3aKF++PBYsWIA1a9agY8eOEELgtddew927d7Fp0ybdcsOHD8f69etx8uRJANrzBfbu3Qs3Nze99efk5MDDwwMpKSkAgLp166JChQqYP3++rk23bt2QmZmJ9evXA9C+C3znnXcwZ84cXZtatWqhatWqmD17tlXbZytP53fW7s15VkEEotDycO2/WtIYFy1ahA8++ED3sy5evDgGDRqETz75RNemRo0aqF69OmbNmoXLly+jdOnS2L17NxITE5GdnY1ff/0V/v7+ALR7EJs2bYqRI0fqlv/xxx8xfPhw3Lx5E4D2dzVq1Ch89tlnAIDMzEz4+Phgw4YNaNWqla4PT09PKJX67ysfPXqEwYMHY8aMGTh//jzKlCmDP/74A3Xq1AEA3L9/HxEREVi8eDG6du2KRYsWoW/fvti/fz9q1qwJADhz5gzKlSuHAwcOoEaNGpJ+bvlRfZ025Dwv2hd4+AR4kG18uSB3INANuJBufL6rEoj1A86nAU81QHwQcKid9eP7/vvvdTXx6NEjBAUF4ebNm9iyZQvmzp2LnTt3Ytu2bWjatCkuX76M2NhYLFq0CD179gQAPH36FKVKlcIHH3yAYcOG6fYorVmzBu3btwegPU+pSJEi+PHHH9G1a1cAwIMHD1CiRAn0798fM2bMAOD4fyOcEfcUkUP74osvsHjxYoN3W6dPn0bdunX1ptWtWxfnz5+HWv3vCROvvfYajh07pvcYP368Ret6vs/atWsbPJfVu8CC2kOktM25GWlpabh586ZFP+sePXogMzMTmzZt0gUiAPjrr78wfvx4+Pj46B5vv/02kpOTkZWVpWtXqVIl3ffe3t7w8/PDnTt39PpYvny5QW28/PLLuvmnT5+Gi4uLLuwAQJEiRRAXF6c3XhcXF1SvXl33vGzZsggICJBXbUAbdkwFIkA7z1QgArRB6FSK9mt+NGrUCJmZmTh06BB2796NMmXKoFixYmjYsKHuvKIdO3YgKioKqampePr0qV7NuLq6okaNGgY/32d/dxcuXMCTJ0/0fndBQUGIi4szGI9T/Y1wAjyTjBxagwYN0LJlS4wcORJvvPGG1cv7+/sjJiZGb1pwcLCNRicjBRaIXKFqOtTWo81T69at8eOPP2Lfvn1o0qSJbnpGRgbGjRuHTp06GSyTe4IuoP3H+SyFQmHwKciIiAiD2vD09LTF8MmMmJgYlChRAtu3b8fDhw/RsGFDAEB4eDgiIiKwd+9ebN++Xe/3bglvb29J43GavxFOgnuKyOFNnjwZ69atw759+3TTypUrhz/++EOv3R9//IEyZcpApVJZtX5T6ypfvrzetGdPAs19Xq5cOav6KhAFGYi6TIcioqpNhunn54fw8HCLftYDBw7E5MmT8corr2Dnzp266VWrVsXZs2cRExNj8Hj+UFh+lStXDjk5OThw4IBu2v3793H27Fm98ebk5ODw4cO652fPnkVKSoo8akOmGjdujB07dmDHjh16H8Vv0KABNmzYgIMHD6Jx48a6E5ufrZmnT5/i0KFDBjXzrOjoaLi6uur97h4+fIhz585JGq/d/41wItxTRJKEhYXZTT8VK1bEa6+9hpkzZ+qm/d///R+qV6+Ozz77DK+++ir27duHb7/9VtKx+2HDhqFbt26Ij49Hs2bNsG7dOvz888/YsmWLXruVK1fi5ZdfRr169bB06VIcPHgQCxYsyPf2SaUoGgXx/Mfu3byAUBP/LJ7/2L1Gbbqt0gWqpkOhiKgKzcEfoKrzpk3GPGzYMIwdOxbR0dGoUqUKFi5ciGPHjmHp0qUGbQcPHgy1Wo22bdtiw4YNqFevHsaMGYO2bdsiMjISXbp0gVKpxF9//YUTJ05gwoQJNhljrtjYWLRv3x5vv/025s2bB19fX4wYMQLFixfXnbsCaPdKDR48GDNnzoSLiwvee+891KpVq1DOJwK0H5WXez+NGzfGoEGD8PTpU92eIgBo2LAh3nvvPTx58gSNGzeGt7c3Bg4ciGHDhiEoKAiRkZGYMmUKsrKy0K9fP5Pr9/HxQb9+/TBs2DAUKVIEwcHB+PTTTyUHZ3v9G+GUBJGV1Gq1rPvr06ePaN++vd60S5cuCTc3N/Fsya9atUqUL19euLq6isjISDF16lS9ZRo2bCiGDBlisP6FCxcKf39/vWmzZ88WUVFRwtXVVZQpU0YsWbJEbz4AMWvWLNG8eXPh7u4uSpUqJZYvX27VdtmSRp1jF/09/7NWq9UiMTFRFC9eXLi6uorKlSuLDRs26OZfunRJABBHjx7VTZs2bZrw9fUVf/zxhxBCiI0bN4o6deoIT09P4efnJ2rUqCHmz5+vaw9A/PLLL3rj8Pf3FwsXLjTZR67na+bBgweiV69ewt/fX3h6eoqWLVuKc+fOGWzf6tWrRVRUlHB3dxfNmjUTV65csf6HZQM5L/alLXI00pbL/R2ULVtWb/rly5cFABEXF6eb9ujRIzF48GBRtGhR4e7uLurWrSsOHjyom799+3YBQDx8+FBvXenp6eL1118XXl5eIiQkREyZMsXg9+vIfyOcFT99RvQCKBQK/PLLL+jQoUNhD4Vk5PlP15Hz4t8IeeA5RURERERgKCIiIiICwIs3EhEREQHgniIiIiIiAAxFRERERAAYioiIiIgAMBQRERERAWAoIiIiIgLAUEREREQEgKGIiIiICABDEREREREAhiIiIiIiAAxFRERERAAYioiIiIgAMBQRERERAWAoIiIiIgLAUEREREQEAPh//1rsSgNxj0AAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 600x400 with 6 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "plot_surprisal_differences_checkpoints(seeds=[53], checkpoints=CHECKPOINTS, pos_encodings=False)\n",
+    "plt.savefig(f\"figures/hop_surprisals_no_pos_encodings.pdf\", format=\"pdf\", bbox_inches=\"tight\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "babyenv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

impossible_llm.yaml

@@ -0,0 +1,154 @@
+name: impossible_llm
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - ca-certificates=2024.7.2=h06a4308_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - libffi=3.4.4=h6a678d5_1
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - ncurses=6.4=h6a678d5_0
+  - openssl=3.0.15=h5eee18b_0
+  - pip=24.2=py39h06a4308_0
+  - python=3.9.19=h955ad1f_1
+  - readline=8.2=h5eee18b_0
+  - setuptools=72.1.0=py39h06a4308_0
+  - sqlite=3.45.3=h5eee18b_0
+  - tk=8.6.14=h39e8969_0
+  - wheel=0.44.0=py39h06a4308_0
+  - xz=5.4.6=h5eee18b_1
+  - zlib=1.2.13=h5eee18b_1
+  - pip:
+      - accelerate==0.34.2
+      - aiohappyeyeballs==2.4.2
+      - aiohttp==3.10.6
+      - aiosignal==1.3.1
+      - asttokens==2.2.1
+      - async-timeout==4.0.3
+      - attrs==24.2.0
+      - backcall==0.2.0
+      - black==24.8.0
+      - blessed==1.20.0
+      - certifi==2023.11.17
+      - charset-normalizer==3.3.2
+      - click==8.1.7
+      - cmake==3.30.3
+      - comm==0.1.2
+      - contourpy==1.2.0
+      - cycler==0.12.1
+      - data==0.4
+      - datasets==3.0.1
+      - debugpy==1.6.7
+      - decorator==5.1.1
+      - dill==0.3.8
+      - emoji==2.8.0
+      - exceptiongroup==1.1.0
+      - executing==1.2.0
+      - filelock==3.12.2
+      - fonttools==4.45.1
+      - frozenlist==1.4.1
+      - fsspec==2023.10.0
+      - funcsigs==1.0.2
+      - future==0.18.3
+      - gmpy2==2.1.0
+      - gpustat==1.1.1
+      - huggingface-hub==0.25.0
+      - idna==3.6
+      - importlib-metadata==6.0.0
+      - importlib-resources==6.4.5
+      - iniconfig==2.0.0
+      - ipykernel==6.23.1
+      - ipython==8.0.0
+      - jedi==0.18.2
+      - jinja2==3.1.2
+      - joblib==1.3.2
+      - jupyter-client==8.1.0
+      - jupyter-core==5.1.0
+      - kiwisolver==1.4.5
+      - latex==0.7.0
+      - latexcodec==1.0.0
+      - lit==18.1.8
+      - markupsafe==2.1.2
+      - matplotlib==3.8.2
+      - matplotlib-inline==0.1.6
+      - mizani==0.9.3
+      - mpmath==1.2.1
+      - multidict==6.1.0
+      - multiprocess==0.70.16
+      - mypy-extensions==1.0.0
+      - nest-asyncio==1.5.6
+      - networkx==2.8.6
+      - nltk==3.8.1
+      - numpy==1.26.2
+      - nvidia-cublas-cu11==11.10.3.66
+      - nvidia-cuda-cupti-cu11==11.7.101
+      - nvidia-cuda-nvrtc-cu11==11.7.99
+      - nvidia-cuda-runtime-cu11==11.7.99
+      - nvidia-cudnn-cu11==8.5.0.96
+      - nvidia-cufft-cu11==10.9.0.58
+      - nvidia-curand-cu11==10.2.10.91
+      - nvidia-cusolver-cu11==11.4.0.1
+      - nvidia-cusparse-cu11==11.7.4.91
+      - nvidia-ml-py==12.560.30
+      - nvidia-nccl-cu11==2.14.3
+      - nvidia-nvtx-cu11==11.7.91
+      - packaging==23.0
+      - pandas==2.1.3
+      - parso==0.8.3
+      - pathspec==0.12.1
+      - patsy==0.5.3
+      - peft==0.13.0
+      - pexpect==4.8.0
+      - pickleshare==0.7.5
+      - pillow==10.1.0
+      - platformdirs==2.5.2
+      - plotnine==0.12.4
+      - pluggy==1.3.0
+      - pluralizer==1.2.0
+      - prompt-toolkit==3.0.30
+      - protobuf==4.25.1
+      - psutil==5.9.1
+      - ptyprocess==0.7.0
+      - pure-eval==0.2.2
+      - pyarrow==17.0.0
+      - pygments==2.15.0
+      - pyparsing==3.1.1
+      - pytest==7.4.3
+      - python-dateutil==2.8.2
+      - pytz==2023.3.post1
+      - pyyaml==6.0.1
+      - pyzmq==23.0.0
+      - regex==2023.10.3
+      - requests==2.32.3
+      - safetensors==0.4.5
+      - scikit-learn==1.3.2
+      - scipy==1.11.4
+      - seaborn==0.13.0
+      - sentencepiece==0.2.0
+      - shutilwhich==1.1.0
+      - six==1.16.0
+      - stack-data==0.6.0
+      - stanza==1.9.2
+      - statsmodels==0.14.0
+      - sympy==1.11.1
+      - tempdir==0.7.1
+      - threadpoolctl==3.2.0
+      - tokenizers==0.20.0
+      - tomli==2.0.1
+      - torch==2.0.0
+      - tornado==6.2
+      - tqdm==4.66.5
+      - traitlets==5.7.1
+      - transformers==4.45.1
+      - triton==2.0.0
+      - typing-extensions==4.6.0
+      - tzdata==2023.3
+      - urllib3==2.1.0
+      - wcwidth==0.2.5
+      - xxhash==3.5.0
+      - yarl==1.13.0
+      - zipp==3.12.0
+prefix: /home/yiren/new_ssd2/chunhui/miniconda/envs/impossible_llm

impossible_llm_update.yaml

@@ -0,0 +1,162 @@
+name: impossible_llm
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - ca-certificates=2024.7.2=h06a4308_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - libffi=3.4.4=h6a678d5_1
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - ncurses=6.4=h6a678d5_0
+  - openssl=3.0.15=h5eee18b_0
+  - pip=24.2=py39h06a4308_0
+  - python=3.9.19=h955ad1f_1
+  - readline=8.2=h5eee18b_0
+  - setuptools=72.1.0=py39h06a4308_0
+  - sqlite=3.45.3=h5eee18b_0
+  - tk=8.6.14=h39e8969_0
+  - wheel=0.44.0=py39h06a4308_0
+  - xz=5.4.6=h5eee18b_1
+  - zlib=1.2.13=h5eee18b_1
+  - pip:
+      - accelerate==1.0.0
+      - aiohappyeyeballs==2.4.2
+      - aiohttp==3.10.6
+      - aiosignal==1.3.1
+      - annotated-types==0.7.0
+      - asttokens==2.2.1
+      - async-timeout==4.0.3
+      - attrs==24.2.0
+      - backcall==0.2.0
+      - black==24.8.0
+      - blessed==1.20.0
+      - certifi==2023.11.17
+      - charset-normalizer==3.3.2
+      - click==8.1.7
+      - cmake==3.30.3
+      - comm==0.1.2
+      - contourpy==1.2.0
+      - cycler==0.12.1
+      - data==0.4
+      - datasets==3.0.1
+      - debugpy==1.6.7
+      - decorator==5.1.1
+      - deepspeed==0.15.2
+      - dill==0.3.8
+      - emoji==2.8.0
+      - exceptiongroup==1.1.0
+      - executing==1.2.0
+      - filelock==3.12.2
+      - fonttools==4.45.1
+      - frozenlist==1.4.1
+      - fsspec==2023.10.0
+      - funcsigs==1.0.2
+      - future==0.18.3
+      - gmpy2==2.1.0
+      - gpustat==1.1.1
+      - hjson==3.1.0
+      - huggingface-hub==0.25.0
+      - idna==3.6
+      - importlib-metadata==6.0.0
+      - importlib-resources==6.4.5
+      - iniconfig==2.0.0
+      - ipykernel==6.23.1
+      - ipython==8.0.0
+      - jedi==0.18.2
+      - jinja2==3.1.2
+      - joblib==1.3.2
+      - jupyter-client==8.1.0
+      - jupyter-core==5.1.0
+      - kiwisolver==1.4.5
+      - latex==0.7.0
+      - latexcodec==1.0.0
+      - lit==18.1.8
+      - markupsafe==2.1.2
+      - matplotlib==3.8.2
+      - matplotlib-inline==0.1.6
+      - mizani==0.9.3
+      - mpmath==1.2.1
+      - msgpack==1.1.0
+      - multidict==6.1.0
+      - multiprocess==0.70.16
+      - mypy-extensions==1.0.0
+      - nest-asyncio==1.5.6
+      - networkx==2.8.6
+      - ninja==1.11.1.1
+      - nltk==3.8.1
+      - numpy==1.26.2
+      - nvidia-cublas-cu11==11.10.3.66
+      - nvidia-cuda-cupti-cu11==11.7.101
+      - nvidia-cuda-nvrtc-cu11==11.7.99
+      - nvidia-cuda-runtime-cu11==11.7.99
+      - nvidia-cudnn-cu11==8.5.0.96
+      - nvidia-cufft-cu11==10.9.0.58
+      - nvidia-curand-cu11==10.2.10.91
+      - nvidia-cusolver-cu11==11.4.0.1
+      - nvidia-cusparse-cu11==11.7.4.91
+      - nvidia-ml-py==12.560.30
+      - nvidia-nccl-cu11==2.14.3
+      - nvidia-nvtx-cu11==11.7.91
+      - packaging==23.0
+      - pandas==2.1.3
+      - parso==0.8.3
+      - pathspec==0.12.1
+      - patsy==0.5.3
+      - peft==0.13.0
+      - pexpect==4.8.0
+      - pickleshare==0.7.5
+      - pillow==10.1.0
+      - platformdirs==2.5.2
+      - plotnine==0.12.4
+      - pluggy==1.3.0
+      - pluralizer==1.2.0
+      - prompt-toolkit==3.0.30
+      - protobuf==4.25.1
+      - psutil==5.9.1
+      - ptyprocess==0.7.0
+      - pure-eval==0.2.2
+      - py-cpuinfo==9.0.0
+      - pyarrow==17.0.0
+      - pydantic==2.9.2
+      - pydantic-core==2.23.4
+      - pygments==2.15.0
+      - pyparsing==3.1.1
+      - pytest==7.4.3
+      - python-dateutil==2.8.2
+      - pytz==2023.3.post1
+      - pyyaml==6.0.1
+      - pyzmq==23.0.0
+      - regex==2023.10.3
+      - requests==2.32.3
+      - safetensors==0.4.5
+      - scikit-learn==1.3.2
+      - scipy==1.11.4
+      - seaborn==0.13.0
+      - sentencepiece==0.2.0
+      - shutilwhich==1.1.0
+      - six==1.16.0
+      - stack-data==0.6.0
+      - stanza==1.9.2
+      - statsmodels==0.14.0
+      - sympy==1.11.1
+      - tempdir==0.7.1
+      - threadpoolctl==3.2.0
+      - tokenizers==0.20.0
+      - tomli==2.0.1
+      - torch==2.0.0
+      - tornado==6.2
+      - tqdm==4.66.5
+      - traitlets==5.7.1
+      - transformers==4.45.1
+      - triton==2.0.0
+      - typing-extensions==4.12.2
+      - tzdata==2023.3
+      - urllib3==2.1.0
+      - wcwidth==0.2.5
+      - xxhash==3.5.0
+      - yarl==1.13.0
+      - zipp==3.12.0
+prefix: /home/yiren/new_ssd2/chunhui/miniconda/envs/impossible_llm

requirements.txt

@@ -0,0 +1,94 @@
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+contourpy==1.1.1
+cycler==0.12.1
+data==0.4
+emoji==2.8.0
+fonttools==4.45.1
+fsspec==2023.10.0
+funcsigs==1.0.2
+future==0.18.3
+huggingface-hub==0.19.4
+idna==3.6
+iniconfig==2.0.0
+joblib==1.3.2
+kiwisolver==1.4.5
+latex==0.7.0
+matplotlib==3.7.5
+mizani==0.9.3
+nltk==3.8.1
+numpy==1.24.4
+pandas==2.0.3
+patsy==0.5.3
+Pillow==10.1.0
+plotnine==0.12.4
+pluggy==1.3.0
+pluralizer==1.2.0
+protobuf==4.25.1
+pyparsing==3.1.1
+pytest==7.4.3
+pytz==2023.3.post1
+PyYAML==6.0.1
+regex==2023.10.3
+requests==2.31.0
+safetensors==0.4.1
+scikit-learn==1.3.2
+scipy==1.10.1
+seaborn==0.13.0
+shutilwhich==1.1.0
+stanza==1.6.1
+statsmodels==0.14.0
+tempdir==0.7.1
+threadpoolctl==3.2.0
+tokenizers==0.15.0
+tomli==2.0.1
+torch==2.0.0
+tqdm==4.66.1
+transformers==4.35.2
+triton==2.0.0
+tzdata==2023.3
+urllib3==2.1.0
+asttokens==2.0.5
+comm==0.1.2
+debugpy==1.6.7
+decorator==5.1.1
+exceptiongroup==1.1.0
+executing==1.2.0
+filelock==3.12.2
+gmpy2==2.1.0
+importlib-metadata==6.0.0
+ipykernel==6.23.1
+ipython==8.0.0
+jedi==0.18.2
+Jinja2==3.1.2
+jupyter_client==8.1.0
+jupyter_core==5.1.0
+latexcodec==1.0.0
+MarkupSafe==2.1.2
+matplotlib-inline==0.1.6
+mpmath==1.2.1
+nest-asyncio==1.5.6
+networkx==2.8.6
+packaging==23.0
+parso==0.8.3
+pexpect==4.8.0
+pickleshare==0.7.5
+platformdirs==2.5.2
+prompt-toolkit==3.0.30
+psutil==5.9.1
+ptyprocess==0.7.0
+pure-eval==0.2.2
+Pygments==2.15.0
+python-dateutil==2.8.2
+pyzmq==23.0.0
+six==1.16.0
+stack-data==0.6.0
+sympy==1.11.1
+tornado==6.2
+traitlets==5.7.1
+typing_extensions==4.6.0
+wcwidth==0.2.5
+zipp==3.12.0
+
+

requirements_1.txt

@@ -0,0 +1,51 @@
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+contourpy==1.2.0
+cycler==0.12.1
+data==0.4
+emoji==2.8.0
+fonttools==4.45.1
+fsspec==2023.10.0
+funcsigs==1.0.2
+future==0.18.3
+huggingface-hub==0.19.4
+idna==3.6
+iniconfig==2.0.0
+joblib==1.3.2
+kiwisolver==1.4.5
+latex==0.7.0
+matplotlib==3.8.2
+mizani==0.9.3
+nltk==3.8.1
+numpy==1.26.2
+pandas==2.1.3
+patsy==0.5.3
+Pillow==10.1.0
+plotnine==0.12.4
+pluggy==1.3.0
+pluralizer==1.2.0
+protobuf==4.25.1
+pyparsing==3.1.1
+pytest==7.4.3
+pytz==2023.3.post1
+PyYAML==6.0.1
+regex==2023.10.3
+requests==2.31.0
+safetensors==0.4.1
+scikit-learn==1.3.2
+scipy==1.11.4
+seaborn==0.13.0
+shutilwhich==1.1.0
+stanza==1.6.1
+statsmodels==0.14.0
+tempdir==0.7.1
+threadpoolctl==3.2.0
+tokenizers==0.15.0
+tomli==2.0.1
+torch==2.0.0
+tqdm==4.66.1
+transformers==4.35.2
+triton==2.0.0
+tzdata==2023.3
+urllib3==2.1.0

requirements_2.txt

@@ -0,0 +1,82 @@
+asttokens==2.2.1
+comm==0.1.2
+debugpy==1.6.7
+decorator==5.1.1
+exceptiongroup==1.1.0
+executing==1.2.0
+filelock==3.12.2
+gmpy2==2.1.0
+importlib-metadata==6.0.0
+ipykernel==6.23.1
+ipython==8.0.0
+jedi==0.18.2
+Jinja2==3.1.2
+jupyter_client==8.1.0
+jupyter_core==5.1.0
+latexcodec==1.0.0
+MarkupSafe==2.1.2
+matplotlib-inline==0.1.6
+mpmath==1.2.1
+nest-asyncio==1.5.6
+networkx==2.8.6
+packaging==23.0
+parso==0.8.3
+pexpect==4.8.0
+pickleshare==0.7.5
+platformdirs==2.5.2
+prompt-toolkit==3.0.30
+psutil==5.9.1
+ptyprocess==0.7.0
+pure-eval==0.2.2
+Pygments==2.15.0
+python-dateutil==2.8.2
+pyzmq==23.0.0
+six==1.16.0
+stack-data==0.6.0
+sympy==1.11.1
+tornado==6.2
+traitlets==5.7.1
+typing_extensions==4.6.0
+wcwidth==0.2.5
+zipp==3.12.0
+# asttokens @ file:///home/conda/feedstock_root/build_artifacts/asttokens_1698341106958/work
+# comm @ file:///home/conda/feedstock_root/build_artifacts/comm_1691044910542/work
+# debugpy @ file:///croot/debugpy_1690905042057/work
+# decorator @ file:///home/conda/feedstock_root/build_artifacts/decorator_1641555617451/work
+# exceptiongroup @ file:///home/conda/feedstock_root/build_artifacts/exceptiongroup_1700579780973/work
+# executing @ file:///home/conda/feedstock_root/build_artifacts/executing_1698579936712/work
+# filelock @ file:///croot/filelock_1700591183607/work
+# gmpy2 @ file:///tmp/build/80754af9/gmpy2_1645455533097/work
+# importlib-metadata @ file:///home/conda/feedstock_root/build_artifacts/importlib-metadata_1688754491823/work
+# ipykernel @ file:///home/conda/feedstock_root/build_artifacts/ipykernel_1698244021190/work
+# ipython @ file:///home/conda/feedstock_root/build_artifacts/ipython_1701092366260/work
+# jedi @ file:///home/conda/feedstock_root/build_artifacts/jedi_1696326070614/work
+# Jinja2 @ file:///croot/jinja2_1666908132255/work
+# jupyter_client @ file:///home/conda/feedstock_root/build_artifacts/jupyter_client_1699283905679/work
+# jupyter_core @ file:///home/conda/feedstock_root/build_artifacts/jupyter_core_1698673647019/work
+# latexcodec @ file:///home/conda/feedstock_root/build_artifacts/latexcodec_1592937263153/work
+# MarkupSafe @ file:///opt/conda/conda-bld/markupsafe_1654597864307/work
+# matplotlib-inline @ file:///home/conda/feedstock_root/build_artifacts/matplotlib-inline_1660814786464/work
+# mpmath @ file:///croot/mpmath_1690848262763/work
+# nest-asyncio @ file:///home/conda/feedstock_root/build_artifacts/nest-asyncio_1697083700168/work
+# networkx @ file:///croot/networkx_1690561992265/work
+# packaging @ file:///home/conda/feedstock_root/build_artifacts/packaging_1696202382185/work
+# parso @ file:///home/conda/feedstock_root/build_artifacts/parso_1638334955874/work
+# pexpect @ file:///home/conda/feedstock_root/build_artifacts/pexpect_1667297516076/work
+# pickleshare @ file:///home/conda/feedstock_root/build_artifacts/pickleshare_1602536217715/work
+# platformdirs @ file:///home/conda/feedstock_root/build_artifacts/platformdirs_1699715570510/work
+# prompt-toolkit @ file:///home/conda/feedstock_root/build_artifacts/prompt-toolkit_1699963054032/work
+# psutil @ file:///home/conda/feedstock_root/build_artifacts/psutil_1695367094274/work
+# ptyprocess @ file:///home/conda/feedstock_root/build_artifacts/ptyprocess_1609419310487/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl
+# pure-eval @ file:///home/conda/feedstock_root/build_artifacts/pure_eval_1642875951954/work
+# Pygments @ file:///home/conda/feedstock_root/build_artifacts/pygments_1700607939962/work
+# python-dateutil @ file:///home/conda/feedstock_root/build_artifacts/python-dateutil_1626286286081/work
+# pyzmq @ file:///home/conda/feedstock_root/build_artifacts/pyzmq_1652965335788/work
+# six @ file:///home/conda/feedstock_root/build_artifacts/six_1620240208055/work
+# stack-data @ file:///home/conda/feedstock_root/build_artifacts/stack_data_1669632077133/work
+# sympy @ file:///croot/sympy_1668202399572/work
+# tornado @ file:///home/conda/feedstock_root/build_artifacts/tornado_1695373560918/work
+# traitlets @ file:///home/conda/feedstock_root/build_artifacts/traitlets_1701095650114/work
+# typing_extensions @ file:///croot/typing_extensions_1690297465030/work
+# wcwidth @ file:///home/conda/feedstock_root/build_artifacts/wcwidth_1700607916581/work
+# zipp @ file:///home/conda/feedstock_root/build_artifacts/zipp_1695255097490/work

test.py

@@ -0,0 +1,23 @@
+from huggingface_hub import upload_folder
+
+upload_folder(
+    folder_path="/home/yiren/new_ssd2/chunhui/yaning/project/mission-impossible-language-models",
+    repo_id="Yaning1001/impossible_llm",
+    path_in_repo="mission-impossible-language-models"
+)
+
+# import torch
+# from transformers import AutoModelForCausalLM, AutoTokenizer
+
+# model_id_1 = "meta-llama/Llama-3.2-3B"
+# model_id_2 = "Qwen/Qwen2.5-7B"
+
+# # Check your authentication - this line should succeed without errors!
+# tokenizer_0 = AutoTokenizer.from_pretrained('gpt2')
+# tokenizer_1 = AutoTokenizer.from_pretrained(model_id_1)
+# tokenizer_2 = AutoTokenizer.from_pretrained(model_id_2)
+# tokenizer_2.pad_token = tokenizer_1.pad_token
+
+# print("tokenizer_0.pad_token:", type(tokenizer_0.pad_token))
+# print("tokenizer_1.pad_token:", type(tokenizer_1.pad_token))
+# print("tokenizer_2.pad_token:", type(tokenizer_2.pad_token))

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/refs/main

@@ -0,0 +1 @@
+13afe5124825b4f3751f836b40dafda64c1ed062

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/5cc0ffe09ee49f7be6ca7c794ee6bd7245e84e60/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "transformers_version": "4.45.0.dev0",
+  "do_sample": true,
+  "temperature": 0.6,
+  "top_p": 0.9
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/5cc0ffe09ee49f7be6ca7c794ee6bd7245e84e60/model.safetensors.index.json

@@ -0,0 +1,261 @@
+{
+  "metadata": {
+    "total_size": 6425499648
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1050/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fba7b84ac8c089d417b794dfc0527040604b90e2cdfe6e9df5b55afe9eab61a
+size 17210282

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1050/tokenizer_config.json

@@ -0,0 +1,2078 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128256": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128257": {
+      "content": "🅁",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.2-3B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70c0a65a1f48609b709cff4968a5ca16f280b65798ab3e96b60a43db042976c5
+size 20599

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01a7577bee4c118f55763b566db086aa10d2ec5a2d46bc623db8c8381f4ba7c5
+size 20599

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db09e5b1b76154e3b7043e11a650fd3051b5b3ab595a43c485df65a086c6f540
+size 20599

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:671e665f8b90df160f325e725a02c3635553d4058486bf76788722e80c8864e6
+size 20599

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7169d16d22347ae1ff1b51cfe165ad5333e47dc6b178e93f63331e743d23305a
+size 20599

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d866764dfa788f78507a77e36fe7f085156ee01dd1d0ef2cc476f85d7a3134dd
+size 20599

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eeb1301b45d7398af92537a9959369547a41f37de86e06f898e44fa1c6e0ffcc
+size 20599

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8b8dd10e3f7f429a30da6714b6bc34fffe93612cca0d55e0213efbb5aac3b13
+size 627

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1200/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d396f0642c83730ebff54eaafc7dfb38092d5d41fd28ce6ebbd56ec539612e4b
+size 6011

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1350/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.2-3B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1350/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.45.1"
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1350/latest

@@ -0,0 +1 @@
+global_step1350

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1350/model.safetensors.index.json

@@ -0,0 +1,262 @@
+{
+  "metadata": {
+    "total_size": 7213504512
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1350/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1350/trainer_state.json

@@ -0,0 +1,96 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 6.467065868263473,
+  "eval_steps": 500,
+  "global_step": 1350,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.718562874251497,
+      "grad_norm": 0.661852240562439,
+      "learning_rate": 4.6899038461538465e-05,
+      "loss": 2.1718,
+      "step": 150
+    },
+    {
+      "epoch": 1.437125748502994,
+      "grad_norm": 0.7592059373855591,
+      "learning_rate": 4.329326923076924e-05,
+      "loss": 1.5828,
+      "step": 300
+    },
+    {
+      "epoch": 2.155688622754491,
+      "grad_norm": 0.6453707218170166,
+      "learning_rate": 3.96875e-05,
+      "loss": 1.478,
+      "step": 450
+    },
+    {
+      "epoch": 2.874251497005988,
+      "grad_norm": 0.6175025105476379,
+      "learning_rate": 3.608173076923077e-05,
+      "loss": 1.4098,
+      "step": 600
+    },
+    {
+      "epoch": 3.592814371257485,
+      "grad_norm": 0.7075008749961853,
+      "learning_rate": 3.247596153846154e-05,
+      "loss": 1.3483,
+      "step": 750
+    },
+    {
+      "epoch": 4.311377245508982,
+      "grad_norm": 0.8413554430007935,
+      "learning_rate": 2.8870192307692306e-05,
+      "loss": 1.3009,
+      "step": 900
+    },
+    {
+      "epoch": 5.029940119760479,
+      "grad_norm": 0.9734054207801819,
+      "learning_rate": 2.5264423076923078e-05,
+      "loss": 1.2617,
+      "step": 1050
+    },
+    {
+      "epoch": 5.748502994011976,
+      "grad_norm": 0.8429175019264221,
+      "learning_rate": 2.1658653846153847e-05,
+      "loss": 1.1814,
+      "step": 1200
+    },
+    {
+      "epoch": 6.467065868263473,
+      "grad_norm": 0.9255201816558838,
+      "learning_rate": 1.8052884615384616e-05,
+      "loss": 1.1207,
+      "step": 1350
+    }
+  ],
+  "logging_steps": 150,
+  "max_steps": 2080,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 150,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.9639047370404004e+18,
+  "train_batch_size": 3,
+  "trial_name": null,
+  "trial_params": null
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1500/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.2-3B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1500/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.45.1"
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1500/latest

@@ -0,0 +1 @@
+global_step1500

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1500/model.safetensors.index.json

@@ -0,0 +1,262 @@
+{
+  "metadata": {
+    "total_size": 7213504512
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1500/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:638670b4f96148564ef368df48443cdaa781c9697294452c3ff15d3f926e388a
+size 20599

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1500/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:595fa5c22607d6ab832d0b14385e9e6b9bfeb5d4c181d0acb849edd1df3e66a1
+size 20599

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1500/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1500/tokenizer_config.json

@@ -0,0 +1,2078 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128256": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128257": {
+      "content": "🅁",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1500/trainer_state.json

@@ -0,0 +1,103 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 7.18562874251497,
+  "eval_steps": 500,
+  "global_step": 1500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.718562874251497,
+      "grad_norm": 0.661852240562439,
+      "learning_rate": 4.6899038461538465e-05,
+      "loss": 2.1718,
+      "step": 150
+    },
+    {
+      "epoch": 1.437125748502994,
+      "grad_norm": 0.7592059373855591,
+      "learning_rate": 4.329326923076924e-05,
+      "loss": 1.5828,
+      "step": 300
+    },
+    {
+      "epoch": 2.155688622754491,
+      "grad_norm": 0.6453707218170166,
+      "learning_rate": 3.96875e-05,
+      "loss": 1.478,
+      "step": 450
+    },
+    {
+      "epoch": 2.874251497005988,
+      "grad_norm": 0.6175025105476379,
+      "learning_rate": 3.608173076923077e-05,
+      "loss": 1.4098,
+      "step": 600
+    },
+    {
+      "epoch": 3.592814371257485,
+      "grad_norm": 0.7075008749961853,
+      "learning_rate": 3.247596153846154e-05,
+      "loss": 1.3483,
+      "step": 750
+    },
+    {
+      "epoch": 4.311377245508982,
+      "grad_norm": 0.8413554430007935,
+      "learning_rate": 2.8870192307692306e-05,
+      "loss": 1.3009,
+      "step": 900
+    },
+    {
+      "epoch": 5.029940119760479,
+      "grad_norm": 0.9734054207801819,
+      "learning_rate": 2.5264423076923078e-05,
+      "loss": 1.2617,
+      "step": 1050
+    },
+    {
+      "epoch": 5.748502994011976,
+      "grad_norm": 0.8429175019264221,
+      "learning_rate": 2.1658653846153847e-05,
+      "loss": 1.1814,
+      "step": 1200
+    },
+    {
+      "epoch": 6.467065868263473,
+      "grad_norm": 0.9255201816558838,
+      "learning_rate": 1.8052884615384616e-05,
+      "loss": 1.1207,
+      "step": 1350
+    },
+    {
+      "epoch": 7.18562874251497,
+      "grad_norm": 1.2465689182281494,
+      "learning_rate": 1.4447115384615385e-05,
+      "loss": 1.0608,
+      "step": 1500
+    }
+  ],
+  "logging_steps": 150,
+  "max_steps": 2080,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 150,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.1821163744893338e+18,
+  "train_batch_size": 3,
+  "trial_name": null,
+  "trial_params": null
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1500/zero_to_fp32.py

@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1650/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.2-3B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1650/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.45.1"
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1650/latest

@@ -0,0 +1 @@
+global_step1650

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1650/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c146f631f86bf0f652d7a36cc3ef8abae906fa1b54abb7af60635fac7919c55
+size 20599

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1650/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:595fa5c22607d6ab832d0b14385e9e6b9bfeb5d4c181d0acb849edd1df3e66a1
+size 20599

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1650/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe730d7c46d9e44b7cee973c364bc4ff93710ee1b6faa309a35e497360023b72
+size 627

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1650/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d396f0642c83730ebff54eaafc7dfb38092d5d41fd28ce6ebbd56ec539612e4b
+size 6011

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1650/zero_to_fp32.py

@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1800/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.2-3B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}

train/checkpoints/Llama-3.2-3B/babylm_reverse_full_10M_seed0/runs/checkpoint-1800/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.45.1"
+}