Upload folder using huggingface_hub

CustomRBFQwen.py

@@ -0,0 +1,28 @@
+import torch
+import torch.nn as nn
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from RBFLayer import RBFLayer
+
+def l_norm(x, p=2):
+    return torch.norm(x, p=p, dim=-1)
+
+
+# Gaussian RBF
+def rbf_gaussian(x):
+    return (-x.pow(2)).exp()
+
+class CustomRBFFeedForward(nn.Module):
+    def __init__(self, in_features, out_features, num_kernels):
+        super(CustomRBFFeedForward, self).__init__()
+        # RBFLayer from the given implementation
+        self.rbf_layer = RBFLayer(
+            in_features_dim=in_features,  # Input size (e.g., 896)
+            num_kernels=num_kernels,  # Number of kernels in the RBF layer (can be tuned)
+            out_features_dim=out_features,  # Output size (e.g., 4864)
+            radial_function=rbf_gaussian,  # Use the Gaussian RBF
+            norm_function=l_norm  # Use Euclidean norm
+        )
+
+    def forward(self, x):
+        # Apply the RBF layer to the input x
+        return self.rbf_layer(x)

RBFLayer.py

@@ -0,0 +1,101 @@
+import torch
+import torch.nn as nn
+from typing import Callable
+
+
+class RBFLayer(nn.Module):
+    def __init__(self,
+                 in_features_dim: int,
+                 num_kernels: int,
+                 out_features_dim: int,
+                 radial_function: Callable[[torch.Tensor], torch.Tensor],
+                 norm_function: Callable[[torch.Tensor], torch.Tensor],
+                 normalization: bool = True,
+                 initial_shape_parameter: torch.Tensor = None,
+                 initial_centers_parameter: torch.Tensor = None,
+                 initial_weights_parameters: torch.Tensor = None,
+                 constant_shape_parameter: bool = False,
+                 constant_centers_parameter: bool = False,
+                 constant_weights_parameters: bool = False):
+        super(RBFLayer, self).__init__()
+
+        self.in_features_dim = in_features_dim
+        self.num_kernels = num_kernels
+        self.out_features_dim = out_features_dim
+        self.radial_function = radial_function
+        self.norm_function = norm_function
+        self.normalization = normalization
+
+        self.initial_shape_parameter = initial_shape_parameter
+        self.constant_shape_parameter = constant_shape_parameter
+
+        self.initial_centers_parameter = initial_centers_parameter
+        self.constant_centers_parameter = constant_centers_parameter
+
+        self.initial_weights_parameters = initial_weights_parameters
+        self.constant_weights_parameters = constant_weights_parameters
+
+        self._make_parameters()
+
+    def _make_parameters(self) -> None:
+        # Initialize linear combination weights
+        if self.constant_weights_parameters:
+            self.weights = nn.Parameter(self.initial_weights_parameters, requires_grad=False)
+        else:
+            self.weights = nn.Parameter(torch.zeros(self.out_features_dim, self.num_kernels, dtype=torch.float32))
+
+        # Initialize kernels' centers
+        if self.constant_centers_parameter:
+            self.kernels_centers = nn.Parameter(self.initial_centers_parameter, requires_grad=False)
+        else:
+            self.kernels_centers = nn.Parameter(torch.zeros(self.num_kernels, self.in_features_dim, dtype=torch.float32))
+
+        # Initialize shape parameter
+        if self.constant_shape_parameter:
+            self.log_shapes = nn.Parameter(self.initial_shape_parameter, requires_grad=False)
+        else:
+            self.log_shapes = nn.Parameter(torch.zeros(self.num_kernels, dtype=torch.float32))
+
+        self.reset()
+
+    def reset(self, upper_bound_kernels: float = 1.0, std_shapes: float = 0.1, gain_weights: float = 1.0) -> None:
+        if self.initial_centers_parameter is None:
+            nn.init.uniform_(self.kernels_centers, a=-upper_bound_kernels, b=upper_bound_kernels)
+
+        if self.initial_shape_parameter is None:
+            nn.init.normal_(self.log_shapes, mean=0.0, std=std_shapes)
+
+        if self.initial_weights_parameters is None:
+            nn.init.xavier_uniform_(self.weights, gain=gain_weights)
+
+    def forward(self, input: torch.Tensor) -> torch.Tensor:
+        """
+        Computes the output of the RBF layer given an input tensor.
+        Input has size [batch_size, sequence_length, in_features].
+        """
+
+        batch_size = input.size(0)
+        sequence_length = input.size(1)
+
+        # Expand centers to match the batch and sequence length
+        c = self.kernels_centers.expand(batch_size, sequence_length, self.num_kernels, self.in_features_dim)
+
+        # Compute differences between input and centers
+        diff = input.unsqueeze(2) - c  # Shape: [batch_size, sequence_length, num_kernels, in_features_dim]
+
+        # Apply norm function to get distances
+        r = self.norm_function(diff)  # Shape: [batch_size, sequence_length, num_kernels]
+
+        # Apply shape parameters (log_shapes) to the distances
+        eps_r = self.log_shapes.exp().unsqueeze(0).unsqueeze(0) * r
+
+        # Apply radial basis function (e.g., Gaussian)
+        rbfs = self.radial_function(eps_r)
+
+        if self.normalization:
+            rbfs = rbfs / (1e-9 + rbfs.sum(dim=-1, keepdim=True))
+
+        # Combine RBF outputs using the weights
+        out = (self.weights.unsqueeze(0).unsqueeze(0) * rbfs.unsqueeze(2)).sum(dim=-1)
+
+        return out

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "Qwen/Qwen2-0.5B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.41.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.41.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:460b1ce33a4a54d698e65d82f5bc3a4def3941de5b5775b0f96182a54ae2a359
+size 729331576

special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}