End of training

README.md

@@ -0,0 +1,48 @@
+---
+library_name: transformers
+base_model: Wonder-Griffin/JudgeLLM2
+tags:
+- generated_from_trainer
+model-index:
+- name: The_Judge
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# The_Judge
+
+This model is a fine-tuned version of [Wonder-Griffin/JudgeLLM2](https://huggingface.co/Wonder-Griffin/JudgeLLM2) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 3.0
+
+### Framework versions
+
+- Transformers 4.45.0.dev0
+- Pytorch 2.4.0+cu124
+- Datasets 2.20.0
+- Tokenizers 0.19.1

config.json

@@ -0,0 +1,102 @@
+{
+  "_name_": "Judge-GPT2",
+  "_name_or_path": "Wonder-Griffin/JudgeLLM2",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2Model"
+  ],
+  "attn_pdrop": 0.1,
+  "batch_size": 32,
+  "bias": true,
+  "block_size": 512,
+  "bos_token_id": 50256,
+  "dim_feedforward": 3072,
+  "dropout": 0.1,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "ff_expansion_factor": 4,
+  "hidden_act": "gelu",
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2",
+    "3": "LABEL_3",
+    "4": "LABEL_4"
+  },
+  "inference_mode": true,
+  "initializer_range": 0.02,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2,
+    "LABEL_3": 3,
+    "LABEL_4": 4
+  },
+  "label_smoothing": 0.1,
+  "layer_norm_epsilon": 1e-05,
+  "learning_rate": 0.0003,
+  "log_interval": 100,
+  "max_grad_norm": 1.0,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 512,
+  "output_dir": "C:/Users/wonde/output",
+  "pretrained_weights": "Wonder-Griffin/JudgeLLM2",
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_heads": {
+    "classifier_head": {
+      "params": {
+        "num_labels": 5
+      },
+      "type": "JudgeClassifier"
+    },
+    "lm_head": {
+      "params": {
+        "vocab_size": 50257
+      },
+      "type": "JudgeCasualLMHead"
+    },
+    "qa_head": {
+      "params": {
+        "num_labels": 2
+      },
+      "type": "JudgeWithQA"
+    }
+  },
+  "task_specific_params": {
+    "question-answering": {
+      "max_answer_length": 100
+    },
+    "sequence-classification": {
+      "eval_steps": 500
+    },
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 100
+    }
+  },
+  "tokenizer": {
+    "params": {
+      "vocab_size": 50257
+    },
+    "type": "AutoTokenizer"
+  },
+  "torch_dtype": "float32",
+  "total_steps": 10000,
+  "transformers_version": "4.45.0.dev0",
+  "use_cache": true,
+  "vocab_size": 30522,
+  "warmup_steps": 1000,
+  "weight_decay": 0.01
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7611a89d0d92c222df86ba901d724f356efb57ae7f96a425528464f3c3a410e
+size 435573648

pyJudgeXL_model.py

@@ -0,0 +1,122 @@
+# Configuration
+config = {
+    "learning_rate": 1e-4,
+    "batch_size": 32,
+    "vocab_size": 30522,
+    "max_len": 256,
+    "hidden_size": 768,
+    "dropout": 0.1,
+    "n_layer": 12,
+    "n_head": 12,
+    "ff_expansion_factor": 4,
+    "rnn_units": 768,
+    "num_labels": 5
+}
+
+class MyClass:
+    def __init__(self, value):
+        self.value = value
+
+# Custom Initializer
+def custom_initializer(shape):
+    return torch.normal(mean=0.0, std=0.02, size=shape)
+
+class CustomEmbedding(nn.Module):
+    def __init__(self, vocab_size, hidden_size):
+        super(CustomEmbedding, self).__init__()
+        self.embedding = nn.Embedding(vocab_size, hidden_size, _weight=custom_initializer((vocab_size, hidden_size)))
+
+    def forward(self, inputs):
+        return self.embedding(inputs)
+
+class PositionalEncoding(nn.Module):
+    def __init__(self, n_embd, max_len=5000):
+        super(PositionalEncoding, self).__init__()
+        self.n_embd = n_embd
+        self.max_len = max_len
+
+        pe = torch.zeros(max_len, n_embd)
+        position = torch.arange(0, max_len).unsqueeze(1).float()
+        div_term = torch.exp(torch.arange(0, n_embd, 2).float() * -(np.log(10000.0) / n_embd))
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+        pe = pe.unsqueeze(0).transpose(0, 1)
+        self.register_buffer('pe', pe)
+
+    def forward(self, x):
+        return x + self.pe[:x.size(0), :]
+
+class MultiheadAttention(nn.Module):
+    def __init__(self, config):
+        super(MultiheadAttention, self).__init__()
+        self.attention = nn.MultiheadAttention(config['hidden_size'], config['n_head'], dropout=config['dropout'])
+
+    def forward(self, v, k, q, mask=None):
+        attn_output, attn_output_weights = self.attention(q, k, v, attn_mask=mask)
+        return attn_output
+
+class FeedForward(nn.Module):
+    def __init__(self, config):
+        super(FeedForward, self).__init__()
+        self.dense1 = nn.Linear(config['hidden_size'], config['hidden_size'] * config['ff_expansion_factor'])
+        self.dense2 = nn.Linear(config['hidden_size'] * config['ff_expansion_factor'], config['hidden_size'])
+        self.dropout = nn.Dropout(config['dropout'])
+
+    def forward(self, x):
+        x = torch.nn.functional.gelu(self.dense1(x))
+        x = self.dropout(x)
+        return self.dense2(x)
+
+class TransformerXLBlock(nn.Module):
+    def __init__(self, config):
+        super(TransformerXLBlock, self).__init__()
+        self.attn = MultiheadAttention(config)
+        self.ff = FeedForward(config)
+        self.ln1 = nn.LayerNorm(config['hidden_size'])
+        self.ln2 = nn.LayerNorm(config['hidden_size'])
+
+    def forward(self, x, mask=None):
+        attn_out = self.attn(v=x, k=x, q=x, mask=mask)
+        out1 = self.ln1(x + attn_out)
+        ff_out = self.ff(out1)
+        return self.ln2(out1 + ff_out)
+
+class JudgeXL(nn.Module):
+    def __init__(self, config):
+        super(JudgeXL, self).__init__()
+        self.token_embedding = CustomEmbedding(config['vocab_size'], config['hidden_size'])
+        self.pos_encoding = PositionalEncoding(config['hidden_size'], config['max_len'])
+        self.transformer_blocks = nn.ModuleList([TransformerXLBlock(config) for _ in range(config['n_layer'])])
+        self.ln_f = nn.LayerNorm(config['hidden_size'])
+        self.rnn = nn.LSTM(config['hidden_size'], config['rnn_units'], num_layers=2, dropout=config['dropout'], bidirectional=True, batch_first=True)
+        self.fc = nn.Linear(config['rnn_units'] * 2, config['vocab_size'])  # Adjusted to rnn_units * 2
+    
+    def forward(self, x, mask=None):
+        x = self.token_embedding(x)
+        x = self.pos_encoding(x)
+        for block in self.transformer_blocks:
+            x = block(x, mask=mask)
+        x = self.ln_f(x)
+        x, _ = self.rnn(x)
+        x = self.fc(x)
+        return x
+    
+    def generate(self, prompt, max_len=100):
+        self.eval()
+        input_ids = self.tokenizer(prompt, return_tensors='pt').input_ids.to(device)
+        generated = input_ids
+        with torch.no_grad():
+            for _ in range(max_len):
+                outputs = self.forward(generated)
+                next_token_logits = outputs[:, :]  # Adjusted indexing
+                next_token_id = torch.argmax(next_token_logits, dim=-1).unsqueeze(0)
+                generated = torch.cat((generated, next_token_id), dim=1)
+                if next_token_id.item() == self.tokenizer.sep_token_id:
+                    break
+        generated_text = self.tokenizer.decode(generated[0], skip_special_tokens=True)
+        return generated_text
+
+# Load the last saved model
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model = JudgeXL(config)
+model = torch.load('C:/AIstuffing/Judge_XL-LLM/xl-llm_weights/judgeXL-LLm_wiki.pth', weights_only=False)

tokenizer1.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a4bfa2daf9cb9275703fcadd2e7953704653c2a206b1ea0852fad26a5e76c80
+size 82362540

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca93e44304ed6ec37809ad1da1d61576ecf6389b60e134a029f36fbbbf24ebec
+size 5176