feat: update model

classification_report.txt

@@ -1,14 +1,14 @@
                 precision    recall  f1-score   support
 
-NOT-A-QUESTION       0.96      0.92      0.94       950
-       FACTOID       0.84      0.79      0.81       980
-        DEBATE       0.90      0.95      0.92       916
-EVIDENCE-BASED       0.86      0.92      0.89       950
-   INSTRUCTION       0.85      0.92      0.88       980
-        REASON       0.88      0.86      0.87       960
-    EXPERIENCE       0.82      0.76      0.79       980
-    COMPARISON       0.93      0.93      0.93       980
+NOT-A-QUESTION       0.99      0.99      0.99       557
+       FACTOID       0.92      0.87      0.90       896
+        DEBATE       0.92      0.96      0.94       472
+EVIDENCE-BASED       0.88      0.95      0.91       568
+   INSTRUCTION       0.95      0.94      0.94       662
+        REASON       0.94      0.94      0.94       493
+    EXPERIENCE       0.86      0.85      0.85       686
+    COMPARISON       0.96      0.96      0.96       679
 
-      accuracy                           0.88      7696
-     macro avg       0.88      0.88      0.88      7696
-  weighted avg       0.88      0.88      0.88      7696
+      accuracy                           0.92      5013
+     macro avg       0.93      0.93      0.93      5013
+  weighted avg       0.93      0.92      0.92      5013

config.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d64b32cf7198deee34a207a62d0681ea08b0b2ae51b5d011324791e5b24c6a9a
+oid sha256:e6f552aee9c2bd981c72a9e0daa6cf9e9a6d343e718338f8462d958b1b9cd73b
 size 1118

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:33a25f9cc0e6e82ac88d37fb2ec3bfb4e61c9751e5db98d13f04692a5ab2f734
+oid sha256:061cc36ce4649a1ca2c988c042eece9a041a7bae6589619c177ef053fdbadeb5
 size 1112223464

special_tokens_map.json

@@ -1,15 +1,3 @@
-{
-  "bos_token": "<s>",
-  "cls_token": "<s>",
-  "eos_token": "</s>",
-  "mask_token": {
-    "content": "<mask>",
-    "lstrip": true,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": "<pad>",
-  "sep_token": "</s>",
-  "unk_token": "<unk>"
-}
+version https://git-lfs.github.com/spec/v1
+oid sha256:06e405a36dfe4b9604f484f6a1e619af1a7f7d09e34a8555eb0b77b66318067f
+size 280

test_results.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b90b48f6007b8ab4fdc46e83e9dcf2561802f5946e48bd665fde14a9ba3fa7d
-size 778
+oid sha256:4f85874cae37b57c474bb4450d166414c18f4e141ad6eefc3233030664397ecc
+size 805

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c088c06cf975b7097e469bd69630cdb0d675c6db1ce3af1042b6e19c6d01f22
-size 17082999
+oid sha256:3a56def25aa40facc030ea8b0b87f3688e4b3c39eb8b45d5702b3a1300fe2a20
+size 17082734

tokenizer_config.json

@@ -1,55 +1,3 @@
-{
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<pad>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "3": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "250001": {
-      "content": "<mask>",
-      "lstrip": true,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "bos_token": "<s>",
-  "clean_up_tokenization_spaces": false,
-  "cls_token": "<s>",
-  "eos_token": "</s>",
-  "extra_special_tokens": {},
-  "mask_token": "<mask>",
-  "model_max_length": 512,
-  "pad_token": "<pad>",
-  "sep_token": "</s>",
-  "tokenizer_class": "XLMRobertaTokenizer",
-  "unk_token": "<unk>"
-}
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccf223ba3d5b3cc7fa6c3bf451f3bb40557a5c92b0aa33f63d17802ff1a96fd9
+size 1178

training_scripts/run_training_manual.sh

@@ -26,7 +26,7 @@ EPOCHS=6
 BATCH_SIZE=16
 LEARNING_RATE=2e-5
 MAX_LENGTH=128
-WARMUP_STEPS=500
+WARMUP_RATIO=0.1
 WEIGHT_DECAY=0.1
 DROPOUT=0.2
 
@@ -44,6 +44,7 @@ echo "  Epochs:           $EPOCHS"
 echo "  Batch size:       $BATCH_SIZE"
 echo "  Learning rate:    $LEARNING_RATE"
 echo "  Max length:       $MAX_LENGTH"
+echo "  Warmup ratio:     $WARMUP_RATIO"
 echo "  Weight decay:     $WEIGHT_DECAY"
 echo "  Dropout:          $DROPOUT"
 echo ""
@@ -84,7 +85,7 @@ python train_nfqa_model.py \
     --batch-size "$BATCH_SIZE" \
     --learning-rate "$LEARNING_RATE" \
     --max-length "$MAX_LENGTH" \
-    --warmup-steps "$WARMUP_STEPS" \
+    --warmup-ratio "$WARMUP_RATIO" \
     --weight-decay "$WEIGHT_DECAY" \
     --dropout "$DROPOUT" \
     "$@"  # Pass any additional arguments from command line

training_scripts/train_nfqa_model.py

@@ -26,6 +26,7 @@ from torch.utils.data import Dataset, DataLoader
 from torch.optim import AdamW
 from transformers import (
     AutoTokenizer,
+    AutoConfig,
     AutoModelForSequenceClassification,
     get_linear_schedule_with_warmup
 )
@@ -113,6 +114,7 @@ def train_epoch(model, train_loader, optimizer, scheduler, device):
         labels = batch['labels'].to(device)
 
         # Forward pass
+        optimizer.zero_grad()
         outputs = model(
             input_ids=input_ids,
             attention_mask=attention_mask,
@@ -123,7 +125,6 @@ def train_epoch(model, train_loader, optimizer, scheduler, device):
         total_loss += loss.item()
 
         # Backward pass
-        optimizer.zero_grad()
         loss.backward()
         torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
         optimizer.step()
@@ -245,45 +246,6 @@ def load_data(file_path):
         raise
 
 
-def create_data_splits(questions, labels, test_size=0.2, val_size=0.1):
-    """Create train/val/test splits"""
-    print("\nCreating data splits...")
-
-    # First split: separate test set
-    train_val_questions, test_questions, train_val_labels, test_labels = train_test_split(
-        questions,
-        labels,
-        test_size=test_size,
-        random_state=RANDOM_SEED,
-        stratify=labels
-    )
-
-    # Second split: separate validation from training
-    train_questions, val_questions, train_labels, val_labels = train_test_split(
-        train_val_questions,
-        train_val_labels,
-        test_size=val_size / (1 - test_size),
-        random_state=RANDOM_SEED,
-        stratify=train_val_labels
-    )
-
-    print(f"\nData splits:")
-    print(f"  Training:   {len(train_questions):4d} examples ({len(train_questions)/len(questions)*100:5.1f}%)")
-    print(f"  Validation: {len(val_questions):4d} examples ({len(val_questions)/len(questions)*100:5.1f}%)")
-    print(f"  Test:       {len(test_questions):4d} examples ({len(test_questions)/len(questions)*100:5.1f}%)")
-    print(f"  Total:      {len(questions):4d} examples")
-
-    # Verify class distribution
-    print("\nClass distribution per split:")
-    for split_name, split_labels in [('Train', train_labels), ('Val', val_labels), ('Test', test_labels)]:
-        counts = Counter(split_labels)
-        print(f"\n{split_name}:")
-        for label_id in sorted(counts.keys()):
-            cat_name = ID2LABEL[label_id]
-            print(f"  {cat_name:20s}: {counts[label_id]:3d}")
-
-    return train_questions, val_questions, test_questions, train_labels, val_labels, test_labels
-
 
 def plot_training_curves(history, best_val_f1, output_dir):
     """Plot and save training curves"""
@@ -495,8 +457,8 @@ def main():
                         help='Number of epochs (default: 10)')
     parser.add_argument('--learning-rate', type=float, default=2e-5,
                         help='Learning rate (default: 2e-5)')
-    parser.add_argument('--warmup-steps', type=int, default=500,
-                        help='Warmup steps (default: 500)')
+    parser.add_argument('--warmup-ratio', type=float, default=0.1,
+                        help='Fraction of total training steps used for warmup (default: 0.1)')
     parser.add_argument('--weight-decay', type=float, default=0.01,
                         help='Weight decay (default: 0.01)')
     parser.add_argument('--dropout', type=float, default=0.1,
@@ -543,6 +505,7 @@ def main():
     print(f"Learning rate: {args.learning_rate}")
     print(f"Max length: {args.max_length}")
     print(f"Weight decay: {args.weight_decay}")
+    print(f"Warmup ratio: {args.warmup_ratio}")
     print(f"Dropout: {args.dropout}")
     print("="*80 + "\n")
 
@@ -621,14 +584,20 @@ def main():
     print("✓ Tokenizer loaded")
 
     print(f"\nLoading model: {args.model_name}")
+
+    # Configure dropout BEFORE instantiating the model
+    config = AutoConfig.from_pretrained(args.model_name)
+    config.num_labels = len(NFQA_CATEGORIES)
+    config.id2label = ID2LABEL
+    config.label2id = LABEL2ID
+    config.hidden_dropout_prob = args.dropout
+    config.attention_probs_dropout_prob = args.dropout
+    config.classifier_dropout = args.dropout
+
+    # Now create model with configured dropout
     model = AutoModelForSequenceClassification.from_pretrained(
         args.model_name,
-        num_labels=len(NFQA_CATEGORIES),
-        id2label=ID2LABEL,
-        label2id=LABEL2ID,
-        hidden_dropout_prob=args.dropout,
-        attention_probs_dropout_prob=args.dropout,
-        classifier_dropout=args.dropout
+        config=config
     )
     model.to(device)
 
@@ -659,15 +628,16 @@ def main():
     )
 
     total_steps = len(train_loader) * args.epochs
+    warmup_steps = int(args.warmup_ratio * total_steps)
     scheduler = get_linear_schedule_with_warmup(
         optimizer,
-        num_warmup_steps=args.warmup_steps,
+        num_warmup_steps=warmup_steps,
         num_training_steps=total_steps
     )
 
     print(f"\n✓ Optimizer and scheduler configured")
     print(f"  Total training steps: {total_steps}")
-    print(f"  Warmup steps: {args.warmup_steps}")
+    print(f"  Warmup steps: {warmup_steps} ({args.warmup_ratio*100:.0f}% of total)")
 
     # Training loop
     history = {
@@ -692,12 +662,12 @@ def main():
         # Train
         train_loss, train_acc = train_epoch(model, train_loader, optimizer, scheduler, device)
 
-        # Validate with detailed analysis
+        # Validate
         val_loss, val_acc, val_f1, val_preds, val_true = evaluate(
             model, val_loader, device,
             languages=val_langs,
             desc="Validating",
-            show_analysis=True
+            show_analysis=False
         )
 
         # Update history
@@ -820,7 +790,8 @@ def main():
             'batch_size': args.batch_size,
             'learning_rate': args.learning_rate,
             'num_epochs': args.epochs,
-            'warmup_steps': args.warmup_steps,
+            'warmup_ratio': args.warmup_ratio,
+            'warmup_steps': warmup_steps,
             'weight_decay': args.weight_decay,
             'dropout': args.dropout,
             'data_source': 'pre-split' if has_split_inputs else 'single_file',