Repository: huggingface/transformers
Pull request: Add DeepseekV3ForSequenceClassification for Deepseek V3 models

# What does this PR do?
Add DeepseekV3ForSequenceClassification for DeepseekV3 models following generic practice

Testing:

Tested with single-node option, loss curve and AUC are all expected with no issue.

- All testes passed
- make fixup passed

Fixes #40199 


## Before submitting
- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
- [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#create-a-pull-request),
      Pull Request section?
- [ x Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. #40199 
- [x] Did you make sure to update the documentation with your changes? Here are the
      [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and
      [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation).
- [x] Did you write any new necessary tests?


## Who can review?

Anyone in the community is free to review the PR once the tests have passed. Feel free to tag
members/contributors who may be interested in your PR.
@ArthurZucker


Changed files:
filename: docs/source/en/model_doc/deepseek_v3.md
patch: @@ -183,3 +183,8 @@ error, it means NCCL was probably not loaded.
 
 [[autodoc]] DeepseekV3ForCausalLM
     - forward
+
+## DeepseekV3ForSequenceClassification
+
+[[autodoc]] DeepseekV3ForSequenceClassification
+    - forward

filename: src/transformers/models/auto/modeling_auto.py
patch: @@ -1183,6 +1183,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("deberta", "DebertaForSequenceClassification"),
         ("deberta-v2", "DebertaV2ForSequenceClassification"),
         ("deepseek_v2", "DeepseekV2ForSequenceClassification"),
+        ("deepseek_v3", "DeepseekV3ForSequenceClassification"),
         ("diffllama", "DiffLlamaForSequenceClassification"),
         ("distilbert", "DistilBertForSequenceClassification"),
         ("doge", "DogeForSequenceClassification"),

filename: src/transformers/models/deepseek_v3/modeling_deepseek_v3.py
patch: @@ -17,7 +17,7 @@
 from ...integrations import use_kernel_forward_from_hub
 from ...masking_utils import create_causal_mask
 from ...modeling_flash_attention_utils import FlashAttentionKwargs
-from ...modeling_layers import GradientCheckpointingLayer
+from ...modeling_layers import GenericForSequenceClassification, GradientCheckpointingLayer
 from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
 from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
 from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
@@ -678,4 +678,13 @@ def forward(
         )
 
 
-__all__ = ["DeepseekV3PreTrainedModel", "DeepseekV3Model", "DeepseekV3ForCausalLM"]
+class DeepseekV3ForSequenceClassification(GenericForSequenceClassification, DeepseekV3PreTrainedModel):
+    pass
+
+
+__all__ = [
+    "DeepseekV3PreTrainedModel",
+    "DeepseekV3Model",
+    "DeepseekV3ForCausalLM",
+    "DeepseekV3ForSequenceClassification",
+]

filename: src/transformers/models/deepseek_v3/modular_deepseek_v3.py
patch: @@ -9,6 +9,7 @@
 from ...activations import ACT2FN
 from ...cache_utils import Cache
 from ...modeling_flash_attention_utils import FlashAttentionKwargs
+from ...modeling_layers import GenericForSequenceClassification
 from ...modeling_utils import ALL_ATTENTION_FUNCTIONS
 from ...processing_utils import Unpack
 from ...utils import logging
@@ -356,8 +357,13 @@ class DeepseekV3ForCausalLM(LlamaForCausalLM):
     pass
 
 
+class DeepseekV3ForSequenceClassification(GenericForSequenceClassification, DeepseekV3PreTrainedModel):
+    pass
+
+
 __all__ = [
     "DeepseekV3PreTrainedModel",
     "DeepseekV3Model",
     "DeepseekV3ForCausalLM",
+    "DeepseekV3ForSequenceClassification",
 ]

filename: tests/models/deepseek_v3/test_modeling_deepseek_v3.py
patch: @@ -42,6 +42,7 @@
 
     from transformers import (
         DeepseekV3ForCausalLM,
+        DeepseekV3ForSequenceClassification,
         DeepseekV3Model,
     )
     from transformers.models.deepseek_v3.modeling_deepseek_v3 import (
@@ -215,6 +216,7 @@ class DeepseekV3ModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTeste
         (
             DeepseekV3Model,
             DeepseekV3ForCausalLM,
+            DeepseekV3ForSequenceClassification,
         )
         if is_torch_available()
         else ()
@@ -223,7 +225,9 @@ class DeepseekV3ModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTeste
     pipeline_model_mapping = (
         {
             "feature-extraction": DeepseekV3Model,
+            "text-classification": DeepseekV3ForSequenceClassification,
             "text-generation": DeepseekV3ForCausalLM,
+            "zero-shot": DeepseekV3ForSequenceClassification,
         }
         if is_torch_available()
         else {}
@@ -503,6 +507,18 @@ def test_flex_attention_with_grads(self):
             # If this does not raise an error, the test passes (see https://github.com/huggingface/transformers/pull/35605)
             _ = model(**dummy_inputs)
 
+    def test_deepseek_v3_sequence_classification_model(self):
+        config, input_dict = self.model_tester.prepare_config_and_inputs_for_common()
+        config.num_labels = 3
+        input_ids = input_dict["input_ids"]
+        attention_mask = input_ids.ne(1).to(torch_device)
+        sequence_labels = ids_tensor([self.model_tester.batch_size], self.model_tester.num_labels)
+        model = DeepseekV3ForSequenceClassification(config)
+        model.to(torch_device)
+        model.eval()
+        result = model(input_ids, attention_mask=attention_mask, labels=sequence_labels)
+        self.assertEqual(result.logits.shape, (self.model_tester.batch_size, self.model_tester.num_labels))
+
 
 @require_torch_accelerator
 class DeepseekV3IntegrationTest(unittest.TestCase):