Repository: vllm-project/vllm

Issue:
GPTBigCodeForCausalLM LoRA support
Would love to see LoRA support for GPTBigCodeForCausalLM. We have fine tuned adapters per-language.


Discussion:
`GPTBigCodeForCausalLM` is already supported, and but it needs to support the weight loading logic so it knows how to load the weights. Do you have a public lora model that contributor can test with? 

For contributors, the model code is here [https://github.com/vllm-project/vllm/blob/57f044945f25d90d1b434014b2719ba6b06fdc44/vllm/model_executor/models/gpt_bigcode.py](https://github.com/vllm-project/vllm/blob/57f044945f25d90d1b434014b2719ba6b06fdc44/vllm/model_executor/models/gpt_bigcode.py) and the lora loading logic in llama should be a good template [https://github.com/vllm-project/vllm/blob/57f044945f25d90d1b434014b2719ba6b06fdc44/vllm/model_executor/models/llama.py#L236](https://github.com/vllm-project/vllm/blob/57f044945f25d90d1b434014b2719ba6b06fdc44/vllm/model_executor/models/llama.py#L236) 
Sure thing, I just uploaded one:
https://huggingface.co/tybritten/lora-for-starcoder

btw what language is this adapter for? I'm trying to test it.

Resolving patch:
diff --git a/vllm/lora/models.py b/vllm/lora/models.py
--- a/vllm/lora/models.py
+++ b/vllm/lora/models.py
@@ -312,7 +312,9 @@ def from_local_checkpoint(
             if part_name not in expected_lora_modules:
                 unexpected_modules.append(module)
         # loaded lora's target modules must be a subset of expected_lora_modules
+
         if unexpected_modules:
+            print(unexpected_modules, "modules")
             raise ValueError(
                 f"While loading {lora_dir}, expected"
                 f" target modules in {expected_lora_modules}"
diff --git a/vllm/model_executor/models/gpt_bigcode.py b/vllm/model_executor/models/gpt_bigcode.py
--- a/vllm/model_executor/models/gpt_bigcode.py
+++ b/vllm/model_executor/models/gpt_bigcode.py
@@ -25,7 +25,7 @@
 from transformers import GPTBigCodeConfig
 
 from vllm.attention import Attention, AttentionMetadata
-from vllm.config import CacheConfig
+from vllm.config import CacheConfig, LoRAConfig
 from vllm.distributed import get_tensor_model_parallel_world_size
 from vllm.model_executor.layers.activation import get_act_fn
 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -190,14 +190,19 @@ def __init__(
         config: GPTBigCodeConfig,
         cache_config: Optional[CacheConfig] = None,
         quant_config: Optional[QuantizationConfig] = None,
+        lora_config: Optional[LoRAConfig] = None,
     ):
         super().__init__()
         self.config = config
         assert not config.add_cross_attention
 
         self.embed_dim = config.hidden_size
-
-        self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
+        lora_vocab = (lora_config.lora_extra_vocab_size *
+                      (lora_config.max_loras or 1)) if lora_config else 0
+        self.vocab_size = config.vocab_size + lora_vocab
+        self.wte = VocabParallelEmbedding(self.vocab_size,
+                                          self.embed_dim,
+                                          org_num_embeddings=config.vocab_size)
         self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
         self.h = nn.ModuleList([
             GPTBigCodeBlock(config, cache_config, quant_config)
@@ -225,19 +230,35 @@ def forward(
 
 
 class GPTBigCodeForCausalLM(nn.Module):
+    packed_modules_mapping = {"c_attn": ["c_attn"]}
+
+    supported_lora_modules = ["c_fc", "c_proj", "wte", "lm_head", "c_attn"]
+
+    embedding_modules = {
+        "wte": "input_embeddings",
+        "lm_head": "output_embeddings",
+    }
+
+    embedding_padding_modules = []
 
     def __init__(
         self,
         config: GPTBigCodeConfig,
         cache_config: Optional[CacheConfig] = None,
         quant_config: Optional[QuantizationConfig] = None,
+        lora_config: Optional[LoRAConfig] = None,
     ):
         super().__init__()
         self.config = config
         self.quant_config = quant_config
-        self.transformer = GPTBigCodeModel(config, cache_config, quant_config)
+        self.transformer = GPTBigCodeModel(config, cache_config, quant_config,
+                                           lora_config)
         self.lm_head_weight = self.transformer.wte.weight
-        self.logits_processor = LogitsProcessor(config.vocab_size)
+        self.unpadded_vocab_size = config.vocab_size
+        if lora_config:
+            self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
+        self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
+                                                config.vocab_size)
         self.sampler = Sampler()
 
     def forward(


Test patch:
diff --git a/tests/lora/test_punica.py b/tests/lora/test_punica.py
--- a/tests/lora/test_punica.py
+++ b/tests/lora/test_punica.py
@@ -58,6 +58,7 @@ def _lora_ref_impl(
     2560,
     2752,
     3072,
+    3328,
     3456,
     3584,
     4096,
@@ -66,6 +67,7 @@ def _lora_ref_impl(
     5504,
     5632,
     6144,
+    6400,
     6848,
     6912,
     7168,