init

.gitignore

@@ -0,0 +1,6 @@
+example*
+*bak
+flagged
+*.sh
+__pycache__/
+*.pth

README.md

@@ -9,4 +9,7 @@ app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+### LLaMA-Adapter
+The official demo for **LLaMA-Adapter: Efficient Fine-tuning of Language Models with Zero-init Attention**.
+Please refer to our [arXiv paper](https://arxiv.org/abs/2303.16199) and [github](https://github.com/ZrrSkywalker/LLaMA-Adapter) for more details.
+

app.py

@@ -0,0 +1,274 @@
+import json
+import os
+import glob
+import sys
+import time
+from pathlib import Path
+from typing import Tuple
+
+from PIL import Image
+import gradio as gr
+import torch
+from fairscale.nn.model_parallel.initialize import initialize_model_parallel
+
+from llama import LLaMA, ModelArgs, Tokenizer, Transformer, VisionModel
+
+PROMPT_DICT = {
+    "prompt_input": (
+        "Below is an instruction that describes a task, paired with an input that provides further context. "
+        "Write a response that appropriately completes the request.\n\n"
+        "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:"
+    ),
+    "prompt_no_input": (
+        "Below is an instruction that describes a task. "
+        "Write a response that appropriately completes the request.\n\n"
+        "### Instruction:\n{instruction}\n\n### Response:"
+    ),
+}
+
+
+def setup_model_parallel() -> Tuple[int, int]:
+    os.environ['RANK'] = '0'
+    os.environ['WORLD_SIZE'] = '1'
+    os.environ['MP'] = '1'
+    os.environ['MASTER_ADDR'] = '127.0.0.1'
+    os.environ['MASTER_PORT'] = '2223'
+    local_rank = int(os.environ.get("LOCAL_RANK", -1))
+    world_size = int(os.environ.get("WORLD_SIZE", -1))
+
+    torch.distributed.init_process_group("nccl")
+    initialize_model_parallel(world_size)
+    torch.cuda.set_device(local_rank)
+
+    # seed must be the same in all processes
+    torch.manual_seed(1)
+    return local_rank, world_size
+
+
+def load(
+    ckpt_dir: str,
+    tokenizer_path: str,
+    instruct_adapter_path: str,
+    caption_adapter_path: str,
+    local_rank: int,
+    world_size: int,
+    max_seq_len: int,
+    max_batch_size: int,
+) -> LLaMA:
+    start_time = time.time()
+    checkpoints = sorted(Path(ckpt_dir).glob("*.pth"))
+    assert world_size == len(
+        checkpoints
+    ), f"Loading a checkpoint for MP={len(checkpoints)} but world size is {world_size}"
+    ckpt_path = checkpoints[local_rank]
+    print("Loading")
+    checkpoint = torch.load(ckpt_path, map_location="cuda")
+    instruct_adapter_checkpoint = torch.load(
+        instruct_adapter_path, map_location="cuda")
+    caption_adapter_checkpoint = torch.load(
+        caption_adapter_path, map_location="cuda")
+    with open(Path(ckpt_dir) / "params.json", "r") as f:
+        params = json.loads(f.read())
+
+    model_args: ModelArgs = ModelArgs(
+        max_seq_len=max_seq_len, max_batch_size=max_batch_size, **params
+    )
+    model_args.adapter_layer = int(
+        instruct_adapter_checkpoint['adapter_query.weight'].shape[0] / model_args.adapter_len)
+    model_args.cap_adapter_layer = int(
+        caption_adapter_checkpoint['cap_adapter_query.weight'].shape[0] / model_args.cap_adapter_len)
+
+    tokenizer = Tokenizer(model_path=tokenizer_path)
+    model_args.vocab_size = tokenizer.n_words
+    torch.set_default_tensor_type(torch.cuda.HalfTensor)
+    model = Transformer(model_args)
+    vision_model = VisionModel(model_args)
+
+    torch.set_default_tensor_type(torch.FloatTensor)
+    model.load_state_dict(checkpoint, strict=False)
+    model.load_state_dict(instruct_adapter_checkpoint, strict=False)
+    model.load_state_dict(caption_adapter_checkpoint, strict=False)
+    vision_model.load_state_dict(caption_adapter_checkpoint, strict=False)
+
+    generator = LLaMA(model, tokenizer, vision_model)
+    print(f"Loaded in {time.time() - start_time:.2f} seconds")
+    return generator
+
+
+def instruct_generate(
+    instruct: str,
+    input: str = 'none',
+    max_gen_len=512,
+    temperature: float = 0.1,
+    top_p: float = 0.75,
+):
+    if input == 'none':
+        prompt = PROMPT_DICT['prompt_no_input'].format_map(
+            {'instruction': instruct, 'input': ''})
+    else:
+        prompt = PROMPT_DICT['prompt_input'].format_map(
+            {'instruction': instruct, 'input': input})
+
+    results = generator.generate(
+        [prompt], max_gen_len=max_gen_len, temperature=temperature, top_p=top_p
+    )
+    result = results[0].strip()
+    print(result)
+    return result
+
+
+def caption_generate(
+    img: str,
+    max_gen_len=512,
+    temperature: float = 0.1,
+    top_p: float = 0.75,
+):
+    imgs = [Image.open(img).convert('RGB')]
+    prompts = ["Generate caption of this image :",] * len(imgs)
+
+    results = generator.generate(
+        prompts, imgs=imgs, max_gen_len=max_gen_len, temperature=temperature, top_p=top_p
+    )
+    result = results[0].strip()
+    print(result)
+    return result
+
+
+def download_llama_7b(ckpt_dir, tokenizer_path):
+    print("LLaMA-7B downloading")
+    os.makedirs(ckpt_dir, exist_ok=True)
+    ckpt_path = os.path.join(ckpt_dir, "consolidated.00.pth")
+    param_path = os.path.join(ckpt_dir, "params.json")
+    if not os.path.exists(ckpt_path):
+        os.system(
+            f"wget -O {ckpt_path} https://huggingface.co/nyanko7/LLaMA-7B/resolve/main/consolidated.00.pth")
+    if not os.path.exists(param_path):
+        os.system(
+            f"wget -O {param_path} https://huggingface.co/nyanko7/LLaMA-7B/raw/main/params.json")
+    if not os.path.exists(tokenizer_path):
+        os.system(
+            f"wget -O {tokenizer_path} https://huggingface.co/nyanko7/LLaMA-7B/resolve/main/tokenizer.model")
+    print("LLaMA-7B downloaded")
+
+def download_llama_adapter(instruct_adapter_path, caption_adapter_path):
+    if not os.path.exists(instruct_adapter_path):
+        os.system(f"wget -O {instruct_adapter_path} https://github.com/ZrrSkywalker/LLaMA-Adapter/releases/download/v.1.0.0/llama_adapter_len10_layer30_release.pth")
+
+    if not os.path.exists(caption_adapter_path):
+        os.system(f"wget -O {caption_adapter_path} https://github.com/ZrrSkywalker/LLaMA-Adapter/releases/download/v.1.0.0/llama_adapter_len10_layer30_caption_vit_l.pth")
+
+
+# ckpt_dir = "/data1/llma/7B"
+# tokenizer_path = "/data1/llma/tokenizer.model"
+ckpt_dir = "llama_7B/"
+tokenizer_path = "tokenizer.model"
+instruct_adapter_path = "llama_adapter_len10_layer30_release.pth"
+caption_adapter_path = "llama_adapter_len10_layer30_caption_vit_l.pth"
+max_seq_len = 512
+max_batch_size = 1
+
+# download models
+download_llama_7b(ckpt_dir, tokenizer_path)
+download_llama_adapter(instruct_adapter_path, caption_adapter_path)
+
+local_rank, world_size = setup_model_parallel()
+if local_rank > 0:
+    sys.stdout = open(os.devnull, "w")
+
+generator = load(
+    ckpt_dir, tokenizer_path, instruct_adapter_path, caption_adapter_path, local_rank, world_size, max_seq_len, max_batch_size
+)
+
+
+def create_instruct_demo():
+    with gr.Blocks() as instruct_demo:
+        with gr.Row():
+            with gr.Column():
+                instruction = gr.Textbox(lines=2, label="Instruction")
+                input = gr.Textbox(
+                    lines=2, label="Context input", placeholder='none')
+                max_len = gr.Slider(minimum=1, maximum=512,
+                                    value=128, label="Max length")
+                with gr.Accordion(label='Advanced options', open=False):
+                    temp = gr.Slider(minimum=0, maximum=1,
+                                     value=0.1, label="Temperature")
+                    top_p = gr.Slider(minimum=0, maximum=1,
+                                      value=0.75, label="Top p")
+
+                run_botton = gr.Button("Run")
+
+            with gr.Column():
+                outputs = gr.Textbox(lines=10, label="Output")
+
+        inputs = [instruction, input, max_len, temp, top_p]
+
+        examples = [
+            "Tell me about alpacas.",
+            "Write a Python program that prints the first 10 Fibonacci numbers.",
+            "Write a conversation between the sun and pluto.",
+            "Write a theory to explain why cat never existed",
+        ]
+        examples = [
+            [x, "none", 128, 0.1, 0.75]
+            for x in examples]
+
+        gr.Examples(
+            examples=examples,
+            inputs=inputs,
+            outputs=outputs,
+            fn=instruct_generate,
+            cache_examples=os.getenv('SYSTEM') == 'spaces'
+        )
+        run_botton.click(fn=instruct_generate, inputs=inputs, outputs=outputs)
+    return instruct_demo
+
+
+def create_caption_demo():
+    with gr.Blocks() as instruct_demo:
+        with gr.Row():
+            with gr.Column():
+                img = gr.Image(label='Input', type='filepath')
+                max_len = gr.Slider(minimum=1, maximum=512,
+                                    value=64, label="Max length")
+                with gr.Accordion(label='Advanced options', open=False):
+                    temp = gr.Slider(minimum=0, maximum=1,
+                                     value=0.1, label="Temperature")
+                    top_p = gr.Slider(minimum=0, maximum=1,
+                                      value=0.75, label="Top p")
+
+                run_botton = gr.Button("Run")
+
+            with gr.Column():
+                outputs = gr.Textbox(lines=10, label="Output")
+
+        inputs = [img, max_len, temp, top_p]
+
+        examples = glob.glob("caption_demo/*.jpg")
+        examples = [
+            [x, 64, 0.1, 0.75]
+            for x in examples]
+
+        gr.Examples(
+            examples=examples,
+            inputs=inputs,
+            outputs=outputs,
+            fn=caption_generate,
+            cache_examples=os.getenv('SYSTEM') == 'spaces'
+        )
+        run_botton.click(fn=caption_generate, inputs=inputs, outputs=outputs)
+    return instruct_demo
+
+description = """
+# LLaMA-Adapter
+The official demo for **LLaMA-Adapter: Efficient Fine-tuning of Language Models with Zero-init Attention**.
+Please refer to our [arXiv paper](https://arxiv.org/abs/2303.16199) and [github](https://github.com/ZrrSkywalker/LLaMA-Adapter) for more details.
+"""
+
+with gr.Blocks(css='style.css') as demo:
+    gr.Markdown(description)
+    with gr.TabItem("Instruction-Following"):
+        create_instruct_demo()
+    with gr.TabItem("Image Captioning"):
+        create_caption_demo()
+
+demo.queue(api_open=True, concurrency_count=1).launch()

llama/__init__.py

@@ -0,0 +1,6 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# This software may be used and distributed according to the terms of the GNU General Public License version 3.
+
+from .generation import LLaMA
+from .model import ModelArgs, Transformer, VisionModel
+from .tokenizer import Tokenizer

llama/generation.py

@@ -0,0 +1,85 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# This software may be used and distributed according to the terms of the GNU General Public License version 3.
+
+from typing import List
+
+import torch
+
+from llama.tokenizer import Tokenizer
+from llama.model import Transformer
+
+
+class LLaMA:
+    def __init__(self, model: Transformer, tokenizer: Tokenizer, vision_model = None):
+        self.model = model
+        self.tokenizer = tokenizer
+        self.vision_model = vision_model
+
+    def generate(
+        self,
+        prompts: List[str],
+        imgs = None,
+        max_gen_len: int = 512,
+        temperature: float = 0.8,
+        top_p: float = 0.95,
+    ) -> List[str]:
+        bsz = len(prompts)
+        params = self.model.params
+        assert bsz <= params.max_batch_size, (bsz, params.max_batch_size)
+
+        mode = 'instruct'
+        vision_tokens = None
+        if imgs is not None and self.vision_model is not None:
+            vision_tokens = self.vision_model(imgs)
+            mode = 'caption'
+
+        prompt_tokens = [self.tokenizer.encode(x, bos=True, eos=False) for x in prompts]
+
+        min_prompt_size = min([len(t) for t in prompt_tokens])
+        max_prompt_size = max([len(t) for t in prompt_tokens])
+
+        total_len = min(params.max_seq_len, max_gen_len + max_prompt_size)
+
+        tokens = torch.full((bsz, total_len), self.tokenizer.pad_id).cuda().long()
+        for k, t in enumerate(prompt_tokens):
+            tokens[k, : len(t)] = torch.tensor(t).long()
+        input_text_mask = tokens != self.tokenizer.pad_id
+        start_pos = min_prompt_size
+        prev_pos = 0
+        for cur_pos in range(start_pos, total_len):
+            logits = self.model.forward(tokens[:, prev_pos:cur_pos], prev_pos, vision_tokens, mode)
+            if temperature > 0:
+                probs = torch.softmax(logits / temperature, dim=-1)
+                next_token = sample_top_p(probs, top_p)
+            else:
+                next_token = torch.argmax(logits, dim=-1)
+            next_token = next_token.reshape(-1)
+            # only replace token if prompt has already been generated
+            next_token = torch.where(
+                input_text_mask[:, cur_pos], tokens[:, cur_pos], next_token
+            )
+            tokens[:, cur_pos] = next_token
+            prev_pos = cur_pos
+
+        decoded = []
+        for i, t in enumerate(tokens.tolist()):
+            # cut to max gen len
+            t = t[len(prompt_tokens[i]) : len(prompt_tokens[i]) + max_gen_len]
+            # cut to eos tok if any
+            try:
+                t = t[: t.index(self.tokenizer.eos_id)]
+            except ValueError:
+                pass
+            decoded.append(self.tokenizer.decode(t))
+        return decoded
+
+
+def sample_top_p(probs, p):
+    probs_sort, probs_idx = torch.sort(probs, dim=-1, descending=True)
+    probs_sum = torch.cumsum(probs_sort, dim=-1)
+    mask = probs_sum - probs_sort > p
+    probs_sort[mask] = 0.0
+    probs_sort.div_(probs_sort.sum(dim=-1, keepdim=True))
+    next_token = torch.multinomial(probs_sort, num_samples=1)
+    next_token = torch.gather(probs_idx, -1, next_token)
+    return next_token

llama/model.py

@@ -0,0 +1,423 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# This software may be used and distributed according to the terms of the GNU General Public License version 3.
+
+from typing import Optional, Tuple
+from dataclasses import dataclass
+import math
+
+import torch
+from torch import nn
+import torch.nn.functional as F
+
+import clip
+from timm.models.vision_transformer import Block
+
+import fairscale.nn.model_parallel.initialize as fs_init
+from fairscale.nn.model_parallel.layers import (
+    ParallelEmbedding,
+    RowParallelLinear,
+    ColumnParallelLinear,
+)
+
+@dataclass
+class ModelArgs:
+    dim: int = 512
+    n_layers: int = 8
+    n_heads: int = 8
+    vocab_size: int = -1  # defined later by tokenizer
+    multiple_of: int = 256  # make SwiGLU hidden layer size multiple of large power of 2
+    norm_eps: float = 1e-5
+
+    max_batch_size: int = 32
+    max_seq_len: int = 2048
+
+    adapter_len: int = 10
+    adapter_layer: int = 30
+
+    cap_adapter_len: int = 10
+    cap_adapter_layer: int = 30
+    cap_vision_model: str = "ViT-L/14"
+    cap_vision_dim: int = 512
+    cap_vision_block: int = 2
+
+
+class RMSNorm(torch.nn.Module):
+    def __init__(self, dim: int, eps: float = 1e-6):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(dim))
+
+    def _norm(self, x):
+        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+
+    def forward(self, x):
+        output = self._norm(x.float()).type_as(x)
+        return output * self.weight
+
+
+def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0):
+    freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim))
+    t = torch.arange(end, device=freqs.device)  # type: ignore
+    freqs = torch.outer(t, freqs).float()  # type: ignore
+    freqs_cis = torch.polar(torch.ones_like(freqs), freqs)  # complex64
+    return freqs_cis
+
+
+def reshape_for_broadcast(freqs_cis: torch.Tensor, x: torch.Tensor):
+    ndim = x.ndim
+    assert 0 <= 1 < ndim
+    assert freqs_cis.shape == (x.shape[1], x.shape[-1])
+    shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)]
+    return freqs_cis.view(*shape)
+
+
+def apply_rotary_emb(
+    xq: torch.Tensor,
+    xk: torch.Tensor,
+    freqs_cis: torch.Tensor,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2))
+    xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2))
+    freqs_cis = reshape_for_broadcast(freqs_cis, xq_)
+    xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3)
+    xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3)
+    return xq_out.type_as(xq), xk_out.type_as(xk)
+
+
+class Attention(nn.Module):
+    def __init__(self, args: ModelArgs):
+        super().__init__()
+
+        self.n_local_heads = args.n_heads // fs_init.get_model_parallel_world_size()
+        self.head_dim = args.dim // args.n_heads
+
+        self.wq = ColumnParallelLinear(
+            args.dim,
+            args.n_heads * self.head_dim,
+            bias=False,
+            gather_output=False,
+            init_method=lambda x: x,
+        )
+        self.wk = ColumnParallelLinear(
+            args.dim,
+            args.n_heads * self.head_dim,
+            bias=False,
+            gather_output=False,
+            init_method=lambda x: x,
+        )
+        self.wv = ColumnParallelLinear(
+            args.dim,
+            args.n_heads * self.head_dim,
+            bias=False,
+            gather_output=False,
+            init_method=lambda x: x,
+        )
+        self.wo = RowParallelLinear(
+            args.n_heads * self.head_dim,
+            args.dim,
+            bias=False,
+            input_is_parallel=True,
+            init_method=lambda x: x,
+        )
+
+        self.cache_k = torch.zeros(
+            (args.max_batch_size, args.max_seq_len, self.n_local_heads, self.head_dim)
+        ).cuda()
+        self.cache_v = torch.zeros(
+            (args.max_batch_size, args.max_seq_len, self.n_local_heads, self.head_dim)
+        ).cuda()
+        self.gate = torch.nn.Parameter(torch.zeros(1))
+
+        self.cap_gate = torch.nn.Parameter(torch.zeros(1, self.n_local_heads, 1, 1))
+
+
+    def forward(self, x: torch.Tensor, start_pos: int, freqs_cis: torch.Tensor, mask: Optional[torch.Tensor], adapter=None, mode='instruct'):
+        if mode == 'instruct':
+            return self.forward_instruct(x, start_pos, freqs_cis, mask, adapter)
+        elif mode == 'caption':
+            return self.forward_caption(x, start_pos, freqs_cis, mask, adapter)
+
+    
+    def forward_instruct(self, x: torch.Tensor, start_pos: int, freqs_cis: torch.Tensor, mask: Optional[torch.Tensor], adapter=None):
+        bsz, seqlen, _ = x.shape
+        xq, xk, xv = self.wq(x), self.wk(x), self.wv(x)
+
+        xq = xq.view(bsz, seqlen, self.n_local_heads, self.head_dim)
+        xk = xk.view(bsz, seqlen, self.n_local_heads, self.head_dim)
+        xv = xv.view(bsz, seqlen, self.n_local_heads, self.head_dim)
+
+        xq, xk = apply_rotary_emb(xq, xk, freqs_cis=freqs_cis)
+
+        self.cache_k = self.cache_k.to(xq)
+        self.cache_v = self.cache_v.to(xq)
+
+        self.cache_k[:bsz, start_pos : start_pos + seqlen] = xk
+        self.cache_v[:bsz, start_pos : start_pos + seqlen] = xv
+
+        keys = self.cache_k[:bsz, : start_pos + seqlen]
+        values = self.cache_v[:bsz, : start_pos + seqlen]
+
+        if adapter is not None:
+           adapter_len = adapter.shape[1]
+           adapter_k = self.wk(adapter).view(1, adapter_len, self.n_local_heads, self.head_dim).repeat(bsz, 1, 1, 1)
+           adapter_v = self.wv(adapter).view(1, adapter_len, self.n_local_heads, self.head_dim).repeat(bsz, 1, 1, 1)
+           adapter_k = adapter_k.transpose(1, 2)
+           adapter_v = adapter_v.transpose(1, 2)
+        xq = xq.transpose(1, 2)
+        keys = keys.transpose(1, 2)
+        values = values.transpose(1, 2)
+        scores = torch.matmul(xq, keys.transpose(2, 3)) / math.sqrt(self.head_dim)
+        if mask is not None:
+            scores = scores + mask  # (bs, n_local_heads, slen, cache_len + slen)
+        scores = F.softmax(scores.float(), dim=-1).type_as(xq)
+        output = torch.matmul(scores, values)  # (bs, n_local_heads, slen, head_dim)
+        if adapter is not None:
+            adapter_scores = torch.matmul(xq, adapter_k.transpose(2, 3)) / math.sqrt(self.head_dim)
+            adapter_scores = self.gate * F.softmax(adapter_scores.float(), dim=-1).type_as(xq)
+            output = output + torch.matmul(adapter_scores, adapter_v)
+        output = output.transpose(
+            1, 2
+        ).contiguous().view(bsz, seqlen, -1)
+
+        return self.wo(output)
+    
+
+    def forward_caption(self, x: torch.Tensor, start_pos: int, freqs_cis: torch.Tensor, mask: Optional[torch.Tensor], adapter=None):
+        bsz, seqlen, _ = x.shape
+        xq, xk, xv = self.wq(x), self.wk(x), self.wv(x)
+
+        xq = xq.view(bsz, seqlen, self.n_local_heads, self.head_dim)
+        xk = xk.view(bsz, seqlen, self.n_local_heads, self.head_dim)
+        xv = xv.view(bsz, seqlen, self.n_local_heads, self.head_dim)
+
+        xq, xk = apply_rotary_emb(xq, xk, freqs_cis=freqs_cis)
+
+        self.cache_k = self.cache_k.to(xq)
+        self.cache_v = self.cache_v.to(xq)
+
+        self.cache_k[:bsz, start_pos : start_pos + seqlen] = xk
+        self.cache_v[:bsz, start_pos : start_pos + seqlen] = xv
+
+        keys = self.cache_k[:bsz, : start_pos + seqlen]
+        values = self.cache_v[:bsz, : start_pos + seqlen]
+
+        if adapter is not None:
+           adapter_len = adapter.shape[1]
+           adapter_k = self.wk(adapter).view(bsz, adapter_len, self.n_local_heads, self.head_dim)
+           adapter_v = self.wv(adapter).view(bsz, adapter_len, self.n_local_heads, self.head_dim)
+           adapter_k = adapter_k.transpose(1, 2)
+           adapter_v = adapter_v.transpose(1, 2)
+        xq = xq.transpose(1, 2)
+        keys = keys.transpose(1, 2)
+        values = values.transpose(1, 2)
+        scores = torch.matmul(xq, keys.transpose(2, 3)) / math.sqrt(self.head_dim)
+        if mask is not None:
+            scores = scores + mask  # (bs, n_local_heads, slen, cache_len + slen)
+        scores = F.softmax(scores.float(), dim=-1).type_as(xq)
+        output = torch.matmul(scores, values)  # (bs, n_local_heads, slen, head_dim)
+        if adapter is not None:
+            adapter_scores = torch.matmul(xq, adapter_k.transpose(2, 3)) / math.sqrt(self.head_dim)
+            adapter_scores = self.cap_gate.tanh() * F.softmax(adapter_scores.float(), dim=-1).type_as(xq)
+
+            output = output + torch.matmul(adapter_scores, adapter_v)
+        output = output.transpose(
+            1, 2
+        ).contiguous().view(bsz, seqlen, -1)
+
+        return self.wo(output)
+
+
+
+class FeedForward(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        hidden_dim: int,
+        multiple_of: int,
+    ):
+        super().__init__()
+        hidden_dim = int(2 * hidden_dim / 3)
+        hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)
+
+        self.w1 = ColumnParallelLinear(
+            dim, hidden_dim, bias=False, gather_output=False, init_method=lambda x: x
+        )
+        self.w2 = RowParallelLinear(
+            hidden_dim, dim, bias=False, input_is_parallel=True, init_method=lambda x: x
+        )
+        self.w3 = ColumnParallelLinear(
+            dim, hidden_dim, bias=False, gather_output=False, init_method=lambda x: x
+        )
+
+    def forward(self, x):
+        return self.w2(F.silu(self.w1(x)) * self.w3(x))
+
+
+class TransformerBlock(nn.Module):
+    def __init__(self, layer_id: int, args: ModelArgs):
+        super().__init__()
+        self.n_heads = args.n_heads
+        self.dim = args.dim
+        self.head_dim = args.dim // args.n_heads
+        self.attention = Attention(args)
+        self.feed_forward = FeedForward(
+            dim=args.dim, hidden_dim=4 * args.dim, multiple_of=args.multiple_of
+        )
+        self.layer_id = layer_id
+        self.attention_norm = RMSNorm(args.dim, eps=args.norm_eps)
+        self.ffn_norm = RMSNorm(args.dim, eps=args.norm_eps)
+
+    def forward(self, x: torch.Tensor, start_pos: int, freqs_cis: torch.Tensor, mask: Optional[torch.Tensor], adapter=None, mode='instruct'):
+        h = x + self.attention.forward(self.attention_norm(x), start_pos, freqs_cis, mask, adapter, mode=mode)
+        out = h + self.feed_forward.forward(self.ffn_norm(h))
+        return out
+
+
+class Transformer(nn.Module):
+    def __init__(self, params: ModelArgs):
+        super().__init__()
+        self.params = params
+        self.vocab_size = params.vocab_size
+        self.n_layers = params.n_layers
+
+        self.tok_embeddings = ParallelEmbedding(
+            params.vocab_size, params.dim, init_method=lambda x: x
+        )
+
+        self.layers = torch.nn.ModuleList()
+        for layer_id in range(params.n_layers):
+            self.layers.append(TransformerBlock(layer_id, params))
+
+        self.norm = RMSNorm(params.dim, eps=params.norm_eps)
+        self.output = ColumnParallelLinear(
+            params.dim, params.vocab_size, bias=False, init_method=lambda x: x
+        )
+
+        self.freqs_cis = precompute_freqs_cis(
+            self.params.dim // self.params.n_heads, self.params.max_seq_len * 2
+        )
+
+        # Note: this is only a preview of multimodal LLaMA-Adapter
+        # and requires more efforts to decouple LLaMA-Adapter from LLaMA.
+        # instruct model
+        self.adapter_query = nn.Embedding(params.adapter_len * params.adapter_layer, params.dim)
+        self.adapter_len = params.adapter_len
+        self.adapter_layer = params.adapter_layer
+
+        # caption model
+        self.cap_adapter_query = nn.Embedding(params.cap_adapter_len * params.cap_adapter_layer, params.dim)
+        self.cap_adapter_len = params.cap_adapter_len
+        self.cap_adapter_layer = params.cap_adapter_layer
+
+    @torch.inference_mode()
+    def forward(self, tokens: torch.Tensor, start_pos: int, visual_tokens: torch.Tensor = None, mode: str = 'instruct'):
+        if mode == 'instruct':
+            return self.forward_instruct(tokens, start_pos, mode)
+        elif mode == 'caption':
+            return self.forward_caption(tokens, start_pos, visual_tokens, mode)
+    
+    def forward_instruct(self, tokens: torch.Tensor, start_pos: int, mode=None):
+        _bsz, seqlen = tokens.shape
+        h = self.tok_embeddings(tokens)
+        self.freqs_cis = self.freqs_cis.to(h.device)
+        freqs_cis = self.freqs_cis[start_pos : start_pos + seqlen]
+        adapter = self.adapter_query.weight.reshape(self.params.adapter_layer, self.params.adapter_len, self.params.dim).unsqueeze(1)
+        mask = None
+        if seqlen > 1:
+            mask = torch.full((1, 1, seqlen, seqlen), float("-inf"), device=tokens.device)
+            mask = torch.triu(mask, diagonal=start_pos + 1).type_as(h)
+
+        for layer in self.layers[: -1 * self.params.adapter_layer]:
+            h = layer(h, start_pos, freqs_cis, mask)
+        layer_index = 0
+        for layer in self.layers[-1 * self.params.adapter_layer:]:
+            h = layer(h, start_pos, freqs_cis, mask, adapter[layer_index], mode=mode)
+            layer_index = layer_index + 1
+        h = self.norm(h)
+        output = self.output(h[:, -1, :])  # only compute last logits
+        return output.float()
+
+    def forward_caption(self, tokens: torch.Tensor, start_pos: int, visual_tokens: torch.Tensor = None, mode=None):
+        _bsz, seqlen = tokens.shape
+        h = self.tok_embeddings(tokens)
+        self.freqs_cis = self.freqs_cis.to(h.device)
+        freqs_cis = self.freqs_cis[start_pos : start_pos + seqlen]
+        adapter = self.cap_adapter_query.weight.reshape(self.params.cap_adapter_layer, self.params.cap_adapter_len, self.params.dim).unsqueeze(1)
+        mask = None
+        if seqlen > 1:
+            mask = torch.full((1, 1, seqlen, seqlen), float("-inf"), device=tokens.device)
+            mask = torch.triu(mask, diagonal=start_pos + 1).type_as(h)
+
+        for layer in self.layers[: -1 * self.params.cap_adapter_layer]:
+            h = layer(h, start_pos, freqs_cis, mask)
+        layer_index = 0
+        for layer in self.layers[-1 * self.params.cap_adapter_layer:]:
+            adapter_per_layer = adapter[layer_index]
+            if visual_tokens is not None:
+                adapter_per_layer = adapter_per_layer + visual_tokens
+            h = layer(h, start_pos, freqs_cis, mask, adapter_per_layer, mode=mode)
+            layer_index = layer_index + 1
+        h = self.norm(h)
+        output = self.output(h[:, -1, :])  # only compute last logits
+        return output.float()
+
+
+
+class VisionModel(nn.Module):
+    def __init__(self, params: ModelArgs):
+        super().__init__()
+
+        self.params = params
+
+        self.clip, self.clip_transform = clip.load(params.cap_vision_model)
+        self.clip.float()
+        for param in self.clip.parameters():
+            param.requires_grad = False
+
+        self.clip_proj = nn.Linear(self.clip.visual.output_dim, params.cap_vision_dim)
+        self.clip_proj_norm = nn.LayerNorm(params.cap_vision_dim)
+
+        self.visual_query = nn.Embedding(params.cap_adapter_len, params.cap_vision_dim)
+
+        self.visual_blocks = nn.ModuleList([
+            Block(params.cap_vision_dim, 16, 4, qkv_bias=True, qk_scale=None, norm_layer=nn.LayerNorm)
+        for i in range(params.cap_vision_block)])
+        
+        self.visual_proj = nn.Linear(params.cap_vision_dim, params.dim)
+        self.visual_proj_norm = nn.LayerNorm(params.dim)
+        
+    def clip_encode_image(self, x):
+        x = self.clip.visual.conv1(x)  # shape = [*, width, grid, grid]
+        x = x.reshape(x.shape[0], x.shape[1], -1)  # shape = [*, width, grid ** 2]
+        x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
+        x = torch.cat([self.clip.visual.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1)  # shape = [*, grid ** 2 + 1, width]
+        x = x + self.clip.visual.positional_embedding.to(x.dtype)
+        x = self.clip.visual.ln_pre(x)
+
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.clip.visual.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+
+        x = self.clip.visual.ln_post(x[:, :, :])
+
+        if self.clip.visual.proj is not None:
+            x = x @ self.clip.visual.proj
+
+        return x
+    
+    def forward(self, imgs):
+        x = [self.clip_transform(img) for img in imgs]
+        x = torch.stack(x, dim=0).to(self.visual_query.weight.device)
+        _bsz = x.shape[0]
+
+        visual_feats = self.clip_encode_image(x).half()
+        visual_feats = self.clip_proj_norm(self.clip_proj(visual_feats))
+        visual_query = self.visual_query.weight.unsqueeze(0).repeat(_bsz, 1, 1)
+        visual_query = torch.cat([visual_query, visual_feats], dim=1)
+        for block in self.visual_blocks:
+            visual_query = block(visual_query)
+        visual_query = visual_query[:, :self.params.cap_adapter_len, :]
+        visual_query = self.visual_proj(visual_query)
+        visual_query = self.visual_proj_norm(visual_query)
+        
+        return visual_query

llama/tokenizer.py

@@ -0,0 +1,40 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# This software may be used and distributed according to the terms of the GNU General Public License version 3.
+
+from sentencepiece import SentencePieceProcessor
+from logging import getLogger
+from typing import List
+import os
+
+
+logger = getLogger()
+
+
+class Tokenizer:
+    def __init__(self, model_path: str):
+        # reload tokenizer
+        assert os.path.isfile(model_path), model_path
+        self.sp_model = SentencePieceProcessor(model_file=model_path)
+        logger.info(f"Reloaded SentencePiece model from {model_path}")
+
+        # BOS / EOS token IDs
+        self.n_words: int = self.sp_model.vocab_size()
+        self.bos_id: int = self.sp_model.bos_id()
+        self.eos_id: int = self.sp_model.eos_id()
+        self.pad_id: int = self.sp_model.pad_id()
+        logger.info(
+            f"#words: {self.n_words} - BOS ID: {self.bos_id} - EOS ID: {self.eos_id}"
+        )
+        assert self.sp_model.vocab_size() == self.sp_model.get_piece_size()
+
+    def encode(self, s: str, bos: bool, eos: bool) -> List[int]:
+        assert type(s) is str
+        t = self.sp_model.encode(s)
+        if bos:
+            t = [self.bos_id] + t
+        if eos:
+            t = t + [self.eos_id]
+        return t
+
+    def decode(self, t: List[int]) -> str:
+        return self.sp_model.decode(t)

requirements.txt

@@ -0,0 +1,6 @@
+torch
+fairscale
+sentencepiece
+Pillow
+timm==0.3.2
+git+https://github.com/openai/CLIP.git

style.css

@@ -0,0 +1,4 @@
+h1,p {
+    text-align: center;
+  }
+