Upload 26 files

README.md

@@ -0,0 +1,81 @@
+# Chat with Meta's LLaMA models at home made easy
+
+This repository is a chat example with [LLaMA](https://ai.facebook.com/blog/large-language-model-llama-meta-ai/) ([arXiv](https://arxiv.org/abs/2302.13971v1)) models running on a typical home PC. You will just need a NVIDIA videocard and some RAM to chat with model.
+
+This repo is heavily based on Meta's original repo: https://github.com/facebookresearch/llama
+
+And on Venuatu's repo: https://github.com/venuatu/llama
+
+### Examples of chats here
+
+https://github.com/facebookresearch/llama/issues/162
+
+### System requirements
+- Modern enough CPU
+- NVIDIA graphics card
+- 64 or better 128 Gb of RAM (192 or 256 would be perfect)
+
+One may run with 32 Gb of RAM, but inference will be slow (with the speed of your swap file reading)
+
+I am running this on 12700k/128 Gb RAM/NVIDIA 3070ti 8Gb/fast huge nvme and getting one token from 30B model in a few seconds.
+
+For example, 30B model uses around 70 Gb of RAM.
+
+If you do not have powerful videocard, you may use another repo for cpu-only inference: https://github.com/randaller/llama-cpu
+
+### Conda Environment Setup Example for Windows 10+
+Download and install Anaconda Python https://www.anaconda.com and run Anaconda Prompt
+```
+conda create -n llama python=3.10
+conda activate llama
+conda install pytorch torchvision torchaudio pytorch-cuda=11.7 -c pytorch -c nvidia
+```
+
+### Setup
+In a conda env with pytorch / cuda available, run
+```
+pip install -r requirements.txt
+```
+Then in this repository
+```
+pip install -e .
+```
+
+### Download tokenizer and models
+magnet:?xt=urn:btih:ZXXDAUWYLRUXXBHUYEMS6Q5CE5WA3LVA&dn=LLaMA
+
+or
+
+magnet:xt=urn:btih:b8287ebfa04f879b048d4d4404108cf3e8014352&dn=LLaMA&tr=udp%3a%2f%2ftracker.opentrackr.org%3a1337%2fannounce
+
+### Prepare model
+
+First, you need to unshard model checkpoints to a single file. Let's do this for 30B model.
+
+```
+python merge-weights.py --input_dir D:\Downloads\LLaMA --model_size 30B
+```
+
+In this example, D:\Downloads\LLaMA is a root folder of downloaded torrent with weights.
+
+This will create merged.pth file in the root folder of this repo.
+
+Place this file and corresponding (torrentroot)/30B/params.json of model into [/model] folder.
+
+So you should end up with two files in [/model] folder: merged.pth and params.json.
+
+Place (torrentroot)/tokenizer.model file to the [/tokenizer] folder of this repo. Now you are ready to go.
+
+### Run the chat
+
+```
+python example-chat.py ./model ./tokenizer/tokenizer.model
+```
+
+### Enable multi-line answers
+
+If you wish to stop generation not by "\n" sign, but by another signature, like "User:" (which is also good idea), or any other, make the following modification in the llama/generation.py:
+
+![image](https://user-images.githubusercontent.com/22396871/224122767-227deda4-a718-4774-a7f9-786c07d379cf.png)
+
+-5 means to remove last 5 chars from resulting context, which is length of your stop signature, "User:" in this example.

example-chat.py

@@ -0,0 +1,115 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# This software may be used and distributed according to the terms of the GNU General Public License version 3.
+
+from typing import Tuple
+import os
+import sys
+import torch
+import fire
+import time
+import json
+import pyarrow as pa
+
+from pathlib import Path
+
+from llama import ModelArgs, Transformer, Tokenizer, LLaMA
+
+
+def load(
+        ckpt_dir: str,
+        tokenizer_path: str,
+        max_seq_len: int,
+        max_batch_size: int,
+) -> LLaMA:
+    start_time = time.time()
+    arrow_dir = Path(ckpt_dir).expanduser() / 'arrow'
+
+    if not arrow_dir.exists():
+        print('Converting checkpoints to arrow format')
+        checkpoints = sorted(Path(ckpt_dir).expanduser().glob("*.pth"))
+        for ckpt_file in checkpoints:
+            print(ckpt_file)
+            index = ckpt_file.parts[-1].split('.')[-2]
+
+            ckpt = torch.load(ckpt_file, map_location='cuda')
+            (arrow_dir / index).mkdir(parents=True, exist_ok=True)
+            for k, v in ckpt.items():
+                tens = pa.Tensor.from_numpy(v.numpy())
+                with pa.output_stream(arrow_dir / index / k) as f:
+                    pa.ipc.write_tensor(tens, f)
+            ckpt = None
+
+    with open(Path(ckpt_dir) / "params.json", "r") as f:
+        params = json.loads(f.read())
+
+    print("Loading checkpoint")
+    segments = sorted((arrow_dir / '00').glob("*"))
+
+    checkpoint = {}
+    files = []
+    for seg in segments:
+        f = pa.memory_map(str(seg))
+        files.append(f)
+        t = pa.ipc.read_tensor(f).to_numpy()
+        t = torch.from_numpy(t)
+        checkpoint[seg.parts[-1]] = t
+
+    # torch.set_default_tensor_type(torch.cuda.HalfTensor)
+    torch.set_default_tensor_type(torch.BFloat16Tensor)
+    # torch.set_default_tensor_type(torch.FloatTensor)
+
+    model_args: ModelArgs = ModelArgs(
+        max_seq_len=max_seq_len, max_batch_size=max_batch_size, **params
+    )
+    print("Loading tokenizer")
+    tokenizer = Tokenizer(model_path=tokenizer_path)
+    model_args.vocab_size = tokenizer.n_words
+    print("Loading model")
+    model = Transformer(model_args)
+
+    checkpoints = sorted(Path(ckpt_dir).glob("*.pth"))
+    model.load_state_dict(torch.load(checkpoints[-1]), strict=False)
+
+    for f in files:
+        f.close()
+    files = None
+
+    generator = LLaMA(model, tokenizer)
+    print(f"Loaded in {time.time() - start_time:.2f} seconds")
+    return generator
+
+
+def main(
+        ckpt_dir: str,
+        tokenizer_path: str,
+        temperature: float = 0.8,
+        top_p: float = 0.95,
+        max_seq_len: int = 2048,
+        max_batch_size: int = 1,  # 16 for 13B, 4 for 30B and 65B, 2 for 1024 seq_len for 30B
+):
+    generator = load(ckpt_dir, tokenizer_path, max_seq_len, max_batch_size)
+
+    ctx = """A dialog, where User interacts with AI. AI is helpful, kind, obedient, honest, and knows its own limits.
+User: Hello, AI.
+AI: Hello! How can I assist you today?
+"""
+
+    while True:
+        prompt = input(f'User: ')
+        if ctx != "":
+            ctx = ctx + "User: " + prompt + "\n"
+        else:
+            ctx = prompt + "\n"
+
+        ctx = (ctx[-1920:]) if len(ctx) >= 2048 else ctx
+
+        if len(ctx.strip()) > 0:
+            prompts = [ctx]
+            results = generator.generate(
+                prompts, max_gen_len=2048, temperature=temperature, top_p=top_p
+            )
+            ctx = results[0]
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

example-chat2.py

@@ -0,0 +1,74 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# This software may be used and distributed according to the terms of the GNU General Public License version 3.
+
+from typing import Tuple
+import os
+import sys
+import torch
+import fire
+import time
+import json
+from pathlib import Path
+from llama import ModelArgs, Transformer, Tokenizer, LLaMA
+
+
+def load(
+        ckpt_dir: str,
+        tokenizer_path: str,
+        max_seq_len: int,
+        max_batch_size: int,
+) -> LLaMA:
+    print("Creating model...")
+    start_time = time.time()
+    checkpoints = sorted(Path(ckpt_dir).glob("*.pth"))
+
+    with open(Path(ckpt_dir) / "params.json", "r") as f:
+        params = json.loads(f.read())
+
+    model_args: ModelArgs = ModelArgs(
+        max_seq_len=max_seq_len, max_batch_size=max_batch_size, **params
+    )
+
+    tokenizer = Tokenizer(model_path=tokenizer_path)
+    model_args.vocab_size = tokenizer.n_words
+
+    model = Transformer(model_args)
+    model.to("cpu")
+
+    print("Loading merged checkpoint...")
+    checkpoint = torch.load(checkpoints[-1], map_location="cuda")
+    model.load_state_dict(checkpoint, strict=False)
+    del checkpoint
+
+    generator = LLaMA(model, tokenizer)
+    print(f"Loaded model in {time.time() - start_time:.2f} seconds")
+    return generator
+
+
+def main(
+        ckpt_dir: str = './model',
+        tokenizer_path: str = './tokenizer/tokenizer.model',
+        temperature: float = 0.8,
+        top_p: float = 0.95,
+        max_seq_len: int = 256,  # up to 2048
+        max_batch_size: int = 5,
+):
+    # torch.manual_seed(1)
+    torch.set_default_dtype(torch.bfloat16)
+
+    generator = load(ckpt_dir, tokenizer_path, max_seq_len, max_batch_size)
+
+    while True:
+        prompt = input(f'prompt> ')
+        if len(prompt.strip()) > 0:
+            prompts = [prompt]
+            results = generator.generate(
+                prompts, max_gen_len=256, temperature=temperature, top_p=top_p
+            )
+
+            for result in results:
+                print(result)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

example-cpu.py

@@ -0,0 +1,133 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# This software may be used and distributed according to the terms of the GNU General Public License version 3.
+
+from typing import Tuple
+import os
+import sys
+import torch
+import fire
+import time
+import json
+from pathlib import Path
+from llama import ModelArgs, Transformer, Tokenizer, LLaMA
+
+
+def load(
+        ckpt_dir: str,
+        tokenizer_path: str,
+        max_seq_len: int,
+        max_batch_size: int,
+) -> LLaMA:
+    print("Creating model...")
+    start_time = time.time()
+    checkpoints = sorted(Path(ckpt_dir).glob("*.pth"))
+
+    with open(Path(ckpt_dir) / "params.json", "r") as f:
+        params = json.loads(f.read())
+
+    model_args: ModelArgs = ModelArgs(
+        max_seq_len=max_seq_len, max_batch_size=max_batch_size, **params
+    )
+
+    tokenizer = Tokenizer(model_path=tokenizer_path)
+    model_args.vocab_size = tokenizer.n_words
+
+    model = Transformer(model_args)
+
+    # Original copyright by tloen
+    # https://github.com/tloen/llama-int8/blob/main/example.py
+    key_to_dim = {
+        "w1": 0,
+        "w2": -1,
+        "w3": 0,
+        "wo": -1,
+        "wq": 0,
+        "wk": 0,
+        "wv": 0,
+        "output": 0,
+        "tok_embeddings": -1,
+        "ffn_norm": None,
+        "attention_norm": None,
+        "norm": None,
+        "rope": None,
+    }
+
+    for i, ckpt in enumerate(checkpoints):
+        print(f"Loading checkpoint {i}")
+        checkpoint = torch.load(ckpt, map_location="cpu")
+        for parameter_name, parameter in model.named_parameters():
+            short_name = parameter_name.split(".")[-2]
+            if key_to_dim[short_name] is None and i == 0:
+                parameter.data = checkpoint[parameter_name]
+            elif key_to_dim[short_name] == 0:
+                size = checkpoint[parameter_name].size(0)
+                parameter.data[size * i: size * (i + 1), :] = checkpoint[
+                    parameter_name
+                ]
+            elif key_to_dim[short_name] == -1:
+                size = checkpoint[parameter_name].size(-1)
+                parameter.data[:, size * i: size * (i + 1)] = checkpoint[
+                    parameter_name
+                ]
+            del checkpoint[parameter_name]
+        del checkpoint
+
+    model.to("cpu")
+
+    generator = LLaMA(model, tokenizer)
+    print(f"Loaded model in {time.time() - start_time:.2f} seconds")
+    return generator
+
+
+def main(
+        ckpt_dir: str = './model',
+        tokenizer_path: str = './tokenizer/tokenizer.model',
+        temperature: float = 0.8,
+        top_p: float = 0.95,
+        max_seq_len: int = 512,  # up to 2048
+        max_batch_size: int = 32,
+):
+    # torch.manual_seed(1)
+    # torch.set_default_dtype(torch.bfloat16)
+
+    generator = load(ckpt_dir, tokenizer_path, max_seq_len, max_batch_size)
+
+    prompts = [
+        ##### For these prompts, the expected answer is the natural continuation of the prompt #####
+
+        "I believe the meaning of life is",
+        # "Simply put, the theory of relativity states that ",
+        # "Building a website can be done in 10 simple steps:\n",
+
+        ##### Few shot prompts: https://huggingface.co/blog/few-shot-learning-gpt-neo-and-inference-api #####
+
+        #         """Tweet: "I hate it when my phone battery dies."
+        # Sentiment: Negative
+        # ###
+        # Tweet: "My day has been 👍"
+        # Sentiment: Positive
+        # ###
+        # Tweet: "This is the link to the article"
+        # Sentiment: Neutral
+        # ###
+        # Tweet: "This new music video was incredibile"
+        # Sentiment:""",
+
+        #         """Translate English to French:
+        # sea otter => loutre de mer
+        # peppermint => menthe poivrée
+        # plush girafe => girafe peluche
+        # cheese =>""",
+    ]
+
+    results = generator.generate(
+        prompts, max_gen_len=256, temperature=temperature, top_p=top_p
+    )
+
+    for result in results:
+        print(result)
+        print("\n==================================\n")
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

llama.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: llama
+Version: 0.0.0
+Summary: UNKNOWN
+Home-page: UNKNOWN
+Author: UNKNOWN
+Author-email: UNKNOWN
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

llama.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+README.md
+setup.py
+llama/__init__.py
+llama/generation.py
+llama/model.py
+llama/tokenizer.py
+llama.egg-info/PKG-INFO
+llama.egg-info/SOURCES.txt
+llama.egg-info/dependency_links.txt
+llama.egg-info/top_level.txt

llama.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

llama.egg-info/top_level.txt

@@ -0,0 +1 @@
+llama

llama/__init__.py

@@ -0,0 +1,6 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# This software may be used and distributed according to the terms of the GNU General Public License version 3.
+
+from .generation import LLaMA
+from .model import ModelArgs, Transformer
+from .tokenizer import Tokenizer

llama/generation.py

@@ -0,0 +1,99 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# This software may be used and distributed according to the terms of the GNU General Public License version 3.
+
+from typing import List
+
+import torch
+import traceback
+
+from llama.tokenizer import Tokenizer
+from llama.model import Transformer
+from tqdm import trange
+
+
+class LLaMA:
+    def __init__(self, model: Transformer, tokenizer: Tokenizer):
+        self.model = model
+        self.tokenizer = tokenizer
+
+    def generate(
+            self,
+            prompts: List[str],
+            max_gen_len: int,
+            temperature: float = 0.8,
+            top_p: float = 0.95,
+    ) -> List[str]:
+        bsz = len(prompts)
+        params = self.model.params
+        assert bsz <= params.max_batch_size, (bsz, params.max_batch_size)
+
+        count_newlines = prompts[0].count("\n")
+
+        prompt_tokens = [self.tokenizer.encode(x, bos=True, eos=False) for x in prompts]
+
+        min_prompt_size = min([len(t) for t in prompt_tokens])
+        max_prompt_size = max([len(t) for t in prompt_tokens])
+
+        total_len = min(params.max_seq_len, max_gen_len + max_prompt_size)
+
+        tokens = torch.full((bsz, total_len), self.tokenizer.pad_id).long()
+        for k, t in enumerate(prompt_tokens):
+            tokens[k, : len(t)] = torch.tensor(t).long()
+            tokens[k, -1] = self.tokenizer.eos_id
+        input_text_mask = tokens != self.tokenizer.pad_id
+        start_pos = min_prompt_size
+        prev_pos = 0
+        decoded = [None] * bsz
+        for cur_pos in trange(start_pos, total_len, desc="forward"):
+            logits = self.model.forward(tokens[:, prev_pos:cur_pos], prev_pos)
+            if temperature > 0:
+                probs = torch.softmax(logits / temperature, dim=-1)
+                next_token = sample_top_p(probs, top_p)
+            else:
+                next_token = torch.argmax(logits, dim=-1)
+            next_token = next_token.reshape(-1).cpu()
+            # only replace token if prompt has already been generated
+            next_token = torch.where(
+                input_text_mask[:, cur_pos], tokens[:, cur_pos], next_token
+            )
+            tokens[:, cur_pos] = next_token
+            prev_pos = cur_pos
+
+            print("-" * 30)
+            for i, t in enumerate(tokens.tolist()):
+                # i = cur_pos
+                # t = next_token
+                # cut to max gen len
+                # t = t[: len(pr-ompt_tokens[i]) + max_gen_len]
+                t = t[: min(cur_pos, len(prompt_tokens[i]) + max_gen_len)]
+                # cut to eos tok if any
+                try:
+                    t = t[: t.index(self.tokenizer.eos_id)]
+                except ValueError:
+                    pass  # traceback.print_exc()
+                try:
+                    d = self.tokenizer.decode(t)
+                    print([i] * 20)
+                    print(d)
+                    decoded[i] = d
+
+                    result_count_newlines = d.count("\n")
+                    if result_count_newlines > count_newlines:
+                        return decoded
+
+                except IndexError:
+                    traceback.print_exc()
+                    print(t)
+            print("-" * 30)
+        return decoded
+
+
+def sample_top_p(probs, p):
+    probs_sort, probs_idx = torch.sort(probs, dim=-1, descending=True)
+    probs_sum = torch.cumsum(probs_sort, dim=-1)
+    mask = probs_sum - probs_sort > p
+    probs_sort[mask] = 0.0
+    probs_sort.div_(probs_sort.sum(dim=-1, keepdim=True))
+    next_token = torch.multinomial(probs_sort, num_samples=1)
+    next_token = torch.gather(probs_idx, -1, next_token)
+    return next_token

llama/model.py

@@ -0,0 +1,270 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# This software may be used and distributed according to the terms of the GNU General Public License version 3.
+
+from typing import Optional, Tuple
+from dataclasses import dataclass
+import math
+
+import torch
+from torch import nn
+import torch.nn.functional as F
+from torch.nn.utils import skip_init
+
+from tqdm import tqdm
+
+@dataclass
+class ModelArgs:
+    dim: int = 512
+    n_layers: int = 8
+    n_heads: int = 8
+    vocab_size: int = -1  # defined later by tokenizer
+    multiple_of: int = 256  # make SwiGLU hidden layer size multiple of large power of 2
+    norm_eps: float = 1e-5
+
+    max_batch_size: int = 32
+    max_seq_len: int = 1024
+
+
+class RMSNorm(torch.nn.Module):
+    def __init__(self, dim: int, eps: float = 1e-6):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(dim))
+
+    def _norm(self, x):
+        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+
+    def forward(self, x):
+        output = self._norm(x.float()).type_as(x)
+        return output * self.weight
+
+
+def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0):
+    freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim))
+    t = torch.arange(end, device=freqs.device)  # type: ignore
+    freqs = torch.outer(t, freqs).float()  # type: ignore
+    freqs_cis = torch.polar(torch.ones_like(freqs), freqs)  # complex64
+    return freqs_cis
+
+
+def reshape_for_broadcast(freqs_cis: torch.Tensor, x: torch.Tensor):
+    ndim = x.ndim
+    assert 0 <= 1 < ndim
+    assert freqs_cis.shape == (x.shape[1], x.shape[-1])
+    shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)]
+    return freqs_cis.view(*shape)
+
+
+def apply_rotary_emb(
+    xq: torch.Tensor,
+    xk: torch.Tensor,
+    freqs_cis: torch.Tensor,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2))
+    xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2))
+    freqs_cis = reshape_for_broadcast(freqs_cis, xq_)
+    xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3)
+    xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3)
+    return xq_out.type_as(xq), xk_out.type_as(xk)
+
+
+class Attention(nn.Module):
+    def __init__(self, args: ModelArgs):
+        super().__init__()
+
+        self.n_local_heads = args.n_heads  # // fs_init.get_model_parallel_world_size()
+        self.head_dim = args.dim // args.n_heads
+
+        self.wq = skip_init(nn.Linear,
+            args.dim,
+            args.n_heads * self.head_dim,
+            bias=False,
+        )
+        self.wk = skip_init(nn.Linear,
+            args.dim,
+            args.n_heads * self.head_dim,
+            bias=False,
+        )
+        self.wv = skip_init(nn.Linear,
+            args.dim,
+            args.n_heads * self.head_dim,
+            bias=False,
+        )
+        self.wo = skip_init(nn.Linear,
+            args.n_heads * self.head_dim,
+            args.dim,
+            bias=False,
+        )
+
+        self.cache_k = torch.zeros(
+            (args.max_batch_size, args.max_seq_len, self.n_local_heads, self.head_dim)
+        ).cuda()
+        self.cache_v = torch.zeros(
+            (args.max_batch_size, args.max_seq_len, self.n_local_heads, self.head_dim)
+        ).cuda()
+
+    def forward(self, x: torch.Tensor, start_pos: int, freqs_cis: torch.Tensor, mask: Optional[torch.Tensor]):
+        bsz, seqlen, _ = x.shape
+        xq, xk, xv = self.wq(x), self.wk(x), self.wv(x)
+
+        xq = xq.view(bsz, seqlen, self.n_local_heads, self.head_dim)
+        xk = xk.view(bsz, seqlen, self.n_local_heads, self.head_dim)
+        xv = xv.view(bsz, seqlen, self.n_local_heads, self.head_dim)
+
+        xq, xk = apply_rotary_emb(xq, xk, freqs_cis=freqs_cis)
+
+        self.cache_k = self.cache_k.to(xq)
+        self.cache_v = self.cache_v.to(xq)
+
+        self.cache_k[:bsz, start_pos : start_pos + seqlen] = xk
+        self.cache_v[:bsz, start_pos : start_pos + seqlen] = xv
+
+        keys = self.cache_k[:bsz, : start_pos + seqlen]
+        values = self.cache_v[:bsz, : start_pos + seqlen]
+
+        xq = xq.transpose(1, 2)
+        keys = keys.transpose(1, 2)
+        values = values.transpose(1, 2)
+        scores = torch.matmul(xq, keys.transpose(2, 3)) / math.sqrt(self.head_dim)
+        if mask is not None:
+            scores = scores + mask  # (bs, n_local_heads, slen, cache_len + slen)
+        scores = F.softmax(scores.float(), dim=-1).type_as(xq)
+        output = torch.matmul(scores, values)  # (bs, n_local_heads, slen, head_dim)
+        output = output.transpose(
+            1, 2
+        ).contiguous().view(bsz, seqlen, -1)
+
+        return self.wo(output)
+
+
+class FeedForward(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        hidden_dim: int,
+        multiple_of: int,
+    ):
+        super().__init__()
+        hidden_dim = int(2 * hidden_dim / 3)
+        hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)
+
+        self.w1 = skip_init(nn.Linear,
+            dim,
+            hidden_dim,
+            bias=False,
+        )
+        self.w2 = skip_init(nn.Linear,
+            hidden_dim,
+            dim,
+            bias=False,
+        )
+        self.w3 = skip_init(nn.Linear,
+            dim,
+            hidden_dim,
+            bias=False,
+        )
+
+    def forward(self, x):
+        return self.w2(F.silu(self.w1(x)) * self.w3(x))
+
+
+class TransformerBlock(nn.Module):
+    def __init__(self, layer_id: int, args: ModelArgs):
+        super().__init__()
+        self.n_heads = args.n_heads
+        self.dim = args.dim
+        self.head_dim = args.dim // args.n_heads
+        self.attention = Attention(args)
+        self.feed_forward = FeedForward(
+            dim=args.dim, hidden_dim=4 * args.dim, multiple_of=args.multiple_of
+        )
+        self.layer_id = layer_id
+        self.attention_norm = RMSNorm(args.dim, eps=args.norm_eps)
+        self.ffn_norm = RMSNorm(args.dim, eps=args.norm_eps)
+
+    def forward(self, x: torch.Tensor, start_pos: int, freqs_cis: torch.Tensor, mask: Optional[torch.Tensor]):
+        h = x + self.attention.forward(self.attention_norm(x), start_pos, freqs_cis, mask)
+        out = h + self.feed_forward.forward(self.ffn_norm(h))
+        return out
+
+# https://github.com/gmorenz/llama/commit/4daf7f1a2f2bb22208b5d464bc2a18511d54408d
+def move_parameters_to_gpu(module):
+    if not hasattr(module, "saved"):
+        module.saved = module._parameters.copy()
+    for k, param in module.saved.items():
+        if param is not None:
+            module._parameters[k] = param.to("cuda", non_blocking=True)
+    for child in module.children():
+        move_parameters_to_gpu(child)
+
+def move_parameters_to_cpu(module):
+    for k, param in module.saved.items():
+        del module._parameters[k]
+        module._parameters[k] = param
+    for child in module.children():
+        move_parameters_to_cpu(child)
+
+
+class Transformer(nn.Module):
+    def __init__(self, params: ModelArgs):
+        super().__init__()
+        self.params = params
+        self.vocab_size = params.vocab_size
+        self.n_layers = params.n_layers
+
+        self.tok_embeddings = skip_init(nn.Embedding,
+            params.vocab_size,
+            params.dim,
+        )
+
+        self.layers = torch.nn.ModuleList()
+        for layer_id in range(params.n_layers):
+            self.layers.append(TransformerBlock(layer_id, params))
+
+        self.layer_locations = [None] * len(self.layers)
+
+        self.norm = RMSNorm(params.dim, eps=params.norm_eps).cuda()
+        self.output = skip_init(nn.Linear,
+            params.dim,
+            params.vocab_size,
+            bias=False,
+        ).cuda()
+
+        self.freqs_cis = precompute_freqs_cis(
+            self.params.dim // self.params.n_heads, self.params.max_seq_len * 2
+        ).cuda()
+
+    @torch.inference_mode()
+    def forward(self, tokens: torch.Tensor, start_pos: int):
+        use_gpu = True  # start_pos == 0
+
+        _bsz, seqlen = tokens.shape
+        h = self.tok_embeddings(tokens)
+        self.freqs_cis = self.freqs_cis
+        freqs_cis = self.freqs_cis[start_pos : start_pos + seqlen]
+        if use_gpu:
+            h = h.cuda()
+
+        mask = None
+        if seqlen > 1:
+            mask = torch.full(
+                (1, 1, seqlen, seqlen), float("-inf"), device=tokens.device
+            )
+            mask = torch.triu(mask, diagonal=start_pos + 1).type_as(h)
+
+        if use_gpu and mask is not None:
+            mask = mask.cuda()
+
+        for layer in tqdm(self.layers, desc="flayers", leave=True):
+            if use_gpu:
+                move_parameters_to_gpu(layer)
+            h = layer(h, start_pos, freqs_cis, mask)
+            if use_gpu:
+                move_parameters_to_cpu(layer)
+
+        h = self.norm(h)
+        if use_gpu:
+            del mask
+            torch.cuda.empty_cache()
+        output = self.output(h[:, -1, :])  # only compute last logits
+        return output.float()

llama/tokenizer.py

@@ -0,0 +1,40 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# This software may be used and distributed according to the terms of the GNU General Public License version 3.
+
+from sentencepiece import SentencePieceProcessor
+from logging import getLogger
+from typing import List
+import os
+
+
+logger = getLogger()
+
+
+class Tokenizer:
+    def __init__(self, model_path: str):
+        # reload tokenizer
+        assert os.path.isfile(model_path), model_path
+        self.sp_model = SentencePieceProcessor(model_file=model_path)
+        logger.info(f"Reloaded SentencePiece model from {model_path}")
+
+        # BOS / EOS token IDs
+        self.n_words: int = self.sp_model.vocab_size()
+        self.bos_id: int = self.sp_model.bos_id()
+        self.eos_id: int = self.sp_model.eos_id()
+        self.pad_id: int = self.sp_model.pad_id()
+        logger.info(
+            f"#words: {self.n_words} - BOS ID: {self.bos_id} - EOS ID: {self.eos_id}"
+        )
+        assert self.sp_model.vocab_size() == self.sp_model.get_piece_size()
+
+    def encode(self, s: str, bos: bool, eos: bool) -> List[int]:
+        assert type(s) is str
+        t = self.sp_model.encode(s)
+        if bos:
+            t = [self.bos_id] + t
+        if eos:
+            t = t + [self.eos_id]
+        return t
+
+    def decode(self, t: List[int]) -> str:
+        return self.sp_model.decode(t)

merge-weights.py

@@ -0,0 +1,168 @@
+# Original copyright by Jason Phang
+# https://github.com/zphang
+# Taken here
+# https://github.com/huggingface/transformers/pull/21955/commits/8978f28e6c44b083c0b190d3931902c2904c940a#diff-110a445233a8b15a0875998eeaf75cb8607b38a5daa736291dd058766879bbdd
+
+import argparse
+import json
+import os
+import shutil
+import torch
+
+"""
+Sample usage:
+    ```
+    python merge_weights.py --input_dir D:\Downloads\LLaMA --model_size 13B
+    ```
+"""
+
+INTERMEDIATE_SIZE_MAP = {
+    "7B": 11008,
+    "13B": 13824,
+    "30B": 17920,
+    "65B": 22016,
+}
+
+NUM_SHARDS = {
+    "7B": 1,
+    "13B": 2,
+    "30B": 4,
+    "65B": 8,
+}
+
+
+def read_json(path):
+    with open(path, "r") as f:
+        return json.loads(f.read())
+
+
+def write_model(input_base_path, model_size):
+    assert model_size in INTERMEDIATE_SIZE_MAP
+
+    params = read_json(os.path.join(input_base_path, "params.json"))
+    num_shards = NUM_SHARDS[model_size]
+    n_layers = params["n_layers"]
+    n_heads = params["n_heads"]
+    n_heads_per_shard = n_heads // num_shards
+    dim = params["dim"]
+    dims_per_head = dim // n_heads
+
+    # Load weights
+    if model_size == "7B":
+        loaded = torch.load(os.path.join(input_base_path, "consolidated.00.pth"), map_location="cuda")
+    else:
+        loaded = [
+            torch.load(os.path.join(input_base_path, f"consolidated.{i:02d}.pth"), map_location="cuda")
+            for i in range(num_shards)
+        ]
+
+    state_dict = {}
+
+    for layer_i in range(n_layers):
+        if model_size == "7B":
+            state_dict |= {
+                f"layers.{layer_i}.attention.wq.weight": loaded[
+                    f"layers.{layer_i}.attention.wq.weight"
+                ],
+                f"layers.{layer_i}.attention.wk.weight": loaded[
+                    f"layers.{layer_i}.attention.wk.weight"
+                ],
+                f"layers.{layer_i}.attention.wv.weight": loaded[
+                    f"layers.{layer_i}.attention.wv.weight"
+                ],
+                f"layers.{layer_i}.attention.wo.weight": loaded[
+                    f"layers.{layer_i}.attention.wo.weight"
+                ],
+                f"layers.{layer_i}.feed_forward.w1.weight": loaded[
+                    f"layers.{layer_i}.feed_forward.w1.weight"
+                ],
+                f"layers.{layer_i}.feed_forward.w2.weight": loaded[
+                    f"layers.{layer_i}.feed_forward.w2.weight"
+                ],
+                f"layers.{layer_i}.feed_forward.w3.weight": loaded[
+                    f"layers.{layer_i}.feed_forward.w3.weight"
+                ],
+                f"layers.{layer_i}.attention_norm.weight": loaded[
+                    f"layers.{layer_i}.attention_norm.weight"
+                ],
+                f"layers.{layer_i}.ffn_norm.weight": loaded[f"layers.{layer_i}.ffn_norm.weight"],
+            }
+        else:
+            state_dict |= {
+                f"layers.{layer_i}.attention_norm.weight": loaded[0][
+                    f"layers.{layer_i}.attention_norm.weight"
+                ],
+                f"layers.{layer_i}.ffn_norm.weight": loaded[0][f"layers.{layer_i}.ffn_norm.weight"],
+            }
+            state_dict[f"layers.{layer_i}.attention.wq.weight"] = torch.cat(
+                [
+                    loaded[i][f"layers.{layer_i}.attention.wq.weight"].view(n_heads_per_shard, dims_per_head, dim)
+                    for i in range(num_shards)
+                ],
+                dim=0,
+            ).reshape(dim, dim)
+            state_dict[f"layers.{layer_i}.attention.wk.weight"] = torch.cat(
+                [
+                    loaded[i][f"layers.{layer_i}.attention.wk.weight"].view(n_heads_per_shard, dims_per_head, dim)
+                    for i in range(num_shards)
+                ],
+                dim=0,
+            ).reshape(dim, dim)
+            state_dict[f"layers.{layer_i}.attention.wv.weight"] = torch.cat(
+                [
+                    loaded[i][f"layers.{layer_i}.attention.wv.weight"].view(n_heads_per_shard, dims_per_head, dim)
+                    for i in range(num_shards)
+                ],
+                dim=0,
+            ).reshape(dim, dim)
+            state_dict[f"layers.{layer_i}.attention.wo.weight"] = torch.cat(
+                [loaded[i][f"layers.{layer_i}.attention.wo.weight"] for i in range(num_shards)], dim=1
+            )
+            state_dict[f"layers.{layer_i}.feed_forward.w1.weight"] = torch.cat(
+                [loaded[i][f"layers.{layer_i}.feed_forward.w1.weight"] for i in range(num_shards)], dim=0
+            )
+            state_dict[f"layers.{layer_i}.feed_forward.w2.weight"] = torch.cat(
+                [loaded[i][f"layers.{layer_i}.feed_forward.w2.weight"] for i in range(num_shards)], dim=1
+            )
+            state_dict[f"layers.{layer_i}.feed_forward.w3.weight"] = torch.cat(
+                [loaded[i][f"layers.{layer_i}.feed_forward.w3.weight"] for i in range(num_shards)], dim=0
+            )
+
+    if model_size == "7B":
+        state_dict |= {
+            "tok_embeddings.weight": loaded["tok_embeddings.weight"],
+            "norm.weight": loaded["norm.weight"],
+            "output.weight": loaded["output.weight"],
+        }
+    else:
+        state_dict |= {
+            "norm.weight": loaded[0]["norm.weight"],
+            "tok_embeddings.weight": torch.cat(
+                [loaded[i]["tok_embeddings.weight"] for i in range(num_shards)], dim=1
+            ),
+            "output.weight": torch.cat([loaded[i]["output.weight"] for i in range(num_shards)], dim=0),
+        }
+
+    torch.save(state_dict, 'merged.pth')
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--input_dir",
+        help="Location of LLaMA weights, which contains tokenizer.model and model folders",
+    )
+    parser.add_argument(
+        "--model_size",
+        choices=["7B", "13B", "30B", "65B"],
+    )
+    args = parser.parse_args()
+
+    write_model(
+        input_base_path=os.path.join(args.input_dir, args.model_size),
+        model_size=args.model_size,
+    )
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,5 @@
+fairscale
+fire
+sentencepiece
+tqdm
+pyarrow

setup.py

@@ -0,0 +1,6 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# This software may be used and distributed according to the terms of the GNU General Public License version 3.
+
+from setuptools import setup, find_packages
+
+setup(name="llama", version="0.0.0", packages=find_packages())

tokenizer/.gitignore

@@ -0,0 +1 @@
+

tokenizer/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723