added

.gitignore

@@ -0,0 +1,163 @@
+.idea
+.env
+poetry.lock
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

Dockerfile

@@ -0,0 +1,36 @@
+FROM docker.io/nvidia/cuda:11.7.1-cudnn8-devel-ubuntu20.04
+COPY --link --chown=1000 ./ /home/user/app
+RUN useradd -m -u 1000 user
+ENV TZ=Asia/Tokyo
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+RUN sed -i 's http://deb.debian.org http://cdn-aws.deb.debian.org g' /etc/apt/sources.list && \
+    sed -i 's http://archive.ubuntu.com http://us-east-1.ec2.archive.ubuntu.com g' /etc/apt/sources.list && \
+    sed -i '/security/d' /etc/apt/sources.list && apt-get update && \
+    apt-get install -y \
+        git \
+        make build-essential libssl-dev zlib1g-dev \
+        libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
+        libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev git-lfs \
+        ffmpeg libsm6 libxext6 cmake libgl1-mesa-glx && \
+    rm -rf /var/lib/apt/lists/* && \
+    git lfs install
+RUN curl https://pyenv.run | bash
+ENV PYENV_ROOT /root/.pyenv
+ENV PATH ${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:${PATH}
+RUN eval "$(pyenv init -)" && \
+    eval "$(pyenv virtualenv-init -)" && \
+    pyenv install 3.9.7 && \
+    pyenv global 3.9.7 && \
+    pyenv rehash && \
+    pip install --no-cache-dir --upgrade pip==22.3.1 setuptools wheel && \
+    pip install --no-cache-dir datasets "huggingface-hub>=0.12.1" "protobuf<4" "click<8.1" && \
+    curl -sSL https://install.python-poetry.org | python -
+ENV PATH /root/.local/bin:${PATH}
+WORKDIR /home/user/app
+
+RUN poetry install
+RUN --mount=type=secret,id=HF_TOKEN,mode=0444,required=true \
+    git config --global credential.helper store && \
+    huggingface-cli login --token $(cat /run/secrets/HF_TOKEN) --add-to-git-credential
+RUN poetry run python model_pull.py
+CMD ["env", "poetry", "run", "python", "app.py", "--host", "0.0.0.0", "--port", "7860"]

Makefile

@@ -0,0 +1,35 @@
+
+RUN := poetry run
+
+.PHONY: check
+check: lint mypy
+
+.PHONY: lint
+lint: lint-black lint-isort lint-flake8
+
+.PHONY: lint-black
+lint-black:
+	$(RUN) black --check --diff --quiet . 
+
+.PHONY: lint-isort
+lint-isort:
+	$(RUN) isort --check --quiet . 
+
+.PHONY: lint-flake8
+lint-flake8:
+	$(RUN) pflake8 . 
+
+.PHONY: mypy
+mypy:
+	$(RUN) mypy . 
+
+.PHONY: format
+format: format-black format-isort
+
+.PHONY: format-black
+format-black:
+	$(RUN) black --quiet . 
+
+.PHONY: format-isort
+format-isort:
+	$(RUN) isort --quiet .

README.md

@@ -1,5 +1,5 @@
 ---
-title: Llama 13b Japanese Lora V0 1ep
+title: LLaMA 13B Japanese LoRA v0 1 epoch
 emoji: 🐨
 colorFrom: gray
 colorTo: gray

app.py

@@ -0,0 +1,138 @@
+from typing import Optional
+
+import gradio as gr
+import torch
+from peft import PeftModel
+from transformers import AutoModelForCausalLM
+from transformers import AutoTokenizer
+from transformers import GenerationConfig
+
+print("starting server ...")
+BASE_MODEL = "decapoda-research/llama-13b-hf"
+LORA_WEIGHTS = "izumi-lab/llama-13b-japanese-lora-v0-1ep"
+
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
+
+if torch.cuda.is_available():
+    device = "cuda"
+else:
+    device = "cpu"
+
+try:
+    if torch.backends.mps.is_available():
+        device = "mps"
+except Exception:
+    pass
+
+if device == "cuda":
+    model = AutoModelForCausalLM.from_pretrained(
+        BASE_MODEL,
+        load_in_8bit=False,
+        torch_dtype=torch.float16,
+        device_map="auto",
+    )
+    model = PeftModel.from_pretrained(
+        model, LORA_WEIGHTS, torch_dtype=torch.float16
+    )
+elif device == "mps":
+    model = AutoModelForCausalLM.from_pretrained(
+        BASE_MODEL,
+        device_map={"": device},
+        torch_dtype=torch.float16,
+    )
+    model = PeftModel.from_pretrained(
+        model,
+        LORA_WEIGHTS,
+        device_map={"": device},
+        torch_dtype=torch.float16,
+    )
+else:
+    model = AutoModelForCausalLM.from_pretrained(
+        BASE_MODEL, device_map={"": device}, low_cpu_mem_usage=True
+    )
+    model = PeftModel.from_pretrained(
+        model,
+        LORA_WEIGHTS,
+        device_map={"": device},
+    )
+
+
+def generate_prompt(instruction: str, input: Optional[str] = None):
+    if input:
+        return f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
+### Instruction:
+{instruction}
+### Input:
+{input}
+### Response:"""
+    else:
+        return f"""Below is an instruction that describes a task. Write a response that appropriately completes the request.
+### Instruction:
+{instruction}
+### Response:"""
+
+
+if device != "cpu":
+    model.half()
+model.eval()
+if torch.__version__ >= "2":
+    model = torch.compile(model)
+
+
+def evaluate(
+    instruction: str,
+    input: Optional[str] = None,
+    temperature: float = 0.7,
+    top_p: float = 1.0,
+    top_k: int = 40,
+    num_beams: int = 4,
+    max_new_tokens: int = 256,
+    **kwargs,
+):
+    prompt = generate_prompt(instruction, input)
+    inputs = tokenizer(prompt, return_tensors="pt")
+    input_ids = inputs["input_ids"].to(device)
+    generation_config = GenerationConfig(
+        temperature=temperature,
+        top_p=top_p,
+        top_k=top_k,
+        num_beams=num_beams,
+        **kwargs,
+    )
+    with torch.no_grad():
+        generation_output = model.generate(
+            input_ids=input_ids,
+            generation_config=generation_config,
+            return_dict_in_generate=True,
+            output_scores=True,
+            max_new_tokens=max_new_tokens,
+        )
+    s = generation_output.sequences[0]
+    output = tokenizer.decode(s)
+    return output.split("### Response:")[1].strip()
+
+
+g = gr.Interface(
+    fn=evaluate,
+    inputs=[
+        gr.components.Textbox(lines=2, label="Instruction", placeholder="東京から大阪に行くには？"),
+        gr.components.Textbox(lines=2, label="Input", placeholder="none"),
+        gr.components.Slider(minimum=0, maximum=1, value=0.7, label="Temperature"),
+        gr.components.Slider(minimum=0, maximum=1, value=1.0, label="Top p"),
+        gr.components.Slider(minimum=0, maximum=100, step=1, value=40, label="Top k"),
+        gr.components.Slider(minimum=1, maximum=4, step=1, value=4, label="Beams"),
+        gr.components.Slider(
+            minimum=1, maximum=512, step=1, value=128, label="Max tokens"
+        ),
+    ],
+    outputs=[
+        gr.inputs.Textbox(
+            lines=5,
+            label="Output",
+        )
+    ],
+    title="izumi-lab/calm-7b-lora-v0-1ep",
+    description="izumi-lab/calm-7b-lora-v0-1ep is a 7B-parameter Calm model finetuned to follow instructions. It is trained on the [izumi-lab/llm-japanese-dataset](https://huggingface.co/datasets/izumi-lab/llm-japanese-dataset) dataset and makes use of the Huggingface Calm-7b implementation. For more information, please visit [the project's website](https://llm.msuzuki.me).",
+)
+g.queue(concurrency_count=1)
+g.launch()

model_pull.py

@@ -0,0 +1,18 @@
+import torch
+from peft import PeftModel
+from transformers import AutoModelForCausalLM
+from transformers import AutoTokenizer
+
+BASE_MODEL = "decapoda-research/llama-13b-hf"
+LORA_WEIGHTS = "izumi-lab/llama-13b-japanese-lora-v0-1ep"
+
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
+model = AutoModelForCausalLM.from_pretrained(
+    BASE_MODEL,
+    load_in_8bit=False,
+    torch_dtype=torch.float16,
+    device_map="auto",
+)
+model = PeftModel.from_pretrained(
+    model, LORA_WEIGHTS, torch_dtype=torch.float16, use_auth_token=True
+)

pyproject.toml

@@ -0,0 +1,60 @@
+[tool.poetry]
+name = "space-llama-13b-japanese-lora-v0-1ep"
+version = "0.1.0"
+description = ""
+authors = ["Masanori HIRANO <masa.hirano.1996@gmail.com>"]
+license = "other"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+peft = "^0.3.0"
+transformers = {git = "https://github.com/huggingface/transformers.git", branch = "main"}
+gradio = "^3.32.0"
+torch = "^2.0.1"
+huggingface-hub = "^0.14.1"
+
+
+[tool.poetry.group.dev.dependencies]
+black = "^23.3.0"
+isort = "^5.12.0"
+mypy = "^1.3.0"
+flake8 = "^6.0.0"
+pyproject-flake8 = "^6.0.0.post1"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.isort]
+profile = 'black'
+force_single_line = true
+skip = [
+    ".git",
+    "__pycache__",
+    "docs",
+    "build",
+    "dist",
+    "examples",
+    ".venv",
+    "tests/examples"
+]
+
+[tool.mypy]
+disallow_untyped_defs = true
+ignore_missing_imports = true
+
+[tool.flake8]
+ignore = "E203,E231,E501,W503"
+max-line-length = 88
+exclude = [
+    ".git",
+    "__pycache__",
+    "docs",
+    "build",
+    "dist",
+    "examples",
+    ".venv",
+    "__init__.py"
+    ]
+select = "B,B950,C,E,F,W"