init code

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.safetensor filter=lfs diff=lfs merge=lfs -text

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vllm"]
+	path = vllm
+	url = https://github.com/vllm-project/vllm.git

Dockerfile

@@ -0,0 +1,37 @@
+FROM ubuntu:22.04 AS cpu-test-1
+
+RUN apt-get update -y \
+    && apt-get install -y curl git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 libnuma-dev \
+    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
+
+# https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/performance_tuning/tuning_guide.html
+# intel-openmp provides additional performance improvement vs. openmp
+# tcmalloc provides better memory allocation efficiency, e.g, holding memory in caches to speed up access of commonly-used objects.
+RUN pip install intel-openmp
+
+ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/usr/local/lib/libiomp5.so:$LD_PRELOAD"
+
+RUN echo 'ulimit -c 0' >> ~/.bashrc
+
+RUN pip install https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_dev/cpu/intel_extension_for_pytorch-2.4.0%2Bgitfbaa4bc-cp310-cp310-linux_x86_64.whl
+
+RUN pip install --upgrade pip \
+    && pip install wheel packaging ninja "setuptools>=49.4.0" numpy
+
+FROM cpu-test-1 AS build
+
+COPY ./ /workspace
+
+WORKDIR /workspace/vllm
+
+RUN pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/test/cpu
+
+# Support for building with non-AVX512 vLLM: docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" ...
+ARG VLLM_CPU_DISABLE_AVX512
+ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512}
+
+RUN VLLM_TARGET_DEVICE=cpu python3 setup.py install
+
+WORKDIR /workspace/
+
+CMD ["/bin/bash", "/workspace/start_vllm.sh"]

app.py

@@ -0,0 +1,54 @@
+from fastapi import FastAPI
+from pydantic import BaseModel, Extra
+import argparse
+from typing import Optional
+import uvicorn
+from model import ChallengePromptGenerator
+
+
+class Prompt(BaseModel, extra=Extra.allow):
+    prompt: str
+    seed: Optional[int] = 0
+    max_length: Optional[int] = 77
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", type=int, default=10001)
+    parser.add_argument("--netuid", type=str, default=23)
+    parser.add_argument("--min_stake", type=int, default=100)
+    parser.add_argument(
+        "--chain_endpoint",
+        type=str,
+        default="finney",
+    )
+    parser.add_argument("--disable_secure", action="store_true", default=False)
+    args = parser.parse_args()
+    return args
+
+
+class ChallengeImage:
+    def __init__(self):
+        self.challenge_prompt = ChallengePromptGenerator()
+        self.app = FastAPI(title="Challenge Prompt")
+        self.app.add_api_route("/", self.__call__, methods=["POST"])
+
+    async def __call__(
+        self,
+        data: Prompt,
+    ):
+        data = dict(data)
+        prompt = data["prompt"]
+        if not prompt:
+            prompt = "an image of "
+        complete_prompt = self.challenge_prompt.infer_prompt(
+            [prompt], max_generation_length=77, sampling_topk=100
+        )[0].strip()
+        return complete_prompt
+
+
+if __name__ == "__main__":
+    args = get_args()
+    print("Args: ", args)
+    app = ChallengeImage()
+    uvicorn.run(app.app, host="0.0.0.0", port=args.port)

checkpoint-15000/added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "</LongCaption>": 50258,
+  "<LongCaption>": 50257,
+  "<PAD>": 50259
+}

checkpoint-15000/config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "succinctly/text2image-prompt-generator",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "torch_dtype": "float16",
+  "transformers_version": "4.43.3",
+  "use_cache": true,
+  "vocab_size": 50260
+}

checkpoint-15000/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.43.3"
+}

checkpoint-15000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df6545a8a4b9a368b3783fdcac59c230697fca5daa8dd744c408176902232bd2
+size 248899120

checkpoint-15000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:863969ceb11b25a11efa54ba947ea3c0d08714d9710bd95e77ca14ac5e9a7772
+size 1385018

checkpoint-15000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:085ddbfee84141c02b403cccce94ddbc0db883e99e4e7ae5fdbfaa20f1b75777
+size 14244

checkpoint-15000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adf5bf452761c234e3daa85a999070bcf12b7ea957b65555b14a80fc069fa45d
+size 1064

checkpoint-15000/special_tokens_map.json

@@ -0,0 +1,12 @@
+{
+  "bos_token": "<LongCaption>",
+  "eos_token": "</LongCaption>",
+  "pad_token": "<PAD>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-15000/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50257": {
+      "content": "<LongCaption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50258": {
+      "content": "</LongCaption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50259": {
+      "content": "<PAD>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<LongCaption>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</LongCaption>",
+  "model_max_length": 1024,
+  "pad_token": "<PAD>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

checkpoint-15000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d966d8425fcd55b317e293394041be36b2ae7882096bdd70cf0ca1695a8d60ec
+size 5304

index.html

@@ -0,0 +1,15 @@
+<main>
+    <section id="T2I Prompt Generatpr">
+      <h2>Text generation using Flan T5</h2>
+      <form class="text-gen-form">
+        <label for="text-gen-input">Text prompt</label>
+        <input
+          id="text-gen-input"
+          type="text"
+          value="German: There are many ducks"
+        />
+        <button id="text-gen-submit">Submit</button>
+        <p class="text-gen-output"></p>
+      </form>
+    </section>
+  </main>

model.py

@@ -0,0 +1,50 @@
+import os
+import torch
+from vllm import LLM, SamplingParams
+import logging
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+class ChallengePromptGenerator:
+    def __init__(
+        self,
+        model_local_dir="checkpoint-15000",
+    ):
+        self.generator = LLM(
+            model_local_dir, 
+            dtype="bfloat16",
+        )
+
+        
+    def infer_prompt(
+        self,
+        prompts,
+        max_generation_length=77,
+        beam_size=1,
+        sampling_temperature=0.9,
+        sampling_topk=1,
+        sampling_topp=1,
+    ):
+        added_prompts = [f"{self.generator.get_tokenizer().bos_token} {prompt}" for prompt in prompts]
+        
+        sampling_params = SamplingParams(
+            max_tokens=max_generation_length,
+            temperature=sampling_temperature,
+            top_k=sampling_topk,
+            top_p=sampling_topp,
+            use_beam_search=(beam_size > 1),
+        )
+        
+        outputs = self.generator.generate(added_prompts, sampling_params)
+        out = []
+        for i in range(len(outputs)):
+            tmp_out = prompts[i] + outputs[i].outputs[0].text
+            if tmp_out[-1] != ".":
+                tmp_out = ".".join(tmp_out.split(".")[:-1]) + "."
+            out.append(tmp_out)
+        return out

requirements.txt

@@ -0,0 +1,3 @@
+fastapi
+uvicorn
+pydantic

start_vllm.sh

@@ -0,0 +1 @@
+python app.py --port 8000

vllm

@@ -0,0 +1 @@
+Subproject commit daed30c4a917c870f8fbddf45e3b027710c0842b