basic working version

Dockerfile

@@ -1,6 +1,3 @@
-# read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
-# you will also find guides on how best to write your Dockerfile
-
 FROM python:3.9
 
 WORKDIR /code
@@ -9,6 +6,15 @@ COPY ./requirements.txt /code/requirements.txt
 
 RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
 
-COPY . .
+RUN useradd -m -u 1000 user
+
+USER user
+
+ENV HOME=/home/user \
+  PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+COPY --chown=user . $HOME/app
 
-CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

main.py

@@ -1,8 +1,47 @@
-from fastapi import FastAPI
+"""Model hosted on Hugging face.
+
+Based on: https://huggingface.co/docs/hub/spaces-sdks-docker-first-demo
+"""
+
+from fastapi import FastAPI, Request
+
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+from transformers import T5Tokenizer, T5ForConditionalGeneration
+
+
+# FROM: https://huggingface.co/facebook/blenderbot-400M-distill?text=Hey+my+name+is+Thomas%21+How+are+you%3F
+# tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot-400M-distill")
+# model = AutoModelForSeq2SeqLM.from_pretrained("facebook/blenderbot-400M-distill")
+# tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot-1B-distill")
+# model = AutoModelForSeq2SeqLM.from_pretrained("facebook/blenderbot-1B-distill")
+tokenizer = T5Tokenizer.from_pretrained("google/flan-t5-small")
+model = T5ForConditionalGeneration.from_pretrained(
+    "google/flan-t5-small", device_map="auto")
+
+token_size_limit = 128
+
 
 app = FastAPI()
 
 
+@app.post('/reply')
+async def Reply(req: Request):
+    request = await req.json()
+    msg = request['msg']
+    print(f'MSG: {msg}')
+
+    input_ids = tokenizer(msg, return_tensors='pt').input_ids  # .to('cuda')
+    output = model.generate(
+        input_ids[:, -token_size_limit:],
+        do_sample=True,
+        temperature=0.9,
+        max_length=100,
+    )
+    reply = tokenizer.batch_decode(output)[0]
+    print(f'REPLY: {reply}')
+    return {'reply': reply}
+
+
 @app.get("/")
 def read_root():
     return {"Hello": "World!"}

requirements.txt

@@ -3,4 +3,5 @@ requests==2.27.*
 sentencepiece==0.1.*
 torch==1.11.*
 transformers==4.*
-uvicorn[standard]==0.17.*
+uvicorn[standard]==0.17.*
+accelerate

tutorial.md

@@ -1,3 +1,16 @@
 ## Use git to push changes to huggingface repository
 
-First use `huggingface_cli.exe login` to login (follow its instruction), then use git commands for pushing.
+First use `huggingface_cli.exe login` to login (follow its instruction), then use git commands for pushing.
+
+## Build/run via docker locally
+
+```bash
+docker build -t fastapi .
+docker run  -it -p 7860:7860 fastapi
+```
+
+## CURL POST example
+
+```bash
+curl -X POST http://localhost:7860/reply -H 'Content-Type: application/json' -d '{"msg": "hi"}'
+```