Hugging Face's logo

Spaces:
yusufs
/
vllm-inference
Paused

App Files
yusufs commited on
Commit
8b37c20
·
1 Parent(s): b826155

feat(llama3.2): using Llama-3.2-3B-Instruct 0cb88a4f764b7a12671c53f0838cd831a0843b95

Browse files
Files changed (2) hide show
  1. run-llama.sh +3 -3
  2. run-sailor.sh +3 -3
run-llama.sh CHANGED
@@ -1,7 +1,7 @@
1
  #!/bin/sh
2
 
3
 
4
- printf "Running meta-llama/Llama-3.2-1B-Instruct using vLLM OpenAI compatible API Server at port %s\n" "7860"
5
 
6
  # Llama-3.2-3B-Instruct max context length is 131072, but we reduce it to 32k.
7
  # 32k tokens, 3/4 of 32k is 24k words, each page average is 500 or 0.5k words,
@@ -19,8 +19,8 @@ printf "Running meta-llama/Llama-3.2-1B-Instruct using vLLM OpenAI compatible AP
19
  # params with the sail/Sailor-1.8B-Chat, I use the
20
  # meta-llama/Llama-3.2-1B-Instruct rev 9213176726f574b556790deb65791e0c5aa438b6
21
  python -u /app/openai_compatible_api_server.py \
22
- --model meta-llama/Llama-3.2-1B-Instruct \
23
- --revision 9213176726f574b556790deb65791e0c5aa438b6 \
24
  --host 0.0.0.0 \
25
  --port 7860 \
26
  --max-num-batched-tokens 32768 \
 
1
  #!/bin/sh
2
 
3
 
4
+ printf "Running meta-llama/Llama-3.2-3B-Instruct using vLLM OpenAI compatible API Server at port %s\n" "7860"
5
 
6
  # Llama-3.2-3B-Instruct max context length is 131072, but we reduce it to 32k.
7
  # 32k tokens, 3/4 of 32k is 24k words, each page average is 500 or 0.5k words,
 
19
  # params with the sail/Sailor-1.8B-Chat, I use the
20
  # meta-llama/Llama-3.2-1B-Instruct rev 9213176726f574b556790deb65791e0c5aa438b6
21
  python -u /app/openai_compatible_api_server.py \
22
+ --model meta-llama/Llama-3.2-3B-Instruct \
23
+ --revision 0cb88a4f764b7a12671c53f0838cd831a0843b95 \
24
  --host 0.0.0.0 \
25
  --port 7860 \
26
  --max-num-batched-tokens 32768 \
run-sailor.sh CHANGED
@@ -1,7 +1,7 @@
1
  #!/bin/sh
2
 
3
 
4
- printf "Running sail/Sailor-1.8B-Chat using vLLM OpenAI compatible API Server at port %s\n" "7860"
5
 
6
  # Using the 1.8B version because the 4B version cannot be loaded due to memory constraints.
7
  # Reducing max-num-batched-tokens to 7536 because got this error:
@@ -11,8 +11,8 @@ printf "Running sail/Sailor-1.8B-Chat using vLLM OpenAI compatible API Server at
11
  # INFO 11-27 15:32:10 gpu_executor.py:117] Maximum concurrency for 32768 tokens per request: 0.23x
12
  # ERROR 11-27 15:32:10 engine.py:366] The model's max seq len (32768) is larger than the maximum number of tokens that can be stored in KV cache (7536). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine.
13
  python -u /app/openai_compatible_api_server.py \
14
- --model sail/Sailor-1.8B-Chat \
15
- --revision 04b86803d4011d4bfd80f3cd3841b005eb899987 \
16
  --host 0.0.0.0 \
17
  --port 7860 \
18
  --max-num-batched-tokens 32768 \
 
1
  #!/bin/sh
2
 
3
 
4
+ printf "Running sail/Sailor-4B-Chat using vLLM OpenAI compatible API Server at port %s\n" "7860"
5
 
6
  # Using the 1.8B version because the 4B version cannot be loaded due to memory constraints.
7
  # Reducing max-num-batched-tokens to 7536 because got this error:
 
11
  # INFO 11-27 15:32:10 gpu_executor.py:117] Maximum concurrency for 32768 tokens per request: 0.23x
12
  # ERROR 11-27 15:32:10 engine.py:366] The model's max seq len (32768) is larger than the maximum number of tokens that can be stored in KV cache (7536). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine.
13
  python -u /app/openai_compatible_api_server.py \
14
+ --model sail/Sailor-4B-Chat \
15
+ --revision 89a866a7041e6ec023dd462adeca8e28dd53c83e \
16
  --host 0.0.0.0 \
17
  --port 7860 \
18
  --max-num-batched-tokens 32768 \