Jae-Won Chung commited on
Commit
9d117bb
1 Parent(s): f0ad70f

Switch to MPT

Browse files
.gitignore CHANGED
@@ -3,6 +3,7 @@
3
 
4
  # Environment
5
  .envrc
 
6
 
7
  # Editor
8
  pyrightconfig.json
 
3
 
4
  # Environment
5
  .envrc
6
+ .env
7
 
8
  # Editor
9
  pyrightconfig.json
deployment/docker-compose-0.yaml CHANGED
@@ -1,8 +1,8 @@
1
  services:
2
- Falcon-7B:
3
  container_name: worker0
4
  image: mlenergy/tgi:latest
5
- command: ["--model-id", "tiiuae/falcon-7b-instruct", "--num-shard", "1", "--otlp-endpoint", "http://jaeger:4317"]
6
  shm_size: 1g
7
  networks:
8
  - leaderboard
 
1
  services:
2
+ MPT-7B:
3
  container_name: worker0
4
  image: mlenergy/tgi:latest
5
+ command: ["--model-id", "mosaicml/mpt-7b-chat", "--num-shard", "1", "--otlp-endpoint", "http://jaeger:4317"]
6
  shm_size: 1g
7
  networks:
8
  - leaderboard
deployment/docker-compose-1.yaml CHANGED
@@ -5,7 +5,7 @@ services:
5
  command: ["--model-id", "meta-llama/Llama-2-70b-chat-hf", "--num-shard", "2", "--otlp-endpoint", "http://jaeger:4317", "--quantize", "bitsandbytes"]
6
  shm_size: 1g
7
  environment:
8
- HUGGING_FACE_HUB_TOKEN: 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
9
  networks:
10
  - leaderboard
11
  volumes:
@@ -19,10 +19,10 @@ services:
19
  - driver: nvidia
20
  device_ids: ["0", "1"]
21
  capabilities: [gpu]
22
- Falcon-40B:
23
  container_name: worker5
24
  image: mlenergy/tgi:latest
25
- command: ["--model-id", "tiiuae/falcon-40b-instruct", "--num-shard", "2", "--otlp-endpoint", "http://jaeger:4317"]
26
  shm_size: 1g
27
  networks:
28
  - leaderboard
 
5
  command: ["--model-id", "meta-llama/Llama-2-70b-chat-hf", "--num-shard", "2", "--otlp-endpoint", "http://jaeger:4317", "--quantize", "bitsandbytes"]
6
  shm_size: 1g
7
  environment:
8
+ HUGGING_FACE_HUB_TOKEN: "${HF_TOKEN}"
9
  networks:
10
  - leaderboard
11
  volumes:
 
19
  - driver: nvidia
20
  device_ids: ["0", "1"]
21
  capabilities: [gpu]
22
+ MPT-30B:
23
  container_name: worker5
24
  image: mlenergy/tgi:latest
25
+ command: ["--model-id", "mosaicml/mpt-30b-chat", "--num-shard", "2", "--otlp-endpoint", "http://jaeger:4317"]
26
  shm_size: 1g
27
  networks:
28
  - leaderboard