deployment/docker-compose-1.yaml

services:
  Llama2-70B-INT8:
    container_name: worker4
    image: mlenergy/tgi:latest
    command: ["--model-id", "meta-llama/Llama-2-70b-chat-hf", "--num-shard", "2", "--otlp-endpoint", "http://jaeger:4317", "--quantize", "bitsandbytes"]
    shm_size: 1g
    environment:
      HUGGING_FACE_HUB_TOKEN: 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
    networks:
      - leaderboard
    volumes:
      - /data/leaderboard/tgi-data:/data
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["0", "1"]
              capabilities: [gpu]
  Falcon-40B:
    container_name: worker5
    image: mlenergy/tgi:latest
    command: ["--model-id", "tiiuae/falcon-40b-instruct", "--num-shard", "2", "--otlp-endpoint", "http://jaeger:4317"]
    shm_size: 1g
    networks:
      - leaderboard
    volumes:
      - /data/leaderboard/tgi-data:/data
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["2", "3"]
              capabilities: [gpu]

networks:
  leaderboard:
    name: leaderboard
    external: true