added mupliple models and revisions

LLaVA

@@ -1 +1 @@
-Subproject commit 3e83206ab58f79936da2742c85c93dfd3890451c
+Subproject commit 3c2f6ba15ed0477f4149fd582d2b640e19da2a57

app.py

@@ -25,7 +25,7 @@ def start_controller():
     return subprocess.Popen(controller_command)
 
 
-def start_worker(model_path: str, bits=16):
+def start_worker(model_path: str, bits=16, revision='main', port=21002):
     print(f"Starting the model worker for the model {model_path}")
     model_name = model_path.strip("/").split("/")[-1]
     assert bits in [4, 8, 16], "It can be only loaded with 16-bit, 8-bit, and 4-bit."
@@ -37,6 +37,10 @@ def start_worker(model_path: str, bits=16):
         "llava.serve.model_worker",
         "--host",
         "0.0.0.0",
+        "--port",
+        port,
+        "--worker-address",
+        f"http://127.0.0.1:{port}",
         "--controller",
         "http://localhost:10000",
         "--model-path",
@@ -44,6 +48,8 @@ def start_worker(model_path: str, bits=16):
         "--model-name",
         model_name,
         "--use-flash-attn",
+        "--revision",
+        revision
     ]
     if bits != 16:
         worker_command += [f"--load-{bits}bit"]
@@ -77,12 +83,21 @@ Set the environment variable `model` to change the model:
 
     print(f"args: {gws.args}")
 
-    model_path = os.getenv("model", "liuhaotian/llava-v1.6-mistral-7b")
+    model_paths = os.getenv("model", "nvandal/LLaVA-Med-v1.5-7b")
+    revisions = os.getenv("revision", "main")
     bits = int(os.getenv("bits", 4))
     concurrency_count = int(os.getenv("concurrency_count", 5))
 
     controller_proc = start_controller()
-    worker_proc = start_worker(model_path, bits=bits)
+    start_worker_port = 21002
+
+    model_paths = model_paths.split(';')
+    revisions = revisions.split(';')
+    assert(len(model_paths)==len(revisions))
+    worker_proc = [None]*len(model_paths)
+    for i, (model_path, revision) in enumerate(zip(model_paths,revisions)):
+        print(model_path, revision)
+        worker_proc[i] = start_worker(model_path, bits=bits, revision=revision, port=str(start_worker_port+i))
 
     # Wait for worker and controller to start
     time.sleep(10)
@@ -103,7 +118,8 @@ Set the environment variable `model` to change the model:
         print(e)
         exit_status = 1
     finally:
-        worker_proc.kill()
+        for w in worker_proc:
+            w.kill()
         controller_proc.kill()
 
         sys.exit(exit_status)