Repository: xorbitsai/inference
Pull request: ENH: Support Enviroment Variable
ATT

This implementation is capable to:

If the user didn't type in the -e or --endpoint when required:
    The program will search if user have exported an endpoint into OS environ. If so, the endpoint variable in the program 
    will automatically be set to that endpoint in OS. If not, we will give it the default endpoint back, which is 
    http://127.0.0.1:9997. 
if the user did have an input at the -e parameters, we keep the users' input to the endpoint variable. 



Problem statement:
ENH: client supports env variable XINFERENCE_ENDPOINT
### Is your feature request related to a problem? Please describe
It could be troublesome when starting xinference with a custom port. You have to specify the endpoint every time when you use xinference CLI.

### Describe the solution you'd like
I hope we can optimize user experience by using an env variable XINFERENCE_ENDPOINT.

When you starts an xinference service, an endpoint will be printed. Then in the terminal for xinference CLI, execute `export XINFERENCE_ENDPOINT=${endpoint}`.

Xinference CLI should firstly check if `-e` is specified. If not, check the env var XINFERENCE_ENDPOINT. If neither of them is set, use the default endpoint.

Hints from issue discussion:

Commit message: ADD the Enviroment Variable Enhancement

Patch to review:
diff --git a/xinference/constants.py b/xinference/constants.py
index 18c606a831..070d6c7510 100644
--- a/xinference/constants.py
+++ b/xinference/constants.py
@@ -22,3 +22,4 @@
 XINFERENCE_DEFAULT_LOCAL_HOST = "127.0.0.1"
 XINFERENCE_DEFAULT_DISTRIBUTED_HOST = "0.0.0.0"
 XINFERENCE_DEFAULT_ENDPOINT_PORT = 9997
+XINFERENCE_ENV_ENDPOINT_VARIABLE = "XINFERENCE_ENDPOINT"
diff --git a/xinference/deploy/cmdline.py b/xinference/deploy/cmdline.py
index 2c7915903b..ee7712e9d3 100644
--- a/xinference/deploy/cmdline.py
+++ b/xinference/deploy/cmdline.py
@@ -14,6 +14,8 @@
 
 
 import logging
+import os
+from typing import Optional
 
 import click
 from xoscar.utils import get_next_port
@@ -24,9 +26,22 @@
     XINFERENCE_DEFAULT_DISTRIBUTED_HOST,
     XINFERENCE_DEFAULT_ENDPOINT_PORT,
     XINFERENCE_DEFAULT_LOCAL_HOST,
+    XINFERENCE_ENV_ENDPOINT_VARIABLE,
 )
 
 
+def getExistingEndpointFromEnv(endpoint: Optional[str]) -> str:
+    # user didn't specify the endpoint.
+    if endpoint is None:
+        if XINFERENCE_ENV_ENDPOINT_VARIABLE in os.environ:
+            return os.environ[XINFERENCE_ENV_ENDPOINT_VARIABLE]
+        else:
+            default_endpoint = f"http://{XINFERENCE_DEFAULT_LOCAL_HOST}:{XINFERENCE_DEFAULT_ENDPOINT_PORT}"
+            return default_endpoint
+    else:
+        return endpoint
+
+
 @click.group(invoke_without_command=True, name="xinference")
 @click.pass_context
 @click.version_option(__version__, "--version", "-v")
@@ -81,17 +96,18 @@ def supervisor(
 @click.option(
     "--endpoint",
     "-e",
-    default=f"http://{XINFERENCE_DEFAULT_LOCAL_HOST}:{XINFERENCE_DEFAULT_ENDPOINT_PORT}",
     type=str,
 )
 @click.option("--host", "-H", default=XINFERENCE_DEFAULT_DISTRIBUTED_HOST, type=str)
-def worker(log_level: str, endpoint: str, host: str):
+def worker(log_level: str, endpoint: Optional[str], host: str):
     from ..deploy.worker import main
 
     if log_level:
         logging.basicConfig(level=logging.getLevelName(log_level.upper()))
     logging_conf = dict(level=log_level.upper())
 
+    endpoint = getExistingEndpointFromEnv(endpoint)
+
     client = RESTfulClient(base_url=endpoint)
     supervisor_internal_addr = client._get_supervisor_internal_address()
 
@@ -107,7 +123,6 @@ def worker(log_level: str, endpoint: str, host: str):
 @click.option(
     "--endpoint",
     "-e",
-    default=f"http://{XINFERENCE_DEFAULT_LOCAL_HOST}:{XINFERENCE_DEFAULT_ENDPOINT_PORT}",
     type=str,
 )
 @click.option("--model-name", "-n", type=str)
@@ -121,6 +136,8 @@ def model_launch(
     model_format: str,
     quantization: str,
 ):
+    endpoint = getExistingEndpointFromEnv(endpoint)
+
     client = RESTfulClient(base_url=endpoint)
     model_uid = client.launch_model(
         model_name=model_name,
@@ -136,7 +153,6 @@ def model_launch(
 @click.option(
     "--endpoint",
     "-e",
-    default=f"http://{XINFERENCE_DEFAULT_LOCAL_HOST}:{XINFERENCE_DEFAULT_ENDPOINT_PORT}",
     type=str,
 )
 @click.option("--all", is_flag=True)
@@ -148,6 +164,8 @@ def model_list(endpoint: str, all: bool):
     # TODO: get from the supervisor
     from ..model.llm import LLM_FAMILIES
 
+    endpoint = getExistingEndpointFromEnv(endpoint)
+
     table = []
     if all:
         for model_family in LLM_FAMILIES:
@@ -195,7 +213,6 @@ def model_list(endpoint: str, all: bool):
 @click.option(
     "--endpoint",
     "-e",
-    default=f"http://{XINFERENCE_DEFAULT_LOCAL_HOST}:{XINFERENCE_DEFAULT_ENDPOINT_PORT}",
     type=str,
 )
 @click.option("--model-uid", type=str)
@@ -203,6 +220,8 @@ def model_terminate(
     endpoint: str,
     model_uid: str,
 ):
+    endpoint = getExistingEndpointFromEnv(endpoint)
+
     client = RESTfulClient(base_url=endpoint)
     client.terminate_model(model_uid=model_uid)
 


Code review comments:
diff_hunk: @@ -22,3 +22,4 @@
 XINFERENCE_DEFAULT_LOCAL_HOST = "127.0.0.1"
 XINFERENCE_DEFAULT_DISTRIBUTED_HOST = "0.0.0.0"
 XINFERENCE_DEFAULT_ENDPOINT_PORT = 9997
+XINFERENCE_ENV_ENDPOINT_VARIABLE = "XINFERENCE_ENDPOINT"
path: xinference/constants.py
text: @user1:
`XINFERENCE_ENV_ENDPOINT_VARIABLE` -> `XINFERENCE_ENV_ENDPOINT`

diff_hunk: @@ -24,9 +26,22 @@
     XINFERENCE_DEFAULT_DISTRIBUTED_HOST,
     XINFERENCE_DEFAULT_ENDPOINT_PORT,
     XINFERENCE_DEFAULT_LOCAL_HOST,
+    XINFERENCE_ENV_ENDPOINT_VARIABLE,
 )
 
 
+def getExistingEndpointFromEnv(endpoint: Optional[str]) -> str:
path: xinference/deploy/cmdline.py
text: @user1:
`getExistingEndpointFromEnv` -> `get_endpoint`