Repository: huggingface/transformers
Pull request: Enable non-streaming mode in `transformers serve`

Needs this to be merged first: https://github.com/huggingface/transformers/pull/41444

Tests and docs need to be added before undraft

Changed files:
filename: src/transformers/commands/serving.py
patch: @@ -26,7 +26,7 @@
 import time
 import uuid
 from argparse import ArgumentParser, Namespace
-from collections.abc import AsyncGenerator, Generator, Iterable
+from collections.abc import Generator, Iterable
 from contextlib import asynccontextmanager
 from dataclasses import dataclass, field
 from io import BytesIO
@@ -35,6 +35,7 @@
 
 from huggingface_hub import model_info
 from huggingface_hub.constants import HF_HUB_OFFLINE
+from openai.types.chat.chat_completion import Choice
 from tokenizers.decoders import DecodeStream
 
 import transformers
@@ -90,14 +91,16 @@
     from fastapi.responses import JSONResponse, StreamingResponse
     from openai.types.audio.transcription import Transcription
     from openai.types.audio.transcription_create_params import TranscriptionCreateParamsBase
-    from openai.types.chat import ChatCompletionMessageParam
+    from openai.types.chat import ChatCompletion, ChatCompletionMessage, ChatCompletionMessageParam
     from openai.types.chat.chat_completion_chunk import (
         ChatCompletionChunk,
-        Choice,
         ChoiceDelta,
         ChoiceDeltaToolCall,
         ChoiceDeltaToolCallFunction,
     )
+    from openai.types.chat.chat_completion_chunk import (
+        Choice as ChoiceChunk,
+    )
     from openai.types.chat.completion_create_params import CompletionCreateParamsStreaming
     from openai.types.responses import (
         Response,
@@ -345,8 +348,11 @@ def delete_model(self):
             self._timer.cancel()
 
     def timeout_reached(self):
-        self.delete_model()
-        logger.info(f"{self._name_or_path} was removed from memory after {self.timeout_seconds} seconds of inactivity")
+        if self.timeout_seconds > 0:
+            self.delete_model()
+            logger.info(
+                f"{self._name_or_path} was removed from memory after {self.timeout_seconds} seconds of inactivity"
+            )
 
     def is_deleted(self):
         """Check if the instances have been deleted."""
@@ -412,9 +418,13 @@ class ServeArguments:
     # Serving settings
     host: str = field(default="localhost", metadata={"help": "Interface the server will listen to."})
     port: int = field(default=8000, metadata={"help": "Port the server will listen to."})
-    model_timeout: int = field(
-        default=300,
-        metadata={"help": "Time in seconds after which a model will be removed from memory."},
+    model_timeout: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": "Time in seconds after which a model will be removed from memory; defaults to 300 unless "
+            "`force_model` is set, in which case the model will not be removed from memory unless a value"
+            "is specified here."
+        },
     )
 
     # Other settings
@@ -512,6 +522,14 @@ def __init__(self, args: ServeArguments):
         self.last_kv_cache = None
         self.last_model = None
 
+        if self.args.model_timeout is None:
+            self.args.model_timeout = -1 if self.args.force_model else 300
+
+        if self.args.force_model:
+            model_id_and_revision = self.process_model_name(self.args.force_model)
+            self.last_model = model_id_and_revision
+            self.load_model_and_processor(model_id_and_revision)
+
     def _validate_request(
         self,
         request: dict,
@@ -595,7 +613,7 @@ def build_chat_completion_chunk(
         tool_calls: Optional[list["ChoiceDeltaToolCall"]] = None,
         decode_stream: Optional[DecodeStream] = None,
         tokenizer: Optional[PreTrainedTokenizerFast] = None,
-    ) -> str:
+    ) -> ChatCompletionChunk:
         """
         Builds a chunk of a streaming OpenAI Chat Completion response.
 
@@ -621,12 +639,13 @@ def build_chat_completion_chunk(
         """
         if decode_stream is not None and content is not None and tokenizer is not None:
             content = decode_stream.step(tokenizer._tokenizer, content)
+
         chunk = ChatCompletionChunk(
             id=request_id,
             created=int(time.time()),
             model=model,
             choices=[
-                Choice(
+                ChoiceChunk(
                     delta=ChoiceDelta(
                         content=content,
                         role=role,
@@ -639,23 +658,25 @@ def build_chat_completion_chunk(
             system_fingerprint="",
             object="chat.completion.chunk",
         )
-        return f"data: {chunk.model_dump_json(exclude_none=True)}\n\n"
 
-    def build_response_event(self, response: "BaseModel") -> str:
+        return chunk
+
+    @staticmethod
+    def chunk_to_sse_element(chunk: ChatCompletionChunk | BaseModel) -> str:
         """
-        Builds a event of a streaming OpenAI Response response.
+        Builds an event of a streaming OpenAI Response model or a ChatCompletion chunk.
 
         IMPORTANT: The serialized chunk won't contain empty fields (fields with `None`). Some downstream apps,
         like Cursor, assume that when the field exists, it has data.
 
         Args:
-            response (`BaseModel`):
+            chunk (`BaseModel` or `ChatCompletionChunk`):
                 The response to build an event from. One of the multiple OpenAI Response output types
 
         Returns:
             `str`: The built chunk, a string containing a JSON string with the payload.
         """
-        return f"data: {response.model_dump_json(exclude_none=True)}\n\n"
+        return f"data: {chunk.model_dump_json(exclude_none=True)}\n\n"
 
     def run(self):
         """
@@ -668,6 +689,7 @@ def run(self):
         - POST /v1/responses: Generates responses.
         - POST /v1/audio/transcriptions: Generates transcriptions from audio.
         - GET /v1/models: Lists available models for 3rd party tools.
+        - GET /health: Health check.
 
         Requires FastAPI and Uvicorn to be installed.
         """
@@ -703,10 +725,9 @@ def chat_completion(request: Request, body: dict):
             self.validate_chat_completion_request(request=body)
 
             if self.use_continuous_batching:
-                output = self.continuous_batching_chat_completion(body, request.state.request_id)
+                return self.continuous_batching_chat_completion(body, request.state.request_id)
             else:
-                output = self.generate_chat_completion(body)
-            return StreamingResponse(output, media_type="text/event-stream")
+                return self.generate_chat_completion(body)
 
         @app.post("/v1/responses")
         def responses(request: dict):
@@ -803,7 +824,7 @@ def get_gen_models(self) -> list[dict[str, any]]:
                 for model in model_infos
             ]
 
-    def continuous_batching_chat_completion(self, req: dict, request_id: str) -> AsyncGenerator[str, None]:
+    def continuous_batching_chat_completion(self, req: dict, request_id: str) -> StreamingResponse | JSONResponse:
         """
         Generates an OpenAI Chat Completion using continuous batching.
 
@@ -816,14 +837,16 @@ def continuous_batching_chat_completion(self, req: dict, request_id: str) -> Asy
 
         model_id_and_revision = self.process_model_name(req["model"])
         must_discard_cache = model_id_and_revision != self.last_model
+
         self.last_model = model_id_and_revision
+
+        # When switching models, terminate a continuous batching manager if it is running.
         if must_discard_cache:
-            # When switching models, terminate a continuous batching manager if it is running.
             if self.running_continuous_batching_manager is not None:
                 self.running_continuous_batching_manager.stop(block=True, timeout=2)
                 self.running_continuous_batching_manager = None
-        model, processor = self.load_model_and_processor(model_id_and_revision)
 
+        model, processor = self.load_model_and_processor(model_id_and_revision)
         tokenizer = processor.tokenizer if hasattr(processor, "tokenizer") else processor
 
         generation_config = create_generation_config_from_req(
@@ -838,18 +861,17 @@ def continuous_batching_chat_completion(self, req: dict, request_id: str) -> Asy
 
         if self.running_continuous_batching_manager is None:
             self.running_continuous_batching_manager = model.init_continuous_batching(
-                generation_config=generation_config, streaming=True
+                generation_config=generation_config
             )
 
-            # TODO (Joao, Lysandre): the logits processors should be fixed in continuous batching
-            # and correctly applied in non-cb
+            # TODO (Joao, Lysandre): the logits processors should be fixed in continuous batching and correctly applied in non-cb
             self.running_continuous_batching_manager.logit_processor = LogitsProcessorList()
             self.running_continuous_batching_manager.start()
 
         # TODO (Joao, Lysandre): this should also work with tool support
         inputs = processor.apply_chat_template(req["messages"], return_tensors="pt", add_generation_prompt=True).to(
             model.device
-        )
+        )[0]
 
         def stream_chat_completion(request_id, decode_stream):
             try:
@@ -879,21 +901,61 @@ def stream_chat_completion(request_id, decode_stream):
                 self.running_continuous_batching_manager.cancel_request(request_id)
                 yield f'data: {{"error": "{str(e)}"}}'
 
-        async def cancellation_wrapper(_inputs, request_id):
+        def buffer_chat_completion(_request_id):
+            result = None
+            while self.running_continuous_batching_manager.is_running() and result is None:
+                result = self.running_continuous_batching_manager.get_result(request_id=_request_id, timeout=1)
+
+            content = tokenizer.decode(result.generated_tokens)
+
+            chat_completion_result = ChatCompletion(
+                id=_request_id,
+                created=int(time.time()),
+                object="chat.completion",
+                model=model_id_and_revision,
+                choices=[
+                    Choice(
+                        # TODO check the index
+                        index=0,
+                        message=ChatCompletionMessage(content=content, role="assistant"),
+                        finish_reason="stop",
+                    )
+                ],
+                # TODO implement function calling
+                # TODO implement usage
+            )
+
+            return chat_completion_result
+
+        async def cancellation_wrapper_stream(_request_id):
+            # Enables cancellation in an async context
             try:
-                decode_stream = DecodeStream(_inputs.tolist(), False)
-                # XXX: using returned request_id as safety in case it is None
-                request_id = self.running_continuous_batching_manager.add_request(
-                    _inputs, request_id=request_id, max_new_tokens=generation_config.max_new_tokens
-                )
-                for chunk in stream_chat_completion(request_id, decode_stream):
-                    yield chunk
-                    await asyncio.sleep(0)  # Yield control to the event loop to check for cancellations
+                decode_stream = DecodeStream(inputs.tolist(), False)
+                for _chunk in stream_chat_completion(_request_id, decode_stream):
+                    yield self.chunk_to_sse_element(_chunk)
+                    await asyncio.sleep(0)
             except asyncio.CancelledError:
-                self.running_continuous_batching_manager.cancel_request(request_id)
-                logger.warning(f"Request {request_id} was cancelled.")
+                self.running_continuous_batching_manager.cancel_request(_request_id)
+                logger.warning(f"Request {_request_id} was cancelled.")
 
-        return cancellation_wrapper(inputs[0], request_id)
+        def cancellation_wrapper_buffer(_request_id):
+            # Enables cancellation in an async context
+            try:
+                return buffer_chat_completion(_request_id)
+            except asyncio.CancelledError:
+                self.running_continuous_batching_manager.cancel_request(_request_id)
+                logger.warning(f"Request {_request_id} was cancelled.")
+
+        request_id = self.running_continuous_batching_manager.add_request(
+            inputs, request_id=request_id, max_new_tokens=generation_config.max_new_tokens, streaming=req.get("stream")
+        )
+
+        if req.get("stream"):
+            return StreamingResponse(cancellation_wrapper_stream(request_id), media_type="text/event-stream")
+        else:
+            chunk = cancellation_wrapper_buffer(request_id)
+            json_chunk = chunk.model_dump_json(exclude_none=True)
+            return JSONResponse(json_chunk, media_type="application/json")
 
     @staticmethod
     def get_model_modality(model: "PreTrainedModel") -> Modality:
@@ -953,7 +1015,7 @@ def get_processor_inputs_from_inbound_messages(messages, modality: Modality):
             processor_inputs.append(parsed_message)
         return processor_inputs
 
-    def generate_chat_completion(self, req: dict) -> Generator[str, None, None]:
+    def generate_chat_completion(self, req: dict) -> StreamingResponse | JSONResponse:
         """
         Generates an OpenAI Chat Completion using `generate`.
 
@@ -1132,7 +1194,10 @@ def generate_with_cache(**kwargs):
                                 )
 
                             yield self.build_chat_completion_chunk(
-                                request_id=_request_id, role=None, tool_calls=[tool], model=model_id_and_revision
+                                request_id=_request_id,
+                                role=None,
+                                tool_calls=[tool],
+                                model=model_id_and_revision,
                             )
                             continue
                     # ====== END OF TOOL CALL LOGIC ======
@@ -1152,7 +1217,47 @@ def generate_with_cache(**kwargs):
             finally:
                 thread.join()
 
-        return stream_chat_completion(generation_streamer, request_id)
+        if req.get("stream"):
+            return StreamingResponse(
+                map(self.chunk_to_sse_element, stream_chat_completion(generation_streamer, request_id)),
+                media_type="text/event-stream",
+            )
+        else:
+            content = []
+            finish_reason = "stop"
+
+            generator = stream_chat_completion(generation_streamer, request_id)
+            usage = None
+
+            for chunk in generator:
+                choice = chunk.choices[0]
+                if getattr(choice.delta, "content", None):
+                    content.append(choice.delta.content)
+                if choice.finish_reason:
+                    finish_reason = choice.finish_reason
+                if getattr(chunk, "usage", None):
+                    usage = chunk.usage
+
+            chat_completion_result = ChatCompletion(
+                id=request_id,
+                created=int(time.time()),
+                object="chat.completion",
+                model=model_id_and_revision,
+                choices=[
+                    Choice(
+                        # TODO check the index
+                        index=0,
+                        message=ChatCompletionMessage(content="".join(content), role="assistant"),
+                        finish_reason=finish_reason,
+                    )
+                ],
+                # TODO implement function calling
+                usage=usage,
+            )
+
+            result = chat_completion_result.model_dump(exclude_none=True)
+
+            return JSONResponse(result, media_type="application/json")
 
     def generate_response(self, req: dict) -> Generator[str, None, None]:
         """
@@ -1263,7 +1368,7 @@ def generate_with_cache(**kwargs):
                     ),
                 )
                 sequence_number += 1
-                yield self.build_response_event(response_created)
+                yield self.chunk_to_sse_element(response_created)
 
                 response_in_progress = ResponseInProgressEvent(
                     type="response.in_progress",
@@ -1284,7 +1389,7 @@ def generate_with_cache(**kwargs):
                     ),
                 )
                 sequence_number += 1
-                yield self.build_response_event(response_in_progress)
+                yield self.chunk_to_sse_element(response_in_progress)
 
                 # Start the output item. Emit the assistant role to start the stream. Other chunks won't have a role,
                 # as it is implicit
@@ -1297,7 +1402,7 @@ def generate_with_cache(**kwargs):
                     ),
                 )
                 sequence_number += 1
-                yield self.build_response_event(response_output_item_added)
+                yield self.chunk_to_sse_element(response_output_item_added)
 
                 # Start the content part of the event
                 response_content_part_added = ResponseContentPartAddedEvent(
@@ -1309,7 +1414,7 @@ def generate_with_cache(**kwargs):
                     part=ResponseOutputText(type="output_text", text="", annotations=[]),
                 )
                 sequence_number += 1
-                yield self.build_response_event(response_content_part_added)
+                yield self.chunk_to_sse_element(response_content_part_added)
 
                 # Stream the actual generated text
                 results = ""
@@ -1336,7 +1441,7 @@ def generate_with_cache(**kwargs):
                                 logprobs=[],
                             )
                             sequence_number += 1
-                            yield self.build_response_event(response_output_text_delta)
+                            yield self.chunk_to_sse_element(response_output_text_delta)
                     else:
                         # Normal path: emit token deltas when not filtering CoT
                         if result:
@@ -1350,7 +1455,7 @@ def generate_with_cache(**kwargs):
                                 logprobs=[],
                             )
                             sequence_number += 1
-                            yield self.build_response_event(response_output_text_delta)
+                            yield self.chunk_to_sse_element(response_output_text_delta)
 
                 # Signal the end of the text generation
                 response_output_text_done = ResponseTextDoneEvent(
@@ -1363,7 +1468,7 @@ def generate_with_cache(**kwargs):
                     logprobs=[],
                 )
                 sequence_number += 1
-                yield self.build_response_event(response_output_text_done)
+                yield self.chunk_to_sse_element(response_output_text_done)
 
                 # Complete the content part
                 response_content_part_done = ResponseContentPartDoneEvent(
@@ -1376,7 +1481,7 @@ def generate_with_cache(**kwargs):
                 )
                 sequence_number += 1
                 content_index += 1
-                yield self.build_response_event(response_content_part_done)
+                yield self.chunk_to_sse_element(response_content_part_done)
 
                 # Complete the output item
                 response_output_item_done = ResponseOutputItemDoneEvent(
@@ -1394,7 +1499,7 @@ def generate_with_cache(**kwargs):
                 )
                 sequence_number += 1
                 output_index += 1
-                yield self.build_response_event(response_output_item_done)
+                yield self.chunk_to_sse_element(response_output_item_done)
 
                 # Finally, Complete the event
                 response_completed = ResponseCompletedEvent(
@@ -1416,7 +1521,7 @@ def generate_with_cache(**kwargs):
                     ),
                 )
                 sequence_number += 1
-                yield self.build_response_event(response_completed)
+                yield self.chunk_to_sse_element(response_completed)
 
                 thread.join()
             except Exception as e:
@@ -1427,7 +1532,7 @@ def generate_with_cache(**kwargs):
                     message=str(e),
                 )
                 sequence_number += 1
-                yield self.build_response_event(error_event)
+                yield self.chunk_to_sse_element(error_event)
 
                 response_failed = ResponseFailedEvent(
                     type="response.failed",
@@ -1452,7 +1557,7 @@ def generate_with_cache(**kwargs):
                     ),
                 )
                 sequence_number += 1
-                yield self.build_response_event(response_failed)
+                yield self.chunk_to_sse_element(response_failed)
 
             finally:
                 thread.join()

filename: src/transformers/generation/continuous_batching/continuous_api.py
patch: @@ -907,7 +907,6 @@ def get_result(
             if request_id is not None and result.request_id != request_id:
                 self.output_queue.put(result)
                 return None
-            logger.debug(f"Retrieved result for request {result.request_id}")
             return result
         except queue.Empty:
             return None

filename: src/transformers/modeling_utils.py
patch: @@ -2510,14 +2510,14 @@ def _check_and_adjust_attn_implementation(
             try:
                 load_and_register_attn_kernel(applicable_attn_implementation)
                 # log that we used kernel fallback if successful
-                if attn_implementation.startswith("flash_attention"):
+                if "flash_" in attn_implementation:
                     logger.warning_once(
                         f"You do not have `flash_attn` installed, using `{applicable_attn_implementation}` "
                         "from the `kernels` library instead!"
                     )
             except Exception as e:
                 # raise the proper exception for requested flash attention
-                if attn_implementation.startswith("flash_attention"):
+                if attn_implementation.startswith("flash_"):
                     if attn_implementation.endswith("2"):
                         self._flash_attn_2_can_dispatch()
                     else:
@@ -2530,7 +2530,7 @@ def _check_and_adjust_attn_implementation(
                 applicable_attn_implementation, is_init_check
             )
             # preload flash attention here to allow compile with fullgraph
-            if applicable_attn_implementation.startswith("flash_attention"):
+            if applicable_attn_implementation.startswith("flash_"):
                 lazy_import_flash_attention(applicable_attn_implementation, force_import=True)
         return applicable_attn_implementation
 

filename: tests/commands/test_serving.py
patch: @@ -85,6 +85,7 @@ def test_build_chat_completion_chunk(self):
         chunk = ServeCommand.build_chat_completion_chunk(
             dummy, request_id="req0", content="hello", finish_reason="stop", role="user", model="dummy_model@main"
         )
+        chunk = ServeCommand.chunk_to_sse_element(chunk)
         for field in MANDATORY_FIELDS:
             self.assertIn(field, chunk)
         self.assertIn(
@@ -93,12 +94,14 @@ def test_build_chat_completion_chunk(self):
 
         # Case 2: only the role is provided -- other fields in 'choices' are omitted
         chunk = dummy.build_chat_completion_chunk(request_id="req0", role="user", model="dummy_model@main")
+        chunk = ServeCommand.chunk_to_sse_element(chunk)
         for field in MANDATORY_FIELDS:
             self.assertIn(field, chunk)
         self.assertIn('"choices":[{"delta":{"role":"user"},"index":0}]', chunk)
 
         # Case 3: only the content is provided -- other fields in 'choices' are omitted
         chunk = dummy.build_chat_completion_chunk(request_id="req0", content="hello", model="dummy_model@main")
+        chunk = ServeCommand.chunk_to_sse_element(chunk)
         for field in MANDATORY_FIELDS:
             self.assertIn(field, chunk)
         self.assertIn('"choices":[{"delta":{"content":"hello"},"index":0}]', chunk)
@@ -110,6 +113,7 @@ def test_build_chat_completion_chunk(self):
             type="function",
         )
         chunk = dummy.build_chat_completion_chunk(request_id="req0", tool_calls=[tool_call], model="dummy_model@main")
+        chunk = ServeCommand.chunk_to_sse_element(chunk)
         for field in MANDATORY_FIELDS:
             self.assertIn(field, chunk)
         expected_choices_content = (
@@ -147,7 +151,7 @@ def test_build_response_event(self):
             ),
         )
 
-        event = dummy.build_response_event(response_created)
+        event = dummy.chunk_to_sse_element(response_created)
         self.assertTrue(event.startswith("data: "))  # Sanity check: event formatting
         self.assertIn('"model":"dummy_model@main"', event)  # Sanity check: set field
         self.assertIn('"status":"queued"', event)
@@ -411,10 +415,18 @@ def setUpClass(cls):
         """Starts a server for tests to connect to."""
         cls.port = 8001
         args = ServeArguments(port=cls.port)
-        serve_command = ServeCommand(args)
-        thread = Thread(target=serve_command.run)
-        thread.daemon = True
-        thread.start()
+        cls.serve_command = ServeCommand(args)
+        cls.thread = Thread(target=cls.serve_command.run)
+        cls.thread.daemon = True
+        cls.thread.start()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.thread.join(timeout=1)
+
+    def setUp(self):
+        """Ensures that the healthcheck works before each test."""
+        _call_healthcheck(f"http://localhost:{self.port}")
 
     @slow
     def test_tool_call(self):
@@ -548,13 +560,19 @@ class ServeCompletionsContinuousBatchingIntegrationTest(ServeCompletionsMixin, u
     def setUpClass(cls):
         """Starts a server for tests to connect to."""
         cls.port = 8002
-        args = ServeArguments(
-            port=cls.port, continuous_batching=True, attn_implementation="sdpa_paged", default_seed=42
-        )
+        args = ServeArguments(port=cls.port, continuous_batching=True, default_seed=42)
         cls.serve_command = ServeCommand(args)
-        thread = Thread(target=cls.serve_command.run)
-        thread.daemon = True
-        thread.start()
+        cls.thread = Thread(target=cls.serve_command.run)
+        cls.thread.daemon = True
+        cls.thread.start()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.thread.join(timeout=1)
+
+    def setUp(self):
+        """Ensures that the healthcheck works before each test."""
+        _call_healthcheck(f"http://localhost:{self.port}")
 
     def test_full_request(self):
         """Tests that an inference using the Responses API and Continuous Batching works"""
@@ -703,9 +721,17 @@ def setUpClass(cls):
         cls.port = 8003
         args = ServeArguments(port=cls.port, default_seed=42)
         serve_command = ServeCommand(args)
-        thread = Thread(target=serve_command.run)
-        thread.daemon = True
-        thread.start()
+        cls.thread = Thread(target=serve_command.run)
+        cls.thread.daemon = True
+        cls.thread.start()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.thread.join(timeout=1)
+
+    def setUp(self):
+        """Ensures that the healthcheck works before each test."""
+        _call_healthcheck(f"http://localhost:{self.port}")
 
     @slow
     def test_full_request(self):
@@ -767,9 +793,17 @@ def setUpClass(cls):
         cls.port = 8042
         args = ServeArguments(port=cls.port)
         serve_command = ServeCommand(args)
-        thread = Thread(target=serve_command.run)
-        thread.daemon = True
-        thread.start()
+        cls.thread = Thread(target=serve_command.run)
+        cls.thread.daemon = True
+        cls.thread.start()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.thread.join(timeout=1)
+
+    def setUp(self):
+        """Ensures that the healthcheck works before each test."""
+        _call_healthcheck(f"http://localhost:{self.port}")
 
     def test_healthcheck(self):
         """Tests that the healthcheck endpoint works."""