Repository: oumi-ai/oumi
File: src/oumi/core/inference/base_inference_engine.py (Python)

Code before:
        p99 = histogram.get_value_at_percentile(99)
        logger.debug(
            f"{self.__class__.__name__}: "
            f"Latency Histogram: {total_count} samples recorded:"
            f"\tp50: {p50:.1f}ms\tp90: {p90:.1f}ms\tp99: {p99:.1f}ms"
        )

    def _read_conversations(self, input_filepath: str) -> list[Conversation]:
        """Reads conversations from a file in Oumi chat format.

        Args:
            input_filepath: The path to the file containing the conversations.

        Returns:
            List[Conversation]: A list of conversations read from the file.
        """
        conversations = []
        with open(input_filepath) as f:
            for line in f:
                # Only parse non-empty lines.
                if line.strip():
                    conversation = Conversation.from_json(line)
                    conversations.append(conversation)
        return conversations

    def _get_scratch_filepath(self, output_filepath: Union[str, None]) -> str:
        """Returns a scratch filepath for the given output filepath.

        For example, if the output filepath is "/foo/bar/output.json", the scratch
        filepath will be "/foo/bar/scratch/output.json"

        If no output filepath is provided, a temporary file is used and placed in the
        current working directory under the name "tmp/temp_inference_output.jsonl".

        Args:
            output_filepath: The output filepath.

        Returns:
            str: The scratch filepath.
        """
        if output_filepath is not None:
            original_filepath = Path(output_filepath)
            return str(original_filepath.parent / "scratch" / original_filepath.name)

        return "tmp/temp_inference_output.jsonl"

    def _save_conversation_to_scratch(
        self, conversation: Conversation, output_filepath: Union[str, None]
    ) -> None:
        """Appends a conversation to a file in Oumi chat format.


Review comment (suggestion):
Nit, here and elsewhere: I'd recommend using `Optional[str]` over `Union[str, None]`. Optional is syntactic sugar for the latter, and it's a bit more concise. 

Code after:
        p99 = histogram.get_value_at_percentile(99)
        logger.debug(
            f"{self.__class__.__name__}: "
            f"Latency Histogram: {total_count} samples recorded:"
            f"\tp50: {p50:.1f}ms\tp90: {p90:.1f}ms\tp99: {p99:.1f}ms"
        )

    def _read_conversations(self, input_filepath: str) -> list[Conversation]:
        """Reads conversations from a file in Oumi chat format.

        Args:
            input_filepath: The path to the file containing the conversations.

        Returns:
            List[Conversation]: A list of conversations read from the file.
        """
        conversations = []
        with open(input_filepath) as f:
            for line in f:
                # Only parse non-empty lines.
                if line.strip():
                    conversation = Conversation.from_json(line)
                    conversations.append(conversation)
        return conversations

    def _get_scratch_filepath(self, output_filepath: Optional[str]) -> str:
        """Returns a scratch filepath for the given output filepath.

        For example, if the output filepath is "/foo/bar/output.json", the scratch
        filepath will be "/foo/bar/scratch/output.json"

        If no output filepath is provided, a temporary file is used and placed in the
        current working directory under the name "tmp/temp_inference_output.jsonl".

        Args:
            output_filepath: The output filepath.

        Returns:
            str: The scratch filepath.
        """
        if output_filepath is not None:
            original_filepath = Path(output_filepath)
            return str(original_filepath.parent / "scratch" / original_filepath.name)

        return str(Path.cwd() / "tmp" / "temp_inference_output.jsonl")

    def _save_conversation_to_scratch(
        self, conversation: Conversation, output_filepath: Optional[str]
    ) -> None:
        """Appends a conversation to a file in Oumi chat format.