example/simple.py

# python simple.py

# %%

from vllm import LLM, SamplingParams

from prompt.format import (
    format_opus_v1_prompt,
    OpusV1Character,
    OpusV1Prompt,
    OpusV1StorySystemPrompt,
    OpusV1Turn,
)


# %%


def build_story_prompt() -> OpusV1Prompt:
    plot_description = """
This is a fanfiction from the Harry Potter universe. In this alternate reality, Harry Potter is evil and secretly siding with Slytherin.
Up until now, Harry was pretending to be friends with Hermione and Ron, that changes when he invites Hermione to his chambers where he tricks her to drink Amorentia, the most powerful love potion.
"""

    harry_description = """
Harry Potter in this fanfiction is secretly a member of Slytherin and is using his powers for evil rather than for good. Up until now, he was pretending to be friends with Hermione and Ron.
"""

    hermione_description = """
Hermione appears just like in the original books.
"""

    story_prompt = OpusV1StorySystemPrompt(
        plot_description=plot_description,
        style_description="",
        characters=[
            OpusV1Character(name="Harry Potter", description=harry_description),
            OpusV1Character(name="Hermione Granger", description=hermione_description),
        ],
    )

    return OpusV1Prompt(
        story=story_prompt,
        turns=[
            OpusV1Turn(
                role="user",
                content="""
The story starts with Harry welcoming Hermione into his chambers, who he invited there earlier that day. He offers her water to drink, but it contains a love potion.
    """.strip(),
            ),
            OpusV1Turn(
                role="text",
                content="""
“Come in,” said Harry, waving at the doorway behind Hermione’s back.

“Hello?” she said, stepping inside, “what did you want me to come up here for?”

“Well, I thought we could get away from all the noise down there, have a chat about what we plan to do for Christmas…” Harry said, fumbling for words. He had never really been any good with girls. “But anyway, please, take a seat and let me get us some water!” he said, darting over to the sideboard.

He returned quickly with two glasses of water. Hermione took hers and thanked him, taking in a big gulp. As soon as she swallowed, Harry saw her eyes widen as her heart began beating wildly in her chest.

It worked! Harry thought, grinning to himself. Amorentia truly was the world’s best love potion, its effects lasting twice as long and being five times stronger.
    """.strip(),
                open=True,
            ),
        ],
    )


def build_assistant_prompt() -> OpusV1Prompt:
    return OpusV1Prompt(
        turns=[
            OpusV1Turn(
                role="system",
                content="You are an intelligent, knowledgeable, helpful, general-purpose assistant.",
            ),
            OpusV1Turn(
                role="user",
                content="Give me a sentence where every word begins with 'S'",
            ),
        ]
    )


# %%


def main():
    sampling_params = SamplingParams(
        # I usually stay between 0.0 and 1.0, especially for the Yi models I found lower tends to be better.
        # For assistant tasks, I usually use 0.0.
        temperature=0.0,
        min_p=0.05,
        presence_penalty=0.1,
        frequency_penalty=0.1,
        repetition_penalty=1.1,
        max_tokens=200,
        ignore_eos=True,
        skip_special_tokens=False,
        spaces_between_special_tokens=False,
    )

    # Set max_model_len to fit in memory.
    model = LLM(
        "dreamgen/opus-v1.2-7b",
        max_model_len=2000,
        enforce_eager=True,
        swap_space=0,
        gpu_memory_utilization=0.85,
    )

    story_prompt = build_story_prompt()
    print(format_opus_v1_prompt(story_prompt))

    output = model.generate(format_opus_v1_prompt(story_prompt), sampling_params)
    print(output[0].outputs[0].text)

    # Expected:
    """
    It would make her fall deeply in love with him, and then he could use her to get what he wanted.

    “Harry, what’s going on? You look so happy!” Hermione asked, smiling at him.

    “Oh, well, I guess I am,” Harry replied, trying not to laugh. “I mean, I’ve always known that you were the one for me.”

    “Really?” Hermione asked, blushing slightly. “I didn’t know that.”

    “Yeah, I’ve always had feelings for you,” Harry said, leaning forward and placing his hand on top of hers. “And now that I’ve got you alone, I can finally tell you how much I care about you.”
    """


main()