Repository: LAION-AI/Open-Assistant
File: data/datasets/poetry_instruction/prepare.py (Python)

Code before:

# Shuffle the rows in the DataFrame.
df = df.sample(frac=1)

# Split the DataFrame into two DataFrames, one for titling tasks and one for writing tasks.
writing_tasks = df.iloc[:num_writing_tasks]
titling_tasks = df.iloc[num_writing_tasks:]

prepared_data = []

# Loop through the writing tasks and process them.
for index, row in writing_tasks.iterrows():
    # Get data from the entry
    poem = row["Poem"]
    topics = row["Tags"]
    title = row["Title"]
    author = row["Poet"]

    # Variables to store to instruction, reply, source, and metadata.
    instruction = random.choice(writing_prompts_topic).replace("$topic", str(topics))
    reply = random.choice(replies_topic).replace("$topic", str(topics)).replace("$title", title).replace("$poem", poem)
    source = "PoetryFoundation.org" + " - " + author
    metadata = {"author": author, "title": title, "tags": str(topics), "task_type": "writing"}

    # If the entry has "nan" as a topic, change the prompts to the non-topic ones.
    if " nan" in instruction:
        instruction = random.choice(writing_prompts_notTopic)
        reply = random.choice(replies_notTopic).replace("$title", title).replace("$poem", poem)

    entry = {"INSTRUCTION": instruction, "RESPONSE": reply, "SOURCE": source, "METADATA": json.dumps(metadata)}
    prepared_data.append(entry)

# Loop through the titling tasks and process them.
for index, row in titling_tasks.iterrows():
    # Get data from the entry
    poem = row["Poem"]
    topics = row["Tags"]
    title = row["Title"]
    author = row["Poet"]

    # Variables to store to instruction, reply, source, and metadata.
    instruction = random.choice(titling_prompts).replace("$poem", poem)
    reply = random.choice(titling_replies).replace("$title", title)
    source = "PoetryFoundation.org" + " - " + author
    metadata = {"author": author, "title": title, "tags": str(topics), "task_type": "titling"}

    # If the entry has a topic, use the topic to generate a writing prompt and a reply.

    entry = {"INSTRUCTION": instruction, "RESPONSE": reply, "SOURCE": source, "METADATA": json.dumps(metadata)}
    prepared_data.append(entry)


Review comment (suggestion):
please use `pd.isna(instruction)` or something similar, the "nan" is an artifact from converting NaN (missing/None) to string

Code after:
]

# Shuffling the dataset and delegating 5% to titling tasks.
# Calculating the number of titling tasks and writing tasks.
num_rows = len(df)
num_titling_tasks = int(num_rows * 0.05)
num_writing_tasks = num_rows - num_titling_tasks

# Shuffle the rows in the DataFrame.
df = df.sample(frac=1)

# Split the DataFrame into two DataFrames, one for titling tasks and one for writing tasks.
writing_tasks = df.iloc[:num_writing_tasks]
titling_tasks = df.iloc[num_writing_tasks:]

prepared_data = []

# Loop through the writing tasks and process them.
for index, row in writing_tasks.iterrows():
    # Get data from the entry
    poem = row["Poem"]
    topics = row["Tags"]
    title = row["Title"]
    author = row["Poet"]

    # Variables to store to instruction, reply, source, and metadata.
    instruction = random.choice(writing_prompts_topic).replace("$topic", str(topics))
    reply = random.choice(replies_topic).replace("$topic", str(topics)).replace("$title", title).replace("$poem", poem)
    source = "PoetryFoundation.org" + " - " + author
    metadata = {"author": author, "title": title, "tags": str(topics), "task_type": "writing"}

    # If the entry has an empty value for the topic, use the non-topic prompts and replies.
    if pd.isna(topics):
        instruction = random.choice(writing_prompts_notTopic)
        reply = random.choice(replies_notTopic).replace("$title", title).replace("$poem", poem)

    # Create a dictionary entry for the entry and append it to the list.
    entry = {"INSTRUCTION": instruction, "RESPONSE": reply, "SOURCE": source, "METADATA": json.dumps(metadata)}
    prepared_data.append(entry)

# Loop through the titling tasks and process them.
for index, row in titling_tasks.iterrows():
    # Get data from the entry
    poem = row["Poem"]
    topics = row["Tags"]
    title = row["Title"]
    author = row["Poet"]

    # Variables to store to instruction, reply, source, and metadata.
    instruction = random.choice(titling_prompts).replace("$poem", poem)
    reply = random.choice(titling_replies).replace("$title", title)