Spaces:
Runtime error
Runtime error
| import pandas as pd | |
| import yaml | |
| import os | |
| def process_data(split="train"): | |
| with open("params.yml") as f: | |
| params = yaml.safe_load(f) | |
| df = pd.read_csv("data/raw/{}.csv".format(split)) | |
| df.columns = ["Unnamed: 0", "input_text", "output_text"] | |
| df = df.sample(frac=params["split"], replace=True, random_state=1) | |
| if os.path.exists("data/raw/{}.csv".format(split)): | |
| os.remove("data/raw/{}.csv".format(split)) | |
| df.to_csv("data/processed/{}.csv".format(split)) | |
| if __name__ == "__main__": | |
| process_data(split="train") | |
| process_data(split="test") | |
| process_data(split="validation") | |