TITLE_TEXT = "PECC - Problem Extraction and Coding Challenges Evaluation Benchmark" INTRODUCTION_TEXT = """📄 PECC: An extensive benchmark centered on code generation from narrative-embedded problem descriptions. Unlike prior benchmarks that evaluate code generation using specific instructions, our dataset requires models to comprehend, extract requirements, and produce the essential code for problem-solving. This approach necessitates syntactically accurate programs and demands reading comprehension skills to derive the desired solution.""" TASK_DESCRIPTION = """## Task Description The task for the model is to generate directly executable python code. ### Instruction The model is first prompted with a system prompt, which is a short description of the problem. The model is then asked to generate the python code that solves the problem. ### Task The model receives the task itself. """ CITATION_BUTTON_LABEL = "Copy the following snippet to cite our paper" CITATION_BUTTON_TEXT = r""" Coming soon! """ # @misc{pecc, # author = {Patrick Haller and Jonas Golde and Alan Akbik}, # title = {PECC - Problem Extraction and Coding Challenges}, # year = {2024}, # publisher = {GitHub}, # journal = {GitHub repository}, # howpublished = {} # }