Repository: huggingface/lighteval
Issue: [BUG] Cannot use custom tasks with pipeline
Root cause: Configuration Error

## Describe the bug
Seeing:
```
custom|aime_2024 not found in provided tasks
[rank0]: Traceback (most recent call last):
[rank0]:   File "/workspace/verl_trainer/lighteval/pipeline.py", line 54, in <module>
[rank0]:     main()
[rank0]:   File "/workspace/verl_trainer/lighteval/pipeline.py", line 42, in main
[rank0]:     pipeline = Pipeline(
[rank0]:                ^^^^^^^^^
[rank0]:   File "/root/anaconda3/envs/eval/lib/python3.11/site-packages/lighteval/pipeline.py", line 152, in __init__
[rank0]:     self._init_tasks_and_requests(tasks=tasks)
[rank0]:   File "/root/anaconda3/envs/eval/lib/python3.11/site-packages/lighteval/pipeline.py", line 211, in _init_tasks_and_requests
[rank0]:     task_dict = registry.get_task_dict(task_names_list)
[rank0]:                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/root/anaconda3/envs/eval/lib/python3.11/site-packages/lighteval/tasks/registry.py", line 212, in get_task_dict
[rank0]:     return {task_name: self.get_task_instance(task_name) for task_name in task_names}
[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/root/anaconda3/envs/eval/lib/python3.11/site-packages/lighteval/tasks/registry.py", line 212, in <dictcomp>
[rank0]:     return {task_name: self.get_task_instance(task_name) for task_name in task_names}
[rank0]:                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/root/anaconda3/envs/eval/lib/python3.11/site-packages/lighteval/tasks/registry.py", line 109, in get_task_instance
[rank0]:     raise ValueError(f"Cannot find tasks {task_name} in task list or in custom task registry)")
[rank0]: ValueError: Cannot find tasks custom|aime_2024 in task list or in custom task registry)
```

## To Reproduce
```python
import lighteval
from lighteval.logging.evaluation_tracker import EvaluationTracker
from lighteval.models.vllm.vllm_model import VLLMModelConfig
from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters
from lighteval.utils.utils import EnvConfig
from lighteval.utils.imports import is_accelerate_available
from datetime import timedelta

if is_accelerate_available():
    from accelerate import Accelerator, InitProcessGroupKwargs
    accelerator = Accelerator(kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=3000))])
else:
    accelerator = None

MODEL = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"

def main():
    evaluation_tracker = EvaluationTracker(
        output_dir=f"./data/evals/{MODEL}",
        save_details=True,
        push_to_hub=False,
    )

    pipeline_params = PipelineParameters(
        launcher_type=ParallelismManager.ACCELERATE,
        env_config=EnvConfig(cache_dir="tmp/"),
        # Remove the 2 parameters below once your configuration is tested
        override_batch_size=1,
        max_samples=10,
        custom_tasks_directory="./tasks/custom_tasks.py", # if using a custom task
    )

    model_config = VLLMModelConfig(
            pretrained=MODEL,
            dtype="bfloat16",
            use_chat_template=True,
    )

    # task = "helm|mmlu|0|0"
    task = "custom_tasks|aime_2024|0|0"

    pipeline = Pipeline(
        tasks=task,
        pipeline_parameters=pipeline_params,
        evaluation_tracker=evaluation_tracker,
        model_config=model_config,
    )

    pipeline.evaluate()
    pipeline.save_and_push_results()
    pipeline.show_results()

if __name__ == "__main__":
    main()
```
using unchanged tasks defs from open-r1:
https://github.com/huggingface/open-r1/blob/main/src/open_r1/evaluate.py

## Expected behavior
.

## Version info
```
# pip freeze
absl-py==2.1.0
accelerate==1.3.0
aenum==3.1.15
aiohappyeyeballs==2.4.6
aiohttp==3.11.12
aiohttp-cors==0.7.0
aiosignal==1.3.2
airportsdata==20241001
annotated-types==0.7.0
antlr4-python3-runtime==4.13.2
anyio==4.8.0
astor==0.8.1
attrs==25.1.0
blake3==1.0.4
blis==0.7.11
cachetools==5.5.1
catalogue==2.0.10
certifi==2025.1.31
chardet==5.2.0
charset-normalizer==3.4.1
click==8.1.8
cloudpathlib==0.16.0
cloudpickle==3.1.1
colorama==0.4.6
colorful==0.5.6
colorlog==6.9.0
compressed-tensors==0.9.1
confection==0.1.5
cymem==2.0.11
DataProperty==1.1.0
datasets==3.3.0
depyf==0.18.0
dill==0.3.8
diskcache==5.6.3
distlib==0.3.9
distro==1.9.0
einops==0.8.1
fastapi==0.115.8
filelock==3.17.0
frozenlist==1.5.0
fsspec==2024.12.0
gguf==0.10.0
gitdb==4.0.12
GitPython==3.1.44
google-api-core==2.24.1
google-auth==2.38.0
googleapis-common-protos==1.67.0
grpcio==1.70.0
h11==0.14.0
httpcore==1.0.7
httptools==0.6.4
httpx==0.28.1
huggingface-hub==0.28.1
idna==3.10
importlib_metadata==8.6.1
iniconfig==2.0.0
interegular==0.3.3
Jinja2==3.1.5
jiter==0.8.2
joblib==1.4.2
jsonschema==4.23.0
jsonschema-specifications==2024.10.1
langcodes==3.5.0
language_data==1.3.0
lark==1.2.2
latex2sympy2_extended==1.0.6
lighteval @ git+https://github.com/huggingface/lighteval.git@86f62259f105ae164f655e0b91c92a823a742724
lm-format-enforcer==0.10.9
lxml==5.3.1
marisa-trie==1.2.1
markdown-it-py==3.0.0
MarkupSafe==3.0.2
math-verify==0.5.2
mbstrdecoder==1.1.4
mdurl==0.1.2
mistral_common==1.5.3
mpmath==1.3.0
msgpack==1.1.0
msgspec==0.19.0
multidict==6.1.0
multiprocess==0.70.16
murmurhash==1.0.12
nest-asyncio==1.6.0
networkx==3.4.2
nltk==3.9.1
numpy==1.26.4
nvidia-cublas-cu12==12.4.5.8
nvidia-cuda-cupti-cu12==12.4.127
nvidia-cuda-nvrtc-cu12==12.4.127
nvidia-cuda-runtime-cu12==12.4.127
nvidia-cudnn-cu12==9.1.0.70
nvidia-cufft-cu12==11.2.1.3
nvidia-curand-cu12==10.3.5.147
nvidia-cusolver-cu12==11.6.1.9
nvidia-cusparse-cu12==12.3.1.170
nvidia-ml-py==12.570.86
nvidia-nccl-cu12==2.21.5
nvidia-nvjitlink-cu12==12.4.127
nvidia-nvtx-cu12==12.4.127
openai==1.63.0
opencensus==0.11.4
opencensus-context==0.1.3
opencv-python-headless==4.11.0.86
outlines==0.1.11
outlines_core==0.1.26
packaging==24.2
pandas==2.2.3
partial-json-parser==0.2.1.1.post5
pathvalidate==3.2.3
pillow==11.1.0
platformdirs==4.3.6
pluggy==1.5.0
portalocker==3.1.1
preshed==3.0.9
prometheus-fastapi-instrumentator==7.0.2
prometheus_client==0.21.1
propcache==0.2.1
proto-plus==1.26.0
protobuf==3.20.3
psutil==7.0.0
py-cpuinfo==9.0.0
py-spy==0.4.0
pyarrow==19.0.0
pyasn1==0.6.1
pyasn1_modules==0.4.1
pybind11==2.13.6
pycountry==24.6.1
pydantic==2.10.6
pydantic_core==2.27.2
Pygments==2.19.1
pytablewriter==1.2.1
pytest==8.3.4
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
pytz==2025.1
PyYAML==6.0.2
pyzmq==26.2.1
ray==2.42.1
referencing==0.36.2
regex==2024.11.6
requests==2.32.3
rich==13.9.4
rouge_score==0.1.2
rpds-py==0.22.3
rsa==4.9
sacrebleu==2.5.1
safetensors==0.5.2
scikit-learn==1.6.1
scipy==1.15.1
sentencepiece==0.2.0
six==1.17.0
smart-open==6.4.0
smmap==5.0.2
sniffio==1.3.1
spacy==3.7.2
spacy-legacy==3.0.12
spacy-loggers==1.0.5
srsly==2.5.1
starlette==0.45.3
sympy==1.13.1
tabledata==1.3.4
tabulate==0.9.0
tcolorpy==0.1.7
termcolor==2.3.0
thinc==8.2.5
threadpoolctl==3.5.0
tiktoken==0.9.0
tokenizers==0.21.0
torch==2.5.1
torchaudio==2.5.1
torchvision==0.20.1
tqdm==4.67.1
transformers==4.48.3
triton==3.1.0
typepy==1.3.4
typer==0.9.4
typing_extensions==4.12.2
tzdata==2025.1
urllib3==2.3.0
uvicorn==0.34.0
uvloop==0.21.0
virtualenv==20.29.2
vllm==0.7.2
wasabi==1.1.3
watchfiles==1.0.4
weasel==0.3.4
websockets==14.2
wrapt==1.17.2
xformers==0.0.28.post3
xgrammar==0.1.11
xxhash==3.5.0
yarl==1.18.3
zipp==3.21.0
```