Repository: stanford-crfm/helm
Issue: Helm-run command issue
Root cause: Validation Gap

I have followed the installation steps, when I am entering command "helm-run --conf-paths run_specs.conf --suite v1 --max-eval-instances 10" I am getting the error mentioned below. Please help

main {
} [0.04s]
pyparsing.exceptions.ParseException: Expected {: ... | {{'=' | ':' | '+='} - [Suppress:({{{'#' | '//'} - SkipTo:({Suppress:(W:(
)) | StringEnd})} | Suppress:(W:(
))})]... - ConcatenatedValueParser:([{{{{{Suppress:({[Suppress:(W:(
,))] {'#' | '//'} - SkipTo:({Suppress:(W:(
)) | StringEnd})}) | {Suppress:('include') {Re:('"(?:[^"\\\n]|\\.)*"[ \t]*') | {{'url' | 'file' | 'package'} - Suppress:('(') - Re:('"(?:[^"\\\n]|\\.)*"[ \t]*') - Suppress:(')')} | {'required' - Suppress:('(') - {Re:('"(?:[^"\\\n]|\\.)*"[ \t]*') | {{'url' | 'file' | 'package'} - Suppress:('(') - Re:('"(?:[^"\\\n]|\\.)*"[ \t]*') - Suppress:(')')}} - Suppress:(')')}}} | Re:('[ \t]*\$\{[^\}]+\}[ \t]*')} | : ...} | Forward: {{{{Suppress:('[') -} ListParser:({{ConcatenatedValueParser:([{{{{{Suppress:({[Suppress:(W:(
,))] {'#' | '//'} - SkipTo:({Suppress:(W:(
)) | StringEnd})}) | {Suppress:('include') {Re:('"(?:[^"\\\n]|\\.)*"[ \t]*') | {{'url' | 'file' | 'package'} - Suppress:('(') - Re:('"(?:[^"\\\n]|\\.)*"[ \t]*') - Suppress:(')')} | {'required' - Suppress:('(') - {Re:('"(?:[^"\\\n]|\\.)*"[ \t]*') | {{'url' | 'file' | 'package'} - Suppress:('(') - Re:('"(?:[^"\\\n]|\\.)*"[ \t]*') - Suppress:(')')}} - Suppress:(')')}}} | Re:('[ \t]*\$\{[^\}]+\}[ \t]*')} | : ...} | : ...} | {{{{{{{W:(0-9) Suppress:([<SP>]...)} {'ns' ^ 'nano' ^ 'nanos' ^ 'nanosecond' ^ 'nanoseconds' ^ 'us' ^ 'micro' ^ 'micros' ^ 'microsecond' ^ 'microseconds' ^ 'ms' ^ 'milli' ^ 'millis' ^ 'millisecond' ^ 'milliseconds' ^ 's' ^ 'second' ^ 'seconds' ^ 'm' ^ 'minute' ^ 'minutes' ^ 'h' ^ 'hour' ^ 'hours' ^ 'w' ^ 
'week' ^ 'weeks' ^ 'd' ^ 'day' ^ 'days' ^ 'mo' ^ 'month' ^ 'months' ^ 'y' ^ 'year' ^ 'years'} Suppress:(WordEnd)} | Re:('[+-]?(\d*\.} | {{{{{{{W:(0-9) Suppress:([<SP>]...)} {'ns' ^ 'nano' ^ 'nanos' ^ 'nanosecond' ^ 'nanoseconds' ^ 'us' ^ 'micro' ^ 'micros' ^ 'microsecond' ^ 'microseconds' ^ 
'ms' ^ 'milli' ^ 'millis' ^ 'millisecond' ^ 'milliseconds' ^ 's' ^ 'second' ^ 'seconds' ^ 'm' ^ 'minute' ^ 'minutes' ^ 'h' ^ 'hour' ^ 'hours' ^ 'w' ^ 'week' ^ 'weeks' ^ 'd' ^ 'day' ^ 'days' ^ 'mo' ^ 'month' ^ 'months' ^ 'y' ^ 'year' ^ 'years'} Suppress:(WordEnd)} | Re:('[+-]?(\d*\.\d+|\d+(\.\d+)?)([eE][+\-]?\d+)?(?=$|[ \t]*([\$\}\],#\n\r]|//))')} | 'true'} | 'false'} | 'null'} | {{Re:('""".*?"*"""') | Re:('"(?:[^"\\\n]|\\.)*"[ \t]*')} | Re:('(?:[^^`+?!@*&"\[\{\s\]\}#,=\$\\]|\\.)+[ \t]*')}}} | Suppress:({{'\\' -} Suppress:(W:(
))})}]...)}}, found '\x00'  (at char 3), (line:1, col:4)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\HP\AppData\Local\Programs\Python\Python38\lib\runpy.py", line 194, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "C:\Users\HP\AppData\Local\Programs\Python\Python38\lib\runpy.py", line 87, in _run_code
    exec(code, run_globals)
  File "C:\Users\HP\Desktop\helm\helm-venv\Scripts\helm-run.exe\__main__.py", line 7, in <module>
    sys.exit(main())
  File "C:\Users\HP\Desktop\helm\helm-venv\lib\site-packages\helm\common\hierarchical_logger.py", line 104, in wrapper
    return fn(*args, **kwargs)
  File "C:\Users\HP\Desktop\helm\helm-venv\lib\site-packages\helm\benchmark\run.py", line 282, in main
    run_entries.extend(read_run_entries(args.conf_paths).entries)
  File "C:\Users\HP\Desktop\helm\helm-venv\lib\site-packages\helm\benchmark\presentation\run_entry.py", line 37, in read_run_entries
  File "C:\Users\HP\Desktop\helm\helm-venv\lib\site-packages\helm\common\general.py", line 45, in parse_hocon
    return pyhocon.ConfigFactory.parse_string(text)
  File "C:\Users\HP\Desktop\helm\helm-venv\lib\site-packages\pyhocon\config_parser.py", line 192, in parse_string
    return ConfigParser().parse(content, basedir, resolve, unresolved_value)
  File "C:\Users\HP\Desktop\helm\helm-venv\lib\site-packages\pyhocon\config_parser.py", line 455, in parse
    config = config_expr.parseString(content, parseAll=True)[0]
  File "C:\Users\HP\Desktop\helm\helm-venv\lib\site-packages\pyparsing\util.py", line 256, in _inner
    return fn(self, *args, **kwargs)
  File "C:\Users\HP\Desktop\helm\helm-venv\lib\site-packages\pyparsing\core.py", line 1197, in parse_string
    raise exc.with_traceback(None)
pyparsing.exceptions.ParseSyntaxException: , found '\x00'  (at char 3), (line:1, col:4)

Hi, the error is telling you that your `run_specs.conf` file is malformed. Specifically, it contains a null byte. If you upload this file somewhere, I could take a look at it.

Additionally, HELM is not _officially_ supported on Windows. I generally recommend running it within WSL2 instead, if you are on a Windows machine.

Hi, thank you for your reply. I have created virtual environment and all the packages were successfully installed. I am sharing the snapshot of  run_specs.conf file and the command I am using to run it. 

run_specs.conf
<img width="563" alt="run_specs" src="https://github.com/stanford-crfm/helm/assets/74587767/2ca732d3-b7e5-4afe-9aa0-85c6df9c7da4">

Command used
helm-run --conf-paths run_specs.conf --suite v1 --max-eval-instances 10

Could you try running directly using `--run-specs`:

```
helm-run --run-specs mmlu:subject=anatomy, model=huggingface/gpt2 --suite v1 --max-eval-instances 10
helm-run --run-specs mmlu:subject=philosophy, model=huggingface/gpt2 --suite v1 --max-eval-instances 10
```

@sahil18930691 I hope the previous suggestion was helpful. I am closing this issue due to inactivity. Feel free to reopen if you have followup questions.