Project Name: EleutherAI+gpt-neox
Language: Python
Statement Type: Constant Assignment

Source Code:
def get_args(input_args=None):
    parser = argparse.ArgumentParser()
    group = parser.add_argument_group(title="input data")
    group.add_argument(
        "--input",
        type=str,
        required=True,
        help="Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated "
        "list",
    )
    group.add_argument(
        "--jsonl-keys",
        nargs="+",
        default=["text"],
        help="space separate listed of keys to extract from jsonl. Defa",
    )
    group.add_argument(
        "--num-docs",
        default=None,
        help="Optional: Number of documents in the input data (if known) for an accurate progress bar.",
        type=int,
    )
    group = parser.add_argument_group(title="tokenizer")
    group.add_argument(
        "--tokenizer-type",
        type=str,
        required=True,
        choices=[
            "HFGPT2Tokenizer",
            "HFTokenizer",
            "GPT2BPETokenizer",
            "CharLevelTokenizer",
            "TiktokenTokenizer",
            "SPMTokenizer",
        ],
        help="What type of tokenizer to use.",
    )
    group.add_argument(
        "--vocab-file", type=str, default=None, help="Path to the vocab file"
    )
    group.add_argument(
        "--merge-file",
        type=str,
        default=None,
        help="Path to the BPE merge file (if necessary).",
    )
    group.add_argument(
        "--append-eod",
        action="store_true",
        help="Append an <eod> token to the end of a document.",
    )
    group.add_argument("--ftfy", action="store_true", help="Use ftfy to clean text")
    group = parser.add_argument_group(title="output data")
    group.add_argument(
        "--output-prefix",
        type=str,
        required=True,
        help="Path to binary output file without suffix",
    )
    group.add_argument(
        "--dataset-impl",
        type=str,
        default="mmap",
        choices=["lazy", "cached", "mmap"],
        help="Dataset implementation to use. Default: mmap",
    )

    group = parser.add_argument_group(title="runtime")
    group.add_argument(
        "--workers", type=int, default=1, help="Number of worker processes to launch"
    )
    group.add_argument(
        "--log-interval",
        type=int,
        default=100,
        help="Interval between progress updates",
    )
    args = parser.parse_args(input_args)
    args.keep_empty = False

    # some default/dummy values for the tokenizer
    args.rank = 0
    args.make_vocab_size_divisible_by = 128
    args.model_parallel_size = 1

    return args

get_args(input_args=['--input', './tests/data/enwik8_first100.txt', '--output-prefix', './tests/data/enwik8_first100', '--vocab', 'gpt2', '--tokenizer-type', 'HFGPT2Tokenizer', '--merge-file', './data/gpt2-merges.txt', '--append-eod'])

Selected Statement: args.keep_empty = False

Function Input:
input_args: --input, ./tests/data/enwik8_first100.txt, --output-prefix, ./tests/data/enwik8_first100, --vocab, gpt2, --tokenizer-type, HFGPT2Tokenizer, --merge-file, ./data/gpt2-merges.txt, --append-eod

Variable Values Before Statement:
Constant: False

Value After Statement Execution: False

Variable States During Runtime (line, value):
input_args: 1
--input, ./tests/data/enwik8_first100.txt, --output-prefix, ./tests/data/enwik8_first100, --vocab, gpt2, --tokenizer-type, HFGPT2Tokenizer, --merge-file, ./data/gpt2-merges.txt, --append-eod
parser: 2
ArgumentParser(prog='__main__.py', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)
group: 3
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='input data', _group_actions=[]}
4
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='input data', _group_actions=[_StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None)]}
11
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='input data', _group_actions=[_StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None)]}
17
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='input data', _group_actions=[_StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None)]}
23
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='tokenizer', _group_actions=[]}
24
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='tokenizer', _group_actions=[_StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None)]}
38
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='tokenizer', _group_actions=[_StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None)]}
41
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='tokenizer', _group_actions=[_StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None)]}
47
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='tokenizer', _group_actions=[_StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None)]}
52
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), '--ftfy': _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='tokenizer', _group_actions=[_StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None)]}
53
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), '--ftfy': _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='output data', _group_actions=[]}
54
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), '--ftfy': _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), '--output-prefix': _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='output data', _group_actions=[_StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None)]}
60
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), '--ftfy': _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), '--output-prefix': _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), '--dataset-impl': _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='output data', _group_actions=[_StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None)]}
68
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), '--ftfy': _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), '--output-prefix': _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), '--dataset-impl': _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='runtime', _group_actions=[]}
69
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None), _StoreAction(option_strings=['--workers'], dest='workers', nargs=None, const=None, default=1, type=<class 'int'>, choices=None, required=False, help='Number of worker processes to launch', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), '--ftfy': _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), '--output-prefix': _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), '--dataset-impl': _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None), '--workers': _StoreAction(option_strings=['--workers'], dest='workers', nargs=None, const=None, default=1, type=<class 'int'>, choices=None, required=False, help='Number of worker processes to launch', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='runtime', _group_actions=[_StoreAction(option_strings=['--workers'], dest='workers', nargs=None, const=None, default=1, type=<class 'int'>, choices=None, required=False, help='Number of worker processes to launch', metavar=None)]}
72
{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None), _StoreAction(option_strings=['--workers'], dest='workers', nargs=None, const=None, default=1, type=<class 'int'>, choices=None, required=False, help='Number of worker processes to launch', metavar=None), _StoreAction(option_strings=['--log-interval'], dest='log_interval', nargs=None, const=None, default=100, type=<class 'int'>, choices=None, required=False, help='Interval between progress updates', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), '--ftfy': _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), '--output-prefix': _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), '--dataset-impl': _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None), '--workers': _StoreAction(option_strings=['--workers'], dest='workers', nargs=None, const=None, default=1, type=<class 'int'>, choices=None, required=False, help='Number of worker processes to launch', metavar=None), '--log-interval': _StoreAction(option_strings=['--log-interval'], dest='log_interval', nargs=None, const=None, default=100, type=<class 'int'>, choices=None, required=False, help='Interval between progress updates', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='runtime', _group_actions=[_StoreAction(option_strings=['--workers'], dest='workers', nargs=None, const=None, default=1, type=<class 'int'>, choices=None, required=False, help='Number of worker processes to launch', metavar=None), _StoreAction(option_strings=['--log-interval'], dest='log_interval', nargs=None, const=None, default=100, type=<class 'int'>, choices=None, required=False, help='Interval between progress updates', metavar=None)]}
args: 78
Namespace(input='./tests/data/enwik8_first100.txt', jsonl_keys=['text'], num_docs=None, tokenizer_type='HFGPT2Tokenizer', vocab_file='gpt2', merge_file='./data/gpt2-merges.txt', append_eod=True, ftfy=False, output_prefix='./tests/data/enwik8_first100', dataset_impl='mmap', workers=1, log_interval=100)
79
Namespace(input='./tests/data/enwik8_first100.txt', jsonl_keys=['text'], num_docs=None, tokenizer_type='HFGPT2Tokenizer', vocab_file='gpt2', merge_file='./data/gpt2-merges.txt', append_eod=True, ftfy=False, output_prefix='./tests/data/enwik8_first100', dataset_impl='mmap', workers=1, log_interval=100, keep_empty=False)
82
Namespace(input='./tests/data/enwik8_first100.txt', jsonl_keys=['text'], num_docs=None, tokenizer_type='HFGPT2Tokenizer', vocab_file='gpt2', merge_file='./data/gpt2-merges.txt', append_eod=True, ftfy=False, output_prefix='./tests/data/enwik8_first100', dataset_impl='mmap', workers=1, log_interval=100, keep_empty=False, rank=0)
83
Namespace(input='./tests/data/enwik8_first100.txt', jsonl_keys=['text'], num_docs=None, tokenizer_type='HFGPT2Tokenizer', vocab_file='gpt2', merge_file='./data/gpt2-merges.txt', append_eod=True, ftfy=False, output_prefix='./tests/data/enwik8_first100', dataset_impl='mmap', workers=1, log_interval=100, keep_empty=False, rank=0, make_vocab_size_divisible_by=128)
84
Namespace(input='./tests/data/enwik8_first100.txt', jsonl_keys=['text'], num_docs=None, tokenizer_type='HFGPT2Tokenizer', vocab_file='gpt2', merge_file='./data/gpt2-merges.txt', append_eod=True, ftfy=False, output_prefix='./tests/data/enwik8_first100', dataset_impl='mmap', workers=1, log_interval=100, keep_empty=False, rank=0, make_vocab_size_divisible_by=128, model_parallel_size=1)