Repository: huggingface/datasets
File: src/datasets/commands/convert_to_parquet.py (Python)

Code before:
    def __init__(
        self,
        dataset_id: str,
        token: Optional[str],
        revision: Optional[str],
    ):
        self._dataset_id = dataset_id
        self._token = token
        self._revision = revision

    def run(self) -> None:
        dataset_id = self._dataset_id
        token = self._token
        revision = self._revision
        print(f"{dataset_id}")
        configs = get_dataset_config_names(dataset_id, token=token, revision=revision)
        print(f"{configs = }")
        default_config = get_dataset_default_config_name(dataset_id, token=token, revision=revision)
        print(f"{default_config = }")
        if default_config:
            config = default_config
            configs.remove(default_config)
        else:
            config = configs.pop(0)
        print(f"{config = }")
        dataset = load_dataset(dataset_id, config, revision=revision)
        commit_info = dataset.push_to_hub(
            dataset_id,
            config_name=config,
            commit_message="Convert dataset to Parquet",
            commit_description="Convert dataset to Parquet.",
            create_pr=True,
            token=token,
            set_default=default_config is not None,
        )
        time.sleep(5)
        if commit_info:
            pr_revision, pr_url = commit_info.pr_revision, commit_info.pr_url
        else:
            pr_revision, pr_url = infer_pr(dataset_id, token=token)
        for config in configs:
            print(f"{config = }")
            dataset = load_dataset(dataset_id, config, revision=revision)
            dataset.push_to_hub(
                dataset_id,
                config_name=config,
                commit_message=f"Add {config} data files",
                revision=pr_revision,
                token=token,
            )
            time.sleep(5)

Review comment (suggestion):
Maybe add `trust_remote_code` ? And maybe a `--trust_remote_code` arg to the cli command

(also in get_dataset_config_names and get_dataset_default_config_name)

Code after:
            "--trust_remote_code", action="store_true", help="whether to trust the code execution of the load script"
        )
        parser.set_defaults(func=_command_factory)

    def __init__(
        self,
        dataset_id: str,
        token: Optional[str],
        revision: Optional[str],
        trust_remote_code: bool,
    ):
        self._dataset_id = dataset_id
        self._token = token
        self._revision = revision
        self._trust_remote_code = trust_remote_code

    def run(self) -> None:
        dataset_id = self._dataset_id
        token = self._token
        revision = self._revision
        trust_remote_code = self._trust_remote_code
        print(f"{dataset_id}")
        configs = get_dataset_config_names(
            dataset_id, token=token, revision=revision, trust_remote_code=trust_remote_code
        )
        print(f"{configs = }")
        default_config = get_dataset_default_config_name(
            dataset_id, token=token, revision=revision, trust_remote_code=trust_remote_code
        )
        print(f"{default_config = }")
        if default_config:
            config = default_config
            configs.remove(default_config)
        else:
            config = configs.pop(0)
        print(f"{config = }")
        dataset = load_dataset(dataset_id, config, revision=revision, trust_remote_code=trust_remote_code)
        commit_info = dataset.push_to_hub(
            dataset_id,
            config_name=config,
            commit_message="Convert dataset to Parquet",
            commit_description="Convert dataset to Parquet.",
            create_pr=True,
            token=token,
            set_default=default_config is not None,
        )
        time.sleep(5)
        if commit_info:
            pr_revision, pr_url = commit_info.pr_revision, commit_info.pr_url
        else:
            pr_revision, pr_url = infer_pr(dataset_id, token=token)