Repository: huggingface/datasets
File: src/datasets/load.py (Python)

Code before:
            dataset_infos=dataset_infos,
            builder_configs_parameters=BuilderConfigsParameters(
                metadata_configs=metadata_configs,
                builder_configs=builder_configs,
                default_config_name=default_config_name,
            ),
        )


class HubDatasetModuleFactoryWithParquetExport(_DatasetModuleFactory):
    """
    Get the module of a dataset loaded from parquet files of a dataset repository parquet export.
    """

    def __init__(
        self,
        name: str,
        from_sha: Optional[str] = None,
        download_config: Optional[DownloadConfig] = None,
    ):
        self.name = name
        self.from_sha = from_sha
        self.download_config = download_config or DownloadConfig()
        increase_load_count(name, resource_type="dataset")

    def _get_exported_parquet_files(self) -> List[Dict[str, Any]]:
        datasets_server_parquet_url = config.HF_ENDPOINT.replace("://", "://datasets-server.") + "/parquet?dataset="
        try:
            parquet_data_files_response = http_get(
                url=datasets_server_parquet_url + self.name,
                temp_file=None,
                headers=get_authentication_headers_for_url(config.HF_ENDPOINT + f"datasets/{self.name}"),
                timeout=100.0,
                max_retries=3,
            )
            parquet_data_files_response.raise_for_status()
            if "X-Revision" in parquet_data_files_response.headers:
                if parquet_data_files_response.headers["X-Revision"] == self.from_sha or self.from_sha is None:
                    parquet_data_files_response_json = parquet_data_files_response.json()
                    if (
                        parquet_data_files_response_json.get("partial") is False
                        and not parquet_data_files_response_json.get("pending", True)
                        and not parquet_data_files_response_json.get("failed", True)
                        and "parquet_files" in parquet_data_files_response_json
                    ):
                        return parquet_data_files_response_json["parquet_files"]
                    else:
                        logger.debug(f"Parquet export for {self.name} is not completely ready yet.")
                else:
                    logger.debug(
                        f"Parquet export for {self.name} is available but outdated (revision='{parquet_data_files_response.headers['X-Revision']}')"

Review comment (suggestion):
Maybe let's define a private `utils/_datasets_server.py` module and put the logic from the `_get_exported_parquet_files` and `_get_exported_dataset_infos` (and `DatasetsServerError`) there.

IIRC our goal is to have an API to interact with `datasets-server` in `huggingface_hub` eventually, so I guess this is a temporary solution

Code after:
            builder_configs_parameters=BuilderConfigsParameters(
                metadata_configs=metadata_configs,
                builder_configs=builder_configs,
                default_config_name=default_config_name,
            ),
        )


class HubDatasetModuleFactoryWithParquetExport(_DatasetModuleFactory):
    """
    Get the module of a dataset loaded from parquet files of a dataset repository parquet export.
    """

    def __init__(
        self,
        name: str,
        revision: Optional[str] = None,
        download_config: Optional[DownloadConfig] = None,
    ):
        self.name = name
        self.revision = revision
        self.download_config = download_config or DownloadConfig()
        increase_load_count(name, resource_type="dataset")

    def get_module(self) -> DatasetModule:
        exported_parquet_files = _datasets_server.get_exported_parquet_files(dataset=self.name, revision=self.revision)
        exported_dataset_infos = _datasets_server.get_exported_dataset_infos(dataset=self.name, revision=self.revision)
        hfh_dataset_info = HfApi(config.HF_ENDPOINT).dataset_info(
            self.name,
            revision="refs/convert/parquet",
            token=self.download_config.token,
            timeout=100.0,
        )
        # even if metadata_configs is not None (which means that we will resolve files for each config later)
        # we cannot skip resolving all files because we need to infer module name by files extensions
        revision = hfh_dataset_info.sha  # fix the revision in case there are new commits in the meantime
        metadata_configs = MetadataConfigs._from_exported_parquet_files(
            revision=revision, exported_parquet_files=exported_parquet_files
        )
        module_path, hash = _PACKAGED_DATASETS_MODULES["parquet"]
        builder_configs, default_config_name = create_builder_configs_from_metadata_configs(
            module_path,
            metadata_configs,
            supports_metadata=False,
            download_config=self.download_config,
        )
        builder_kwargs = {
            "hash": hash,
            "repo_id": self.name,
            "dataset_name": camelcase_to_snakecase(Path(self.name).name),
        }