Repository: pola-rs/polars
File: crates/polars-io/src/path_utils/hugging_face.rs (Rust)

Code before:

/// Percent-encoding character set for HF Hub paths.
/// Like URL_ENCODE_CHAR_SET but preserves slashes (/) and encodes brackets.
/// HF Hub rate limiting classifies URLs by path structure - encoding slashes
/// causes requests to be misclassified as "pages" (100/5min limit) instead of
/// "resolvers" (3000/5min limit). See: https://github.com/pola-rs/polars/issues/25389
const HF_PATH_ENCODE_CHAR_SET: &percent_encoding::AsciiSet =
    &URL_ENCODE_CHAR_SET.remove(b'/').add(b'[').add(b']');

#[derive(Debug, PartialEq)]
struct HFPathParts {
    bucket: String,
    repository: String,
    revision: String,
    /// Path relative to the repository root.
    path: String,
}

struct HFRepoLocation {
    api_base_path: String,
    download_base_path: String,
}

impl HFRepoLocation {
    fn new(bucket: &str, repository: &str, revision: &str) -> Self {
        // Don't percent-encode bucket/repository/revision - they are path segments
        // that should preserve slashes. E.g. "HuggingFaceFW/fineweb-2" must stay as-is,
        // not become "HuggingFaceFW%2Ffineweb-2" which would be misclassified by the Hub.
        // See: https://github.com/pola-rs/polars/issues/25389
        let api_base_path = format!(
            "https://huggingface.co/api/{}/{}/tree/{}/",
            bucket, repository, revision
        );
        let download_base_path = format!(
            "https://huggingface.co/{}/{}/resolve/{}/",
            bucket, repository, revision
        );

        Self {
            api_base_path,
            download_base_path,
        }
    }

    fn get_file_uri(&self, rel_path: &str) -> String {
        // Encode special characters but preserve slashes.
        // Slashes must remain unencoded for correct HF rate limit classification.
        // See: https://github.com/pola-rs/polars/issues/25389
        format!(
            "{}{}",
            self.download_base_path,

Review comment (suggestion):
I think you should percent-encode revisions, e.g. `refs/convert/parquet` -> `refs%2F/convert%2Fparquet` in the final URL

Code after:
/// Percent-encoding character set for HF Hub paths.
///
/// This is URL_ENCODE_CHARSET with slashes preserved - by not encoding slashes,
/// the API request will be counted under a higher "resolvers" ratelimit of (3000/5min)
/// compared to the default "pages" limit of (100/5min limit).
///
/// ref <https://github.com/pola-rs/polars/issues/25389>
const HF_PATH_ENCODE_CHARSET: &percent_encoding::AsciiSet = &URL_ENCODE_CHARSET.remove(b'/');

#[derive(Debug, PartialEq)]
struct HFPathParts {
    bucket: String,
    repository: String,
    revision: String,
    /// Path relative to the repository root.
    path: String,
}

struct HFRepoLocation {
    api_base_path: String,
    download_base_path: String,
}

impl HFRepoLocation {
    fn new(bucket: &str, repository: &str, revision: &str) -> Self {
        // * Don't percent-encode bucket/repository - they are path segments where
        //   slashes are separators. E.g. "HuggingFaceFW/fineweb-2" must stay as-is.
        // * DO encode revision - slashes in revisions like "refs/convert/parquet"
        //   are part of the revision name, not path separators.
        //   See: https://github.com/pola-rs/polars/issues/25389
        let encoded_revision =
            percent_encoding::percent_encode(revision.as_bytes(), URL_ENCODE_CHARSET);
        let api_base_path = format!(
            "https://huggingface.co/api/{}/{}/tree/{}/",
            bucket, repository, encoded_revision
        );
        let download_base_path = format!(
            "https://huggingface.co/{}/{}/resolve/{}/",
            bucket, repository, encoded_revision
        );

        Self {
            api_base_path,
            download_base_path,
        }
    }

    fn get_file_uri(&self, rel_path: &str) -> String {
        format!(
            "{}{}",
            self.download_base_path,