Repository: huggingface/dataset-viewer

Issue:
Upgrade to duckdb 0.10.1
https://github.com/duckdb/duckdb/releases/tag/v0.10.1


Discussion:

Resolving patch:
diff --git a/libs/libapi/src/libapi/duckdb.py b/libs/libapi/src/libapi/duckdb.py
--- a/libs/libapi/src/libapi/duckdb.py
+++ b/libs/libapi/src/libapi/duckdb.py
@@ -9,7 +9,7 @@
 
 import anyio
 from anyio import Path
-from libcommon.constants import DUCKDB_INDEX_DOWNLOADS_SUBDIRECTORY, DUCKDB_VERSION, SPLIT_DUCKDB_INDEX_KIND
+from libcommon.constants import DUCKDB_INDEX_DOWNLOADS_SUBDIRECTORY, SPLIT_DUCKDB_INDEX_KIND
 from libcommon.parquet_utils import extract_split_name_from_parquet_url
 from libcommon.prometheus import StepProfiler
 from libcommon.simple_cache import CacheEntry
@@ -64,7 +64,7 @@ def get_download_folder(root_directory: StrPath, dataset: str, revision: str, co
     payload = (dataset, config, split, revision)
     hash_suffix = sha1(json.dumps(payload, sort_keys=True).encode(), usedforsecurity=False).hexdigest()[:8]
     subdirectory = "".join([c if re.match(r"[\w-]", c) else "-" for c in f"{dataset}-{hash_suffix}"])
-    return f"{root_directory}/{DUCKDB_INDEX_DOWNLOADS_SUBDIRECTORY}/{DUCKDB_VERSION}/{subdirectory}"
+    return f"{root_directory}/{DUCKDB_INDEX_DOWNLOADS_SUBDIRECTORY}/{subdirectory}"
 
 
 def download_index_file(
diff --git a/libs/libcommon/src/libcommon/constants.py b/libs/libcommon/src/libcommon/constants.py
--- a/libs/libcommon/src/libcommon/constants.py
+++ b/libs/libcommon/src/libcommon/constants.py
@@ -9,7 +9,6 @@
 DUCKDB_INDEX_CACHE_APPNAME = "dataset_viewer_duckdb_index"
 DUCKDB_INDEX_DOWNLOADS_SUBDIRECTORY = "downloads"
 DUCKDB_INDEX_JOB_RUNNER_SUBDIRECTORY = "job_runner"
-DUCKDB_VERSION = "0.10.0"
 CACHE_METRICS_COLLECTION = "cacheTotalMetric"
 TYPE_AND_STATUS_JOB_COUNTS_COLLECTION = "jobTotalMetric"
 WORKER_TYPE_JOB_COUNTS_COLLECTION = "workerTypeJobCounts"
diff --git a/services/worker/src/worker/job_runners/split/duckdb_index.py b/services/worker/src/worker/job_runners/split/duckdb_index.py
--- a/services/worker/src/worker/job_runners/split/duckdb_index.py
+++ b/services/worker/src/worker/job_runners/split/duckdb_index.py
@@ -16,10 +16,7 @@
 )
 from huggingface_hub.hf_api import HfApi
 from huggingface_hub.utils._errors import HfHubHTTPError, RepositoryNotFoundError
-from libcommon.constants import (
-    DUCKDB_INDEX_JOB_RUNNER_SUBDIRECTORY,
-    DUCKDB_VERSION,
-)
+from libcommon.constants import DUCKDB_INDEX_JOB_RUNNER_SUBDIRECTORY
 from libcommon.dtos import JobInfo
 from libcommon.exceptions import (
     CacheDirectoryNotInitializedError,
@@ -314,7 +311,7 @@ def compute_split_duckdb_index_response(
         partial=partial,
         num_rows=num_rows,
         num_bytes=num_bytes,
-        duckdb_version=DUCKDB_VERSION,
+        duckdb_version=duckdb.__version__,
     )
 
 


Test patch:
diff --git a/services/worker/tests/job_runners/split/test_duckdb_index.py b/services/worker/tests/job_runners/split/test_duckdb_index.py
--- a/services/worker/tests/job_runners/split/test_duckdb_index.py
+++ b/services/worker/tests/job_runners/split/test_duckdb_index.py
@@ -19,7 +19,6 @@
 import requests
 from datasets import Features, Image, Sequence, Value
 from datasets.packaged_modules.csv.csv import CsvConfig
-from libcommon.constants import DUCKDB_VERSION
 from libcommon.dtos import Priority
 from libcommon.resources import CacheMongoResource, QueueMongoResource
 from libcommon.simple_cache import upsert_response
@@ -344,7 +343,6 @@ def test_compute(
         features = content["features"]
         has_fts = content["has_fts"]
         partial = content["partial"]
-        assert content["duckdb_version"] == DUCKDB_VERSION
         assert isinstance(has_fts, bool)
         assert has_fts == expected_has_fts
         assert isinstance(url, str)