Repository: huggingface/dataset-viewer

Issue:
Values of `__hf_index_id` in DuckDB indexes are not in the right order
For example the first item in de C4 dataset is supposed to be

```
Beginners BBQ Class Taking Place in Missoula!...
```

This is also the first example when querying the DuckDB index, but it has `__hf_index_id == 1000`

```python
con.sql("select * from data limit 5;")
```

```
┌───────────────┬──────────────────────┬──────────────────────┬────────────────────────────────────────────────────────┐
│ __hf_index_id │         text         │      timestamp       │                          url                           │
│     int64     │       varchar        │       varchar        │                        varchar                         │
├───────────────┼──────────────────────┼──────────────────────┼────────────────────────────────────────────────────────┤
│          1000 │ Beginners BBQ Clas…  │ 2019-04-25T12:57:54Z │ https://klyq.com/beginners-bbq-class-taking-place-in…  │
│          1001 │ Discussion in 'Mac…  │ 2019-04-21T10:07:13Z │ https://forums.macrumors.com/threads/restore-from-la…  │
│          1002 │ Foil plaid lycra a…  │ 2019-04-25T10:40:23Z │ https://awishcometrue.com/Catalogs/Clearance/Tweens/…  │
│          1003 │ How many backlinks…  │ 2019-04-21T12:46:19Z │ https://www.blackhatworld.com/seo/how-many-backlinks…  │
│          1004 │ The Denver Board o…  │ 2019-04-20T14:33:21Z │ http://bond.dpsk12.org/category/news/                  │
└───────────────┴──────────────────────┴──────────────────────┴────────────────────────────────────────────────────────┘
```

```python
con.sql("select * from data where __hf_index_id < 5;")
```

```
┌───────────────┬──────────────────────┬──────────────────────┬────────────────────────────────────────────────────────┐
│ __hf_index_id │         text         │      timestamp       │                          url                           │
│     int64     │       varchar        │       varchar        │                        varchar                         │
├───────────────┼──────────────────────┼──────────────────────┼────────────────────────────────────────────────────────┤
│             0 │ Eimhir Ltd, its af…  │ 2019-04-24T10:02:25Z │ https://www.game-pal.com/signup                        │
│             1 │ Roger & Gallet Thé…  │ 2019-04-21T06:22:38Z │ https://www.farmaline.uk/health/order/roger-gallet-t…  │
│             2 │ Folio: 166v - Feas…  │ 2019-04-19T17:07:24Z │ http://cantus.uwaterloo.ca/chant/481394                │
│             3 │ DIY Co Sleeper For…  │ 2019-04-21T16:15:08Z │ https://www.musely.com/tips/DIY-Co-Sleeper-For-Baby-…  │
│             4 │ Rep. Ilhan Omar (D…  │ 2019-04-26T07:55:58Z │ http://theminnesotasun.com/2019/01/13/ilhan-omar-dec…  │
└───────────────┴──────────────────────┴──────────────────────┴────────────────────────────────────────────────────────┘
```

This would show the wrong row idx when clicking on the example after search (which times out atm though)

Not a big deal for now imo


Discussion:

Resolving patch:
diff --git a/services/worker/src/worker/job_runners/split/duckdb_index.py b/services/worker/src/worker/job_runners/split/duckdb_index.py
--- a/services/worker/src/worker/job_runners/split/duckdb_index.py
+++ b/services/worker/src/worker/job_runners/split/duckdb_index.py
@@ -54,9 +54,11 @@
 VALUE_FEATURE_TYPE = "Value"
 DUCKDB_DEFAULT_INDEX_FILENAME = "index.duckdb"
 DUCKDB_DEFAULT_PARTIAL_INDEX_FILENAME = "partial-index.duckdb"
-CREATE_SEQUENCE_COMMAND = "CREATE OR REPLACE SEQUENCE serial START 0 MINVALUE 0;"
 CREATE_INDEX_COMMAND = "PRAGMA create_fts_index('data', '__hf_index_id', {columns}, overwrite=1);"
-CREATE_TABLE_COMMAND = "CREATE OR REPLACE TABLE data AS SELECT nextval('serial') AS __hf_index_id, {columns} FROM"
+CREATE_TABLE_COMMAND = "CREATE OR REPLACE TABLE data AS SELECT {columns} FROM '{source}';"
+CREATE_SEQUENCE_COMMAND = "CREATE OR REPLACE SEQUENCE serial START 0 MINVALUE 0;"
+ALTER_TABLE_BY_ADDING_SEQUENCE_COLUMN = "ALTER TABLE data ADD COLUMN __hf_index_id BIGINT DEFAULT nextval('serial');"
+CREATE_TABLE_COMMANDS = CREATE_TABLE_COMMAND + CREATE_SEQUENCE_COMMAND + ALTER_TABLE_BY_ADDING_SEQUENCE_COLUMN
 INSTALL_EXTENSION_COMMAND = "INSTALL '{extension}';"
 LOAD_EXTENSION_COMMAND = "LOAD '{extension}';"
 SET_EXTENSIONS_DIRECTORY_COMMAND = "SET extension_directory='{directory}';"
@@ -169,9 +171,6 @@ def compute_index_rows(
     con.execute(INSTALL_EXTENSION_COMMAND.format(extension="fts"))
     con.execute(LOAD_EXTENSION_COMMAND.format(extension="fts"))
 
-    logging.debug(CREATE_SEQUENCE_COMMAND)
-    con.sql(CREATE_SEQUENCE_COMMAND)
-
     # see https://pypi.org/project/hf-transfer/ for more details about how to enable hf_transfer
     os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
     for parquet_file in parquet_file_names:
@@ -189,7 +188,7 @@ def compute_index_rows(
         )
 
     all_split_parquets = f"{duckdb_index_file_directory}/{config}/{split_directory}/*.parquet"
-    create_command_sql = f"{CREATE_TABLE_COMMAND.format(columns=column_names)} '{all_split_parquets}';"
+    create_command_sql = CREATE_TABLE_COMMANDS.format(columns=column_names, source=all_split_parquets)
     logging.debug(create_command_sql)
     con.sql(create_command_sql)
 


Test patch:
diff --git a/services/worker/tests/job_runners/split/test_duckdb_index.py b/services/worker/tests/job_runners/split/test_duckdb_index.py
--- a/services/worker/tests/job_runners/split/test_duckdb_index.py
+++ b/services/worker/tests/job_runners/split/test_duckdb_index.py
@@ -6,12 +6,15 @@
 from contextlib import ExitStack
 from dataclasses import replace
 from http import HTTPStatus
+from pathlib import Path
 from typing import Optional
 from unittest.mock import patch
 
 import datasets.config
 import duckdb
 import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
 import pytest
 import requests
 from datasets import Features, Image, Sequence, Value
@@ -29,8 +32,7 @@
 from worker.job_runners.config.parquet_metadata import ConfigParquetMetadataJobRunner
 from worker.job_runners.split.duckdb_index import (
     CREATE_INDEX_COMMAND,
-    CREATE_SEQUENCE_COMMAND,
-    CREATE_TABLE_COMMAND,
+    CREATE_TABLE_COMMANDS,
     SplitDuckDbIndexJobRunner,
     get_indexable_columns,
 )
@@ -517,8 +519,21 @@ def test_get_indexable_columns(features: Features, expected: list[str]) -> None:
 )
 def test_index_command(df: pd.DataFrame, query: str, expected_ids: list[int]) -> None:
     columns = ",".join('"' + str(column) + '"' for column in df.columns)
-    duckdb.sql(CREATE_SEQUENCE_COMMAND)
-    duckdb.sql(CREATE_TABLE_COMMAND.format(columns=columns) + " df;")
+    duckdb.sql(CREATE_TABLE_COMMANDS.format(columns=columns, source="df"))
     duckdb.sql(CREATE_INDEX_COMMAND.format(columns=columns))
     result = duckdb.execute(FTS_COMMAND, parameters=[query]).df()
     assert list(result.__hf_index_id) == expected_ids
+
+
+def test_table_column_hf_index_id_is_monotonic_increasing(tmp_path: Path) -> None:
+    pa_table = pa.Table.from_pydict({"text": [f"text-{i}" for i in range(100)]})
+    parquet_path = str(tmp_path / "0000.parquet")
+    pq.write_table(pa_table, parquet_path, row_group_size=2)
+    db_path = str(tmp_path / "index.duckdb")
+    column_names = ",".join(f'"{column}"' for column in pa_table.column_names)
+    with duckdb.connect(db_path) as con:
+        con.sql(CREATE_TABLE_COMMANDS.format(columns=column_names, source=parquet_path))
+    with duckdb.connect(db_path) as con:
+        df = con.sql("SELECT * FROM data").to_df()
+    assert df["__hf_index_id"].is_monotonic_increasing
+    assert df["__hf_index_id"].is_unique