chroma pathing

src/legisqa_local/config/settings.py

@@ -141,10 +141,13 @@ def download_chromadb_from_hf_dataset(dataset_repo: str, local_path: str) -> boo
         logger.info(f"Downloading dataset: {dataset_repo}")
         logger.info("This may take several minutes for large datasets...")
         
+        # Download to a temporary location first
+        temp_download_path = f"{local_path}_temp"
+        
         downloaded_path = snapshot_download(
             repo_id=dataset_repo,
             repo_type="dataset",
-            local_dir=local_path,
+            local_dir=temp_download_path,
             cache_dir="/tmp/hf_chromadb_cache"
             # Note: resume_download and local_dir_use_symlinks are now handled automatically
         )
@@ -152,7 +155,27 @@ def download_chromadb_from_hf_dataset(dataset_repo: str, local_path: str) -> boo
         logger.info(f"✅ ChromaDB download from HF Dataset complete!")
         logger.info(f"Downloaded to: {downloaded_path}")
         
-        # Verify the download by checking for expected files
+        # The HF dataset contains a 'chromadb' subdirectory with the actual ChromaDB files
+        chromadb_subdir = os.path.join(temp_download_path, "chromadb")
+        
+        if os.path.exists(chromadb_subdir):
+            logger.info(f"📁 Found ChromaDB subdirectory: {chromadb_subdir}")
+            
+            # Move the ChromaDB files from the subdirectory to the target location
+            import shutil
+            if os.path.exists(local_path):
+                shutil.rmtree(local_path)
+            shutil.move(chromadb_subdir, local_path)
+            
+            # Clean up the temporary download directory
+            shutil.rmtree(temp_download_path)
+            
+            logger.info(f"✅ ChromaDB files moved to: {local_path}")
+        else:
+            logger.error(f"❌ ChromaDB subdirectory not found in downloaded data: {chromadb_subdir}")
+            return False
+        
+        # Verify the final ChromaDB structure
         if os.path.exists(local_path) and os.listdir(local_path):
             file_count = sum(len(files) for _, _, files in os.walk(local_path))
             total_size = sum(
@@ -161,13 +184,21 @@ def download_chromadb_from_hf_dataset(dataset_repo: str, local_path: str) -> boo
                 for filename in filenames
             ) / (1024 * 1024 * 1024)  # Convert to GB
             
-            logger.info(f"📊 Download verification:")
+            logger.info(f"📊 ChromaDB verification:")
             logger.info(f"   Files: {file_count}")
             logger.info(f"   Total size: {total_size:.2f} GB")
             
-            return True
+            # Check for key ChromaDB files
+            sqlite_file = os.path.join(local_path, "chroma.sqlite3")
+            if os.path.exists(sqlite_file):
+                sqlite_size = os.path.getsize(sqlite_file) / (1024 * 1024 * 1024)
+                logger.info(f"   SQLite database: {sqlite_size:.2f} GB")
+                return True
+            else:
+                logger.error("❌ chroma.sqlite3 not found in ChromaDB directory")
+                return False
         else:
-            logger.error("❌ Download completed but no files found in target directory")
+            logger.error("❌ ChromaDB directory is empty after processing")
             return False
             
     except ImportError:
@@ -212,15 +243,16 @@ def inspect_chromadb(chroma_path: str):
             if count > 0:
                 # Get a sample item
                 logger.info("🔍 Fetching sample items...")
-                sample = collection.get(limit=3, include=["documents", "metadatas", "ids"])
+                sample = collection.get(limit=3, include=["documents", "metadatas"])
                 
-                logger.info(f"📝 Sample IDs: {sample['ids']}")
+                if sample.get('ids'):
+                    logger.info(f"📝 Sample IDs: {sample['ids']}")
                 
-                if sample['documents']:
+                if sample.get('documents'):
                     logger.info(f"📄 Sample document (first 200 chars):")
                     logger.info(f"   {sample['documents'][0][:200]}...")
                 
-                if sample['metadatas']:
+                if sample.get('metadatas'):
                     logger.info(f"🏷️  Sample metadata:")
                     for i, metadata in enumerate(sample['metadatas'][:2]):
                         logger.info(f"   Item {i}: {metadata}")

uv.lock

@@ -191,34 +191,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" },
 ]
 
-[[package]]
-name = "boto3"
-version = "1.40.40"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "botocore" },
-    { name = "jmespath" },
-    { name = "s3transfer" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/3c/12/1a31b36802d0f33bc6982ab8b7e6437d75ef3c179abe6c53d4d8f7b4248f/boto3-1.40.40.tar.gz", hash = "sha256:f384d3a0410d0f1a4d4ae7aa69c41d0549c6ca5a76667dc25fc97d50ad6db740", size = 111606, upload-time = "2025-09-26T19:23:46.923Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/90/69/c65566dbdaaea3af0c23f7731ab0f185a38b593fd449d2423374150dbfe0/boto3-1.40.40-py3-none-any.whl", hash = "sha256:385904de68623e1c341bdc095d94a30006843032c912adeb1e0752a343632ec6", size = 139340, upload-time = "2025-09-26T19:23:45.557Z" },
-]
-
-[[package]]
-name = "botocore"
-version = "1.40.40"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "jmespath" },
-    { name = "python-dateutil" },
-    { name = "urllib3" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/83/5a/43a7fea503ad14fa79819f2b3103a38977fb587a3663d1ac6e958fccf592/botocore-1.40.40.tar.gz", hash = "sha256:78eb121a16a6481ed0f6e1aebe53a4f23aa121f34466846c13a5ca48fa980e31", size = 14363370, upload-time = "2025-09-26T19:23:37.853Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ed/5e/3bbf6d34cbf307c1b9e58e0204ceba2d35bbc0c93b4e3b3cc895aae0a5fd/botocore-1.40.40-py3-none-any.whl", hash = "sha256:68506142b3cde93145ef3ee0268f2444f2b68ada225a151f714092bbd3d6516a", size = 14031738, upload-time = "2025-09-26T19:23:35.475Z" },
-]
-
 [[package]]
 name = "build"
 version = "1.3.0"
@@ -845,15 +817,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/af/22/7ab7b4ec3a1c1f03aef376af11d23b05abcca3fb31fbca1e7557053b1ba2/jiter-0.11.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e2bbf24f16ba5ad4441a9845e40e4ea0cb9eed00e76ba94050664ef53ef4406", size = 347102, upload-time = "2025-09-15T09:20:20.16Z" },
 ]
 
-[[package]]
-name = "jmespath"
-version = "1.0.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" },
-]
-
 [[package]]
 name = "joblib"
 version = "1.5.2"
@@ -1113,9 +1076,9 @@ name = "legisqa-local"
 version = "0.1.0"
 source = { editable = "." }
 dependencies = [
-    { name = "boto3" },
     { name = "chromadb" },
     { name = "datasets" },
+    { name = "huggingface-hub" },
     { name = "langchain" },
     { name = "langchain-anthropic" },
     { name = "langchain-chroma" },
@@ -1132,9 +1095,9 @@ dependencies = [
 
 [package.metadata]
 requires-dist = [
-    { name = "boto3", specifier = ">=1.35.0" },
     { name = "chromadb", specifier = ">=1.1.0" },
     { name = "datasets", specifier = ">=3.0.0" },
+    { name = "huggingface-hub", specifier = ">=0.19.0" },
     { name = "langchain", specifier = ">=0.3.27" },
     { name = "langchain-anthropic", specifier = ">=0.3.19" },
     { name = "langchain-chroma", specifier = ">=0.1.4" },
@@ -2390,18 +2353,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" },
 ]
 
-[[package]]
-name = "s3transfer"
-version = "0.14.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "botocore" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/62/74/8d69dcb7a9efe8baa2046891735e5dfe433ad558ae23d9e3c14c633d1d58/s3transfer-0.14.0.tar.gz", hash = "sha256:eff12264e7c8b4985074ccce27a3b38a485bb7f7422cc8046fee9be4983e4125", size = 151547, upload-time = "2025-09-09T19:23:31.089Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" },
-]
-
 [[package]]
 name = "safetensors"
 version = "0.6.2"