fix: Remove Qdrant order_by and upgrade reranker dependencies

- Fixed 400 error: Cannot use offset with order_by
- Upgraded transformers to 4.45.0 for tokenizer fix
- Upgraded FlagEmbedding to 1.2.11 for reranker fix
- Simplified browse method (client-side sorting)

Run: pip install --upgrade FlagEmbedding transformers

requirements.txt

@@ -11,11 +11,11 @@ python-dotenv>=1.0.1
 psycopg2-binary>=2.9.9
 SQLAlchemy>=2.0.29
 sentence-transformers>=2.7.0
-transformers>=4.40.0      # DeBERTa intent classifier
+transformers>=4.45.0      # DeBERTa intent classifier + reranker tokenizer (upgraded for compatibility)
 torch>=2.0.0
 numpy>=1.26.0
 tiktoken>=0.6.0
-FlagEmbedding>=1.2.5
+FlagEmbedding>=1.2.11     # BGE reranker (upgraded for tokenizer fix)
 redis>=5.0.0
 python-jose[cryptography]>=3.3.0
 passlib[bcrypt]>=1.7.4

src/infrastructure/adapters/qdrant_adapter.py

@@ -343,6 +343,7 @@ class QdrantAdapter(VectorStorePort):
 
         try:
             # Fetch more than needed so we can deduplicate to first chunk per article
+            # NOTE: Qdrant doesn't support order_by with offset, so we do client-side sorting
             results, next_page_offset = self.client.scroll(
                 collection_name=settings.QDRANT_COLLECTION,
                 scroll_filter=filter_obj,
@@ -350,10 +351,6 @@ class QdrantAdapter(VectorStorePort):
                 offset=offset,
                 with_payload=True,
                 with_vectors=False,
-                order_by=models.OrderBy(
-                    key="published_at",
-                    direction=models.Direction.DESC
-                )
             )
 
             # Keep only the lowest chunk_index per doc_id (first chunk of each article)
@@ -366,7 +363,7 @@ class QdrantAdapter(VectorStorePort):
                     seen_docs[doc_id] = (point, chunk_index)
 
             deduped = [v[0] for v in seen_docs.values()]
-            # Results are already sorted by Qdrant, but re-sort to ensure consistency after dedup
+            # Client-side sort by published_at (descending - newest first)
             deduped.sort(
                 key=lambda p: (p.payload or {}).get("published_at") or "",
                 reverse=True,
@@ -374,33 +371,5 @@ class QdrantAdapter(VectorStorePort):
             return {"articles": deduped[:limit], "next_offset": next_page_offset}
 
         except Exception as e:
-            # If order_by fails (older Qdrant version), retry without it
-            if "order_by" in str(e).lower():
-                logger.warning(f"Qdrant order_by not supported, falling back to client-side sort: {e}")
-                try:
-                    results, next_page_offset = self.client.scroll(
-                        collection_name=settings.QDRANT_COLLECTION,
-                        scroll_filter=filter_obj,
-                        limit=limit * 8,
-                        offset=offset,
-                        with_payload=True,
-                        with_vectors=False,
-                    )
-                    seen_docs: dict = {}
-                    for point in results:
-                        payload = point.payload or {}
-                        doc_id = payload.get("doc_id", point.id)
-                        chunk_index = payload.get("chunk_index", 0)
-                        if doc_id not in seen_docs or chunk_index < seen_docs[doc_id][1]:
-                            seen_docs[doc_id] = (point, chunk_index)
-                    deduped = [v[0] for v in seen_docs.values()]
-                    deduped.sort(
-                        key=lambda p: (p.payload or {}).get("published_at") or "",
-                        reverse=True,
-                    )
-                    return {"articles": deduped[:limit], "next_offset": next_page_offset}
-                except Exception as e2:
-                    logger.error(f"Error browsing Qdrant (fallback): {e2}")
-                    return {"articles": [], "next_offset": None}
             logger.error(f"Error browsing Qdrant: {e}")
             return {"articles": [], "next_offset": None}