Refac: pipelinge message
Browse files- lightrag/api/routers/document_routes.py +21 -32
- lightrag/lightrag.py +5 -3
lightrag/api/routers/document_routes.py
CHANGED
@@ -836,24 +836,24 @@ async def background_delete_documents(
|
|
836 |
# Loop through each document ID and delete them one by one
|
837 |
for i, doc_id in enumerate(doc_ids, 1):
|
838 |
async with pipeline_status_lock:
|
|
|
|
|
839 |
pipeline_status["cur_batch"] = i
|
840 |
-
pipeline_status["latest_message"] =
|
841 |
-
|
842 |
-
)
|
843 |
-
pipeline_status["history_messages"].append(
|
844 |
-
f"Processing document {i}/{total_docs}: {doc_id}"
|
845 |
-
)
|
846 |
|
|
|
847 |
try:
|
848 |
result = await rag.adelete_by_doc_id(doc_id)
|
849 |
-
|
|
|
|
|
850 |
if result.status == "success":
|
851 |
successful_deletions.append(doc_id)
|
852 |
success_msg = (
|
853 |
-
f"
|
854 |
)
|
855 |
logger.info(success_msg)
|
856 |
-
|
857 |
async with pipeline_status_lock:
|
858 |
pipeline_status["history_messages"].append(success_msg)
|
859 |
|
@@ -872,6 +872,7 @@ async def background_delete_documents(
|
|
872 |
)
|
873 |
logger.info(file_delete_msg)
|
874 |
async with pipeline_status_lock:
|
|
|
875 |
pipeline_status["history_messages"].append(
|
876 |
file_delete_msg
|
877 |
)
|
@@ -881,6 +882,9 @@ async def background_delete_documents(
|
|
881 |
)
|
882 |
logger.warning(file_not_found_msg)
|
883 |
async with pipeline_status_lock:
|
|
|
|
|
|
|
884 |
pipeline_status["history_messages"].append(
|
885 |
file_not_found_msg
|
886 |
)
|
@@ -888,6 +892,7 @@ async def background_delete_documents(
|
|
888 |
file_error_msg = f"Failed to delete file {result.file_path}: {str(file_error)}"
|
889 |
logger.error(file_error_msg)
|
890 |
async with pipeline_status_lock:
|
|
|
891 |
pipeline_status["history_messages"].append(
|
892 |
file_error_msg
|
893 |
)
|
@@ -895,52 +900,36 @@ async def background_delete_documents(
|
|
895 |
no_file_msg = f"No valid file path found for document {doc_id}"
|
896 |
logger.warning(no_file_msg)
|
897 |
async with pipeline_status_lock:
|
|
|
898 |
pipeline_status["history_messages"].append(no_file_msg)
|
899 |
else:
|
900 |
failed_deletions.append(doc_id)
|
901 |
-
error_msg = f"Failed to delete
|
902 |
logger.error(error_msg)
|
903 |
-
|
904 |
async with pipeline_status_lock:
|
|
|
905 |
pipeline_status["history_messages"].append(error_msg)
|
906 |
|
907 |
except Exception as e:
|
908 |
failed_deletions.append(doc_id)
|
909 |
-
error_msg = (
|
910 |
-
f"Error deleting document {i}/{total_docs}: {doc_id} - {str(e)}"
|
911 |
-
)
|
912 |
logger.error(error_msg)
|
913 |
logger.error(traceback.format_exc())
|
914 |
-
|
915 |
async with pipeline_status_lock:
|
|
|
916 |
pipeline_status["history_messages"].append(error_msg)
|
917 |
|
918 |
-
# Final summary
|
919 |
-
summary_msg = f"Deletion completed: {len(successful_deletions)} successful, {len(failed_deletions)} failed"
|
920 |
-
logger.info(summary_msg)
|
921 |
-
|
922 |
-
async with pipeline_status_lock:
|
923 |
-
pipeline_status["history_messages"].append(summary_msg)
|
924 |
-
if successful_deletions:
|
925 |
-
pipeline_status["history_messages"].append(
|
926 |
-
f"Successfully deleted: {', '.join(successful_deletions)}"
|
927 |
-
)
|
928 |
-
if failed_deletions:
|
929 |
-
pipeline_status["history_messages"].append(
|
930 |
-
f"Failed to delete: {', '.join(failed_deletions)}"
|
931 |
-
)
|
932 |
-
|
933 |
except Exception as e:
|
934 |
error_msg = f"Critical error during batch deletion: {str(e)}"
|
935 |
logger.error(error_msg)
|
936 |
logger.error(traceback.format_exc())
|
937 |
-
|
938 |
async with pipeline_status_lock:
|
939 |
pipeline_status["history_messages"].append(error_msg)
|
940 |
finally:
|
|
|
941 |
async with pipeline_status_lock:
|
942 |
pipeline_status["busy"] = False
|
943 |
-
completion_msg = "
|
944 |
pipeline_status["latest_message"] = completion_msg
|
945 |
pipeline_status["history_messages"].append(completion_msg)
|
946 |
|
|
|
836 |
# Loop through each document ID and delete them one by one
|
837 |
for i, doc_id in enumerate(doc_ids, 1):
|
838 |
async with pipeline_status_lock:
|
839 |
+
start_msg = f"Deleting document {i}/{total_docs}: {doc_id}"
|
840 |
+
logger.info(start_msg)
|
841 |
pipeline_status["cur_batch"] = i
|
842 |
+
pipeline_status["latest_message"] = start_msg
|
843 |
+
pipeline_status["history_messages"].append(start_msg)
|
|
|
|
|
|
|
|
|
844 |
|
845 |
+
file_path = "#"
|
846 |
try:
|
847 |
result = await rag.adelete_by_doc_id(doc_id)
|
848 |
+
file_path = (
|
849 |
+
getattr(result, "file_path", "-") if "result" in locals() else "-"
|
850 |
+
)
|
851 |
if result.status == "success":
|
852 |
successful_deletions.append(doc_id)
|
853 |
success_msg = (
|
854 |
+
f"Deleted document {i}/{total_docs}: {doc_id}[{file_path}]"
|
855 |
)
|
856 |
logger.info(success_msg)
|
|
|
857 |
async with pipeline_status_lock:
|
858 |
pipeline_status["history_messages"].append(success_msg)
|
859 |
|
|
|
872 |
)
|
873 |
logger.info(file_delete_msg)
|
874 |
async with pipeline_status_lock:
|
875 |
+
pipeline_status["latest_message"] = file_delete_msg
|
876 |
pipeline_status["history_messages"].append(
|
877 |
file_delete_msg
|
878 |
)
|
|
|
882 |
)
|
883 |
logger.warning(file_not_found_msg)
|
884 |
async with pipeline_status_lock:
|
885 |
+
pipeline_status["latest_message"] = (
|
886 |
+
file_not_found_msg
|
887 |
+
)
|
888 |
pipeline_status["history_messages"].append(
|
889 |
file_not_found_msg
|
890 |
)
|
|
|
892 |
file_error_msg = f"Failed to delete file {result.file_path}: {str(file_error)}"
|
893 |
logger.error(file_error_msg)
|
894 |
async with pipeline_status_lock:
|
895 |
+
pipeline_status["latest_message"] = file_error_msg
|
896 |
pipeline_status["history_messages"].append(
|
897 |
file_error_msg
|
898 |
)
|
|
|
900 |
no_file_msg = f"No valid file path found for document {doc_id}"
|
901 |
logger.warning(no_file_msg)
|
902 |
async with pipeline_status_lock:
|
903 |
+
pipeline_status["latest_message"] = no_file_msg
|
904 |
pipeline_status["history_messages"].append(no_file_msg)
|
905 |
else:
|
906 |
failed_deletions.append(doc_id)
|
907 |
+
error_msg = f"Failed to delete {i}/{total_docs}: {doc_id}[{file_path}] - {result.message}"
|
908 |
logger.error(error_msg)
|
|
|
909 |
async with pipeline_status_lock:
|
910 |
+
pipeline_status["latest_message"] = error_msg
|
911 |
pipeline_status["history_messages"].append(error_msg)
|
912 |
|
913 |
except Exception as e:
|
914 |
failed_deletions.append(doc_id)
|
915 |
+
error_msg = f"Error deleting document {i}/{total_docs}: {doc_id}[{file_path}] - {str(e)}"
|
|
|
|
|
916 |
logger.error(error_msg)
|
917 |
logger.error(traceback.format_exc())
|
|
|
918 |
async with pipeline_status_lock:
|
919 |
+
pipeline_status["latest_message"] = error_msg
|
920 |
pipeline_status["history_messages"].append(error_msg)
|
921 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
922 |
except Exception as e:
|
923 |
error_msg = f"Critical error during batch deletion: {str(e)}"
|
924 |
logger.error(error_msg)
|
925 |
logger.error(traceback.format_exc())
|
|
|
926 |
async with pipeline_status_lock:
|
927 |
pipeline_status["history_messages"].append(error_msg)
|
928 |
finally:
|
929 |
+
# Final summary
|
930 |
async with pipeline_status_lock:
|
931 |
pipeline_status["busy"] = False
|
932 |
+
completion_msg = f"Deletion completed: {len(successful_deletions)} successful, {len(failed_deletions)} failed"
|
933 |
pipeline_status["latest_message"] = completion_msg
|
934 |
pipeline_status["history_messages"].append(completion_msg)
|
935 |
|
lightrag/lightrag.py
CHANGED
@@ -1712,6 +1712,7 @@ class LightRAG:
|
|
1712 |
try:
|
1713 |
# 1. Get the document status and related data
|
1714 |
doc_status_data = await self.doc_status.get_by_id(doc_id)
|
|
|
1715 |
if not doc_status_data:
|
1716 |
logger.warning(f"Document {doc_id} not found")
|
1717 |
return DeletionResult(
|
@@ -1719,6 +1720,7 @@ class LightRAG:
|
|
1719 |
doc_id=doc_id,
|
1720 |
message=f"Document {doc_id} not found.",
|
1721 |
status_code=404,
|
|
|
1722 |
)
|
1723 |
|
1724 |
# 2. Get all chunks related to this document
|
@@ -1770,6 +1772,7 @@ class LightRAG:
|
|
1770 |
doc_id=doc_id,
|
1771 |
message=log_message,
|
1772 |
status_code=200,
|
|
|
1773 |
)
|
1774 |
|
1775 |
chunk_ids = set(related_chunks.keys())
|
@@ -1962,9 +1965,6 @@ class LightRAG:
|
|
1962 |
logger.error(f"Failed to delete document and status: {e}")
|
1963 |
raise Exception(f"Failed to delete document and status: {e}") from e
|
1964 |
|
1965 |
-
# Get file path from document status for return value
|
1966 |
-
file_path = doc_status_data.get("file_path") if doc_status_data else None
|
1967 |
-
|
1968 |
return DeletionResult(
|
1969 |
status="success",
|
1970 |
doc_id=doc_id,
|
@@ -1983,6 +1983,7 @@ class LightRAG:
|
|
1983 |
doc_id=doc_id,
|
1984 |
message=error_message,
|
1985 |
status_code=500,
|
|
|
1986 |
)
|
1987 |
|
1988 |
finally:
|
@@ -2002,6 +2003,7 @@ class LightRAG:
|
|
2002 |
doc_id=doc_id,
|
2003 |
message=f"Deletion completed but failed to persist changes: {persistence_error}",
|
2004 |
status_code=500,
|
|
|
2005 |
)
|
2006 |
# If there was an original exception, log the persistence error but don't override the original error
|
2007 |
# The original error result was already returned in the except block
|
|
|
1712 |
try:
|
1713 |
# 1. Get the document status and related data
|
1714 |
doc_status_data = await self.doc_status.get_by_id(doc_id)
|
1715 |
+
file_path = doc_status_data.get("file_path") if doc_status_data else None
|
1716 |
if not doc_status_data:
|
1717 |
logger.warning(f"Document {doc_id} not found")
|
1718 |
return DeletionResult(
|
|
|
1720 |
doc_id=doc_id,
|
1721 |
message=f"Document {doc_id} not found.",
|
1722 |
status_code=404,
|
1723 |
+
file_path="",
|
1724 |
)
|
1725 |
|
1726 |
# 2. Get all chunks related to this document
|
|
|
1772 |
doc_id=doc_id,
|
1773 |
message=log_message,
|
1774 |
status_code=200,
|
1775 |
+
file_path=file_path,
|
1776 |
)
|
1777 |
|
1778 |
chunk_ids = set(related_chunks.keys())
|
|
|
1965 |
logger.error(f"Failed to delete document and status: {e}")
|
1966 |
raise Exception(f"Failed to delete document and status: {e}") from e
|
1967 |
|
|
|
|
|
|
|
1968 |
return DeletionResult(
|
1969 |
status="success",
|
1970 |
doc_id=doc_id,
|
|
|
1983 |
doc_id=doc_id,
|
1984 |
message=error_message,
|
1985 |
status_code=500,
|
1986 |
+
file_path=file_path,
|
1987 |
)
|
1988 |
|
1989 |
finally:
|
|
|
2003 |
doc_id=doc_id,
|
2004 |
message=f"Deletion completed but failed to persist changes: {persistence_error}",
|
2005 |
status_code=500,
|
2006 |
+
file_path=file_path,
|
2007 |
)
|
2008 |
# If there was an original exception, log the persistence error but don't override the original error
|
2009 |
# The original error result was already returned in the except block
|