added typing
Browse files- lightrag/operate.py +3 -3
lightrag/operate.py
CHANGED
@@ -2,7 +2,7 @@ import asyncio
|
|
2 |
import json
|
3 |
import re
|
4 |
from tqdm.asyncio import tqdm as tqdm_async
|
5 |
-
from typing import Union
|
6 |
from collections import Counter, defaultdict
|
7 |
from .utils import (
|
8 |
logger,
|
@@ -42,9 +42,9 @@ def chunking_by_token_size(
|
|
42 |
max_token_size=1024,
|
43 |
tiktoken_model="gpt-4o",
|
44 |
**kwargs,
|
45 |
-
):
|
46 |
tokens = encode_string_by_tiktoken(content, model_name=tiktoken_model)
|
47 |
-
results = []
|
48 |
if split_by_character:
|
49 |
raw_chunks = content.split(split_by_character)
|
50 |
new_chunks = []
|
|
|
2 |
import json
|
3 |
import re
|
4 |
from tqdm.asyncio import tqdm as tqdm_async
|
5 |
+
from typing import Any, Union
|
6 |
from collections import Counter, defaultdict
|
7 |
from .utils import (
|
8 |
logger,
|
|
|
42 |
max_token_size=1024,
|
43 |
tiktoken_model="gpt-4o",
|
44 |
**kwargs,
|
45 |
+
) -> list[dict[str, Any]]:
|
46 |
tokens = encode_string_by_tiktoken(content, model_name=tiktoken_model)
|
47 |
+
results: list[dict[str, Any]] = []
|
48 |
if split_by_character:
|
49 |
raw_chunks = content.split(split_by_character)
|
50 |
new_chunks = []
|