| from langchain.docstore.document import Document | |
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from logging import getLogger | |
| logger = getLogger(__name__) | |
| def get_input_token_count(text:str,tokenizer)->int: | |
| tokens = tokenizer.tokenize(text) | |
| return len(tokens) | |
| def get_document_splits_from_text(text:str) -> Document: | |
| document = Document(page_content=text) | |
| text_splitter = RecursiveCharacterTextSplitter( | |
| separators=["\n\n","\n",".","?"," "], | |
| chunk_size=15000, | |
| chunk_overlap = 50 | |
| ) | |
| split_documents = text_splitter.split_documents([document]) | |
| logger.info(f"Splitting Document: Total Chunks: {len(split_documents)} ") | |
| return split_documents | |
| def prepare_for_summarize(text:str,tokenizer): | |
| no_input_tokens = get_input_token_count(text,tokenizer) | |
| if no_input_tokens<12000: | |
| text_to_summarize = text | |
| length_type = "short" | |
| return text_to_summarize,length_type | |
| else: | |
| text_to_summarize = get_document_splits_from_text(text) | |
| length_type = "long" | |
| return text_to_summarize, length_type | |