File size: 4,033 Bytes
df47775 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 |
import streamlit as st
import re
# Define the split functions
def recursive_splitter(data):
paragraphs = data.split('\n\n')
sentences = [sentence for para in paragraphs for sentence in para.split('.')]
return [sentence.strip() + '.' for sentence in sentences if sentence.strip()]
def html_splitter(data):
parts = re.split(r'(<[^>]+>)', data)
return [part for part in parts if part.strip()]
def markdown_splitter(data):
parts = re.split(r'(^#{1,6} .*$)', data, flags=re.MULTILINE)
return [part.strip() for part in parts if part.strip()]
def code_splitter(data):
parts = re.split(r'(?m)^def ', data)
return [f'def {part.strip()}' if idx > 0 else part.strip() for idx, part in enumerate(parts) if part.strip()]
def token_splitter(data):
tokens = re.findall(r'\b\w+\b', data)
return tokens
def character_splitter(data):
return list(data)
def semantic_chunker(data):
sentences = re.split(r'(?<=\.)\s+', data)
return [sentence.strip() for sentence in sentences if sentence.strip()]
# Mapping splitter names to functions and descriptions
splitter_details = {
"Recursive Splitter": {
"function": recursive_splitter,
"description": "Recursively splits the data into smaller chunks, like paragraphs into sentences. Useful for processing text at different levels of granularity."
},
"HTML Splitter": {
"function": html_splitter,
"description": "Splits data based on HTML tags, making it easier to work with structured web content, such as isolating specific sections of HTML code."
},
"Markdown Splitter": {
"function": markdown_splitter,
"description": "Splits markdown content based on headings (e.g., '# ', '## '). Useful for processing documents written in Markdown format."
},
"Code Splitter": {
"function": code_splitter,
"description": "Splits programming code into logical blocks like functions or classes. Useful for code analysis and documentation."
},
"Token Splitter": {
"function": token_splitter,
"description": "Splits data into individual tokens/words, which is often the first step in natural language processing (NLP) tasks."
},
"Character Splitter": {
"function": character_splitter,
"description": "Splits text into individual characters. Useful for character-level analysis or encoding tasks."
},
"Semantic Chunker": {
"function": semantic_chunker,
"description": "Splits data based on semantic meaning, typically by sentences. Ensures that related information stays together."
},
}
# Streamlit app
st.sidebar.title("Splitter Settings")
st.sidebar.subheader("Data Input")
user_data = st.sidebar.text_area("Enter the data you want to split:", "This is a sample text. Enter your data here...")
st.sidebar.subheader("Splitter Type")
splitter_type = st.sidebar.selectbox(
"Choose a splitter type:",
list(splitter_details.keys())
)
st.sidebar.subheader("Options")
show_info = st.sidebar.checkbox("Show information about all splitter types")
st.title("RAG Splitter System")
st.markdown('<p class="title">Developed By: Irfan Ullah Khan</p>', unsafe_allow_html=True)
# Display selected splitter description
st.subheader(f"Selected Splitter: {splitter_type}")
st.write(splitter_details[splitter_type]["description"])
# Processing
if st.button("Split Data"):
with st.spinner('Processing data...'):
splitter_function = splitter_details[splitter_type]["function"]
split_output = splitter_function(user_data)
if split_output:
st.subheader(f"Output using {splitter_type}")
for idx, part in enumerate(split_output):
st.write(f"**Part {idx + 1}:**")
st.write(part)
if show_info:
for name, details in splitter_details.items():
st.subheader(name)
st.write(details["description"])
|