m-ric HF staff commited on
Commit
262dfde
1 Parent(s): 133d9a7

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +4 -1
app.py CHANGED
@@ -35,6 +35,7 @@ def change_split_selection(split_selection):
35
 
36
  def chunk(text, length, splitter_selection, separators_str, length_unit_selection, chunk_overlap):
37
  separators = extract_separators_from_string(separators_str)
 
38
  length_function = (length_tokens if "token" in length_unit_selection.lower() else len)
39
  if splitter_selection == LABEL_TEXTSPLITTER:
40
  text_splitter = CharacterTextSplitter(
@@ -63,12 +64,14 @@ def chunk(text, length, splitter_selection, separators_str, length_unit_selectio
63
 
64
  def change_preset_separators(choice):
65
  text_splitter = RecursiveCharacterTextSplitter()
66
- if choice == "Default":
67
  return ["\n\n", "\n", " ", ""]
68
  elif choice == "Markdown":
69
  return text_splitter.get_separators_for_language(Language.MARKDOWN)
70
  elif choice == "Python":
71
  return text_splitter.get_separators_for_language(Language.PYTHON)
 
 
72
 
73
 
74
  EXAMPLE_TEXT = """### Chapter 6
 
35
 
36
  def chunk(text, length, splitter_selection, separators_str, length_unit_selection, chunk_overlap):
37
  separators = extract_separators_from_string(separators_str)
38
+ print('Extracted separators:', ',,'.join(separators))
39
  length_function = (length_tokens if "token" in length_unit_selection.lower() else len)
40
  if splitter_selection == LABEL_TEXTSPLITTER:
41
  text_splitter = CharacterTextSplitter(
 
64
 
65
  def change_preset_separators(choice):
66
  text_splitter = RecursiveCharacterTextSplitter()
67
+ if choice == "Default recursive":
68
  return ["\n\n", "\n", " ", ""]
69
  elif choice == "Markdown":
70
  return text_splitter.get_separators_for_language(Language.MARKDOWN)
71
  elif choice == "Python":
72
  return text_splitter.get_separators_for_language(Language.PYTHON)
73
+ else:
74
+ raise gr.Error("Choice of preset not recognized.")
75
 
76
 
77
  EXAMPLE_TEXT = """### Chapter 6