Spaces:
Runtime error
Runtime error
File size: 1,336 Bytes
e4f9cbe |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
"""Text splitters using spaCy."""
from typing import Any, Iterable, Optional
import spacy
from spacy import Language
from typing_extensions import override
from ...data.dataset_utils import lilac_span
from ...schema import Item, RichData
from ...signals.signal import TextSplitterSignal
class SentenceSplitterSpacy(TextSplitterSignal):
"""Splits documents into sentences using the SpaCy sentence tokenizer."""
name = 'sentences'
display_name = 'Sentence Splitter'
language: str = 'en'
_tokenizer: Language
def __init__(self, **kwargs: Any):
super().__init__(**kwargs)
self._tokenizer = spacy.blank(self.language)
self._tokenizer.add_pipe('sentencizer')
# Increase the number of characters of the tokenizer as we're not using a parser or NER.
self._tokenizer.max_length = 10_000_000
@override
def compute(self, data: Iterable[RichData]) -> Iterable[Optional[Item]]:
text_data = (row if isinstance(row, str) else '' for row in data)
for doc in self._tokenizer.pipe(text_data):
sentences = doc.sents
result = [lilac_span(token.start_char, token.end_char) for token in sentences]
if result:
yield result
else:
yield None
class Config:
# Language is required even though it has a default value.
schema_extra = {'required': ['language']}
|