|
class Doc: |
|
def __init__(self, fulltext: str = '', title: str = '', params: dict = {}): |
|
self.params = params |
|
self.lines = [Line(text.strip(), self.params) for text in fulltext.split("\n") if text.strip()] |
|
self.title, self.lines = self._get_title(title) |
|
self.container = Container(lines=self.lines, title=self.title, father=self, params=params) |
|
self.fulltext = fulltext |
|
|
|
def _get_title(self, title): |
|
lines = self.lines |
|
if self.params['type'] == 'input_text': |
|
if self.lines and self.lines[0] and self.lines[0].type == 'title': |
|
title = self.lines[0].text |
|
lines = lines[1:] |
|
else: |
|
title = 'the title is missing' |
|
return title, lines |
|
|
|
|
|
class WikiPage(Doc): |
|
|
|
def __init__(self, fulltext='', title=''): |
|
self.params = { |
|
'type': 'wiki', |
|
'startswith_': |
|
{'== ': '1', '=== ': '2', '==== ': '3', '===== ': '4', '====== ': '5', '======= ': '6'}, |
|
'endswith_': |
|
[' ==', ' ===', ' ====', ' =====', ' ======', ' ======'], |
|
|
|
'discarded': ["See also", "Notes", "References", "Sources", "External links", "Bibliography", |
|
"Cinematic adaptations", "Further reading", "Maps"] |
|
} |
|
super().__init__(fulltext=fulltext, title=title, params=self.params) |
|
|
|
def get_paragraphs(self, chunk=500): |
|
return self.container.get_paragraphs(chunk) |
|
|
|
|
|
class Container: |
|
|
|
def __init__(self, lines=[], level=0, title='', father=None, params={}): |
|
|
|
self.children = [] |
|
self.level = level |
|
self.title = title |
|
self.father = father |
|
self.lines = [] |
|
self._expand(lines) |
|
if params and 'discarded' in params.keys(): |
|
self.children = [child for child in self.children if child.title not in params['discarded']] |
|
self.containers = [self] |
|
for child in self.children: |
|
self.containers += child.containers |
|
self.text = '' |
|
for child in self.children: |
|
self.text += ' ' + child.text |
|
|
|
def _expand(self, lines): |
|
new_child = False |
|
new_child_lines = [] |
|
new_child_title = [] |
|
for line in lines: |
|
if not new_child: |
|
if line.is_structure: |
|
new_child = True |
|
new_child_lines = [] |
|
new_child_title = line.text |
|
line.level = self.level + 1 |
|
else: |
|
self.lines.append(line) |
|
|
|
else: |
|
if self.level + 1 < line.level or not line.is_structure: |
|
new_child_lines.append(line) |
|
elif self.level + 1 == line.level: |
|
self.children.append(Container(lines=new_child_lines, |
|
level=self.level + 1, |
|
title=new_child_title, |
|
father=self)) |
|
new_child_lines = [] |
|
new_child_title = line.text |
|
if new_child: |
|
self.children.append(Container(lines=new_child_lines, |
|
level=self.level + 1, |
|
title=new_child_title, |
|
father=self)) |
|
|
|
def get_paragraphs(self, chunk=500): |
|
if len(self.text) < chunk: |
|
paragraphs = [self.text] |
|
else: |
|
paragraphs = [self.root_text] |
|
for child in self.children: |
|
paragraphs += child.get_paragraphs(chunk) |
|
return paragraphs |
|
|
|
|
|
class Line: |
|
|
|
def __init__(self, text, params): |
|
self.text = text |
|
self.params = params |
|
self.type, self.text = self._parse_text() |
|
self.level = int(self.type) if self.type.isdigit() else -1 |
|
self.is_structure = 0 < self.level |
|
|
|
|
|
def _parse_text(self): |
|
def strip_text(text_, start, end): |
|
text_ = text_.split(start)[1] |
|
if end != "": |
|
text_ = text_.split(end)[0] |
|
|
|
return text_.strip() |
|
|
|
startswith_ = self.params['startswith_'] |
|
|
|
endswith_ = self.params['endswith_'] if 'endswith_' in self.params.keys() else [""] * len(startswith_) |
|
types = [(strip_text(self.text, starter, endswith_[i]), startswith_[starter]) |
|
for i, starter in enumerate(startswith_.keys()) |
|
if self.text.startswith(starter)] |
|
(text, type_) = types[0] if types else (self.text, 'normal') |
|
return type_, text.strip() |
|
|
