Commit
•
9b24dfb
1
Parent(s):
355cfaa
Create textractor.py
Browse files- textractor.py +23 -0
textractor.py
ADDED
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"""
|
2 |
+
Textractor module
|
3 |
+
"""
|
4 |
+
|
5 |
+
from urllib.request import urlopen
|
6 |
+
from bs4 import BeautifulSoup
|
7 |
+
|
8 |
+
from txtai.pipeline.segmentation import Segmentation
|
9 |
+
|
10 |
+
|
11 |
+
class Textractor(Segmentation):
|
12 |
+
"""
|
13 |
+
Extracts text from files.
|
14 |
+
"""
|
15 |
+
|
16 |
+
def __init__(self, sentences=False, lines=False, paragraphs=False, minlength=None, join=False):
|
17 |
+
super().__init__(sentences, lines, paragraphs, minlength, join)
|
18 |
+
|
19 |
+
def text(self, text):
|
20 |
+
# text is a path to a file
|
21 |
+
html = urlopen(text).read()
|
22 |
+
soup = BeautifulSoup(html, features="html.parser")
|
23 |
+
return soup.get_text()
|