davidmezzetti commited on
Commit
daa1a9a
1 Parent(s): 9b24dfb

Update textractor.py

Browse files
Files changed (1) hide show
  1. textractor.py +6 -4
textractor.py CHANGED
@@ -2,12 +2,12 @@
2
  Textractor module
3
  """
4
 
5
- from urllib.request import urlopen
 
6
  from bs4 import BeautifulSoup
7
 
8
  from txtai.pipeline.segmentation import Segmentation
9
 
10
-
11
  class Textractor(Segmentation):
12
  """
13
  Extracts text from files.
@@ -17,7 +17,9 @@ class Textractor(Segmentation):
17
  super().__init__(sentences, lines, paragraphs, minlength, join)
18
 
19
  def text(self, text):
20
- # text is a path to a file
21
- html = urlopen(text).read()
 
 
22
  soup = BeautifulSoup(html, features="html.parser")
23
  return soup.get_text()
 
2
  Textractor module
3
  """
4
 
5
+ import requests
6
+
7
  from bs4 import BeautifulSoup
8
 
9
  from txtai.pipeline.segmentation import Segmentation
10
 
 
11
  class Textractor(Segmentation):
12
  """
13
  Extracts text from files.
 
17
  super().__init__(sentences, lines, paragraphs, minlength, join)
18
 
19
  def text(self, text):
20
+ # text is a url
21
+ response = requests.get(text)
22
+ html = response.text
23
+
24
  soup = BeautifulSoup(html, features="html.parser")
25
  return soup.get_text()