Hansimov commited on
Commit
6cf0820
0 Parent(s):

:gem: [Feature] New GoogleSearcher: Enable google search with query

Browse files
networks/__init__.py ADDED
File without changes
networks/google_searcher.py ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import requests
2
+ from pathlib import Path
3
+ from utils.enver import enver
4
+
5
+
6
+ class RequestHeaderConstructor:
7
+ def __init__(self):
8
+ self.construct()
9
+
10
+ def construct(self):
11
+ self.headers = {
12
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.62",
13
+ }
14
+
15
+
16
+ class GoogleSearcher:
17
+ # https://github.com/Nv7-GitHub/googlesearch/blob/master/googlesearch/__init__.py
18
+ def __init__(self):
19
+ self.url = "https://www.google.com/search"
20
+ self.enver = enver
21
+ self.enver.set_envs(proxies=True)
22
+ self.output_root = Path(__file__).parents[1] / "files"
23
+
24
+ def send_request(self, query, result_num=10):
25
+ res = requests.get(
26
+ url=self.url,
27
+ headers=RequestHeaderConstructor().headers,
28
+ params={
29
+ "q": self.query,
30
+ "num": result_num,
31
+ # "hl": "en",
32
+ # "start": 0,
33
+ },
34
+ proxies=self.enver.requests_proxies,
35
+ )
36
+ return res
37
+
38
+ def save_response(self, res, query):
39
+ output_filename = query.replace(" ", "_") + ".html"
40
+ if not self.output_root.exists():
41
+ self.output_root.mkdir(parents=True, exist_ok=True)
42
+ output_path = self.output_root / output_filename
43
+ with open(output_path, "wb") as wf:
44
+ wf.write(res.content)
45
+
46
+ def search(self, query):
47
+ self.query = query
48
+ res = self.send_request(query)
49
+ self.save_response(res, query)
50
+
51
+
52
+ if __name__ == "__main__":
53
+ searcher = GoogleSearcher()
54
+ # searcher.search("python tutorials")
55
+ searcher.search("python教程")