neon_arch commited on
Commit
fe74f2e
1 Parent(s): b42adaa

♻️ refactor: code to take prebuilt client to request search results (#384)

Browse files
src/engines/brave.rs CHANGED
@@ -4,7 +4,7 @@
4
 
5
  use std::collections::HashMap;
6
 
7
- use reqwest::header::HeaderMap;
8
  use scraper::Html;
9
 
10
  use crate::models::aggregation_models::SearchResult;
@@ -42,7 +42,7 @@ impl SearchEngine for Brave {
42
  query: &str,
43
  page: u32,
44
  user_agent: &str,
45
- request_timeout: u8,
46
  safe_search: u8,
47
  ) -> Result<HashMap<String, SearchResult>, EngineError> {
48
  let url = format!("https://search.brave.com/search?q={query}&offset={page}");
@@ -68,7 +68,7 @@ impl SearchEngine for Brave {
68
  .change_context(EngineError::UnexpectedError)?;
69
 
70
  let document: Html = Html::parse_document(
71
- &Brave::fetch_html_from_upstream(self, &url, header_map, request_timeout).await?,
72
  );
73
 
74
  if let Some(no_result_msg) = self.parser.parse_for_no_results(&document).nth(0) {
 
4
 
5
  use std::collections::HashMap;
6
 
7
+ use reqwest::{header::HeaderMap, Client};
8
  use scraper::Html;
9
 
10
  use crate::models::aggregation_models::SearchResult;
 
42
  query: &str,
43
  page: u32,
44
  user_agent: &str,
45
+ client: &Client,
46
  safe_search: u8,
47
  ) -> Result<HashMap<String, SearchResult>, EngineError> {
48
  let url = format!("https://search.brave.com/search?q={query}&offset={page}");
 
68
  .change_context(EngineError::UnexpectedError)?;
69
 
70
  let document: Html = Html::parse_document(
71
+ &Brave::fetch_html_from_upstream(self, &url, header_map, client).await?,
72
  );
73
 
74
  if let Some(no_result_msg) = self.parser.parse_for_no_results(&document).nth(0) {
src/engines/duckduckgo.rs CHANGED
@@ -5,6 +5,7 @@
5
  use std::collections::HashMap;
6
 
7
  use reqwest::header::HeaderMap;
 
8
  use scraper::Html;
9
 
10
  use crate::models::aggregation_models::SearchResult;
@@ -44,7 +45,7 @@ impl SearchEngine for DuckDuckGo {
44
  query: &str,
45
  page: u32,
46
  user_agent: &str,
47
- request_timeout: u8,
48
  _safe_search: u8,
49
  ) -> Result<HashMap<String, SearchResult>, EngineError> {
50
  // Page number can be missing or empty string and so appropriate handling is required
@@ -76,7 +77,7 @@ impl SearchEngine for DuckDuckGo {
76
  .change_context(EngineError::UnexpectedError)?;
77
 
78
  let document: Html = Html::parse_document(
79
- &DuckDuckGo::fetch_html_from_upstream(self, &url, header_map, request_timeout).await?,
80
  );
81
 
82
  if self.parser.parse_for_no_results(&document).next().is_some() {
 
5
  use std::collections::HashMap;
6
 
7
  use reqwest::header::HeaderMap;
8
+ use reqwest::Client;
9
  use scraper::Html;
10
 
11
  use crate::models::aggregation_models::SearchResult;
 
45
  query: &str,
46
  page: u32,
47
  user_agent: &str,
48
+ client: &Client,
49
  _safe_search: u8,
50
  ) -> Result<HashMap<String, SearchResult>, EngineError> {
51
  // Page number can be missing or empty string and so appropriate handling is required
 
77
  .change_context(EngineError::UnexpectedError)?;
78
 
79
  let document: Html = Html::parse_document(
80
+ &DuckDuckGo::fetch_html_from_upstream(self, &url, header_map, client).await?,
81
  );
82
 
83
  if self.parser.parse_for_no_results(&document).next().is_some() {
src/engines/searx.rs CHANGED
@@ -3,6 +3,7 @@
3
  //! number if provided.
4
 
5
  use reqwest::header::HeaderMap;
 
6
  use scraper::Html;
7
  use std::collections::HashMap;
8
 
@@ -40,7 +41,7 @@ impl SearchEngine for Searx {
40
  query: &str,
41
  page: u32,
42
  user_agent: &str,
43
- request_timeout: u8,
44
  mut safe_search: u8,
45
  ) -> Result<HashMap<String, SearchResult>, EngineError> {
46
  // Page number can be missing or empty string and so appropriate handling is required
@@ -68,7 +69,7 @@ impl SearchEngine for Searx {
68
  .change_context(EngineError::UnexpectedError)?;
69
 
70
  let document: Html = Html::parse_document(
71
- &Searx::fetch_html_from_upstream(self, &url, header_map, request_timeout).await?,
72
  );
73
 
74
  if let Some(no_result_msg) = self.parser.parse_for_no_results(&document).nth(1) {
 
3
  //! number if provided.
4
 
5
  use reqwest::header::HeaderMap;
6
+ use reqwest::Client;
7
  use scraper::Html;
8
  use std::collections::HashMap;
9
 
 
41
  query: &str,
42
  page: u32,
43
  user_agent: &str,
44
+ client: &Client,
45
  mut safe_search: u8,
46
  ) -> Result<HashMap<String, SearchResult>, EngineError> {
47
  // Page number can be missing or empty string and so appropriate handling is required
 
69
  .change_context(EngineError::UnexpectedError)?;
70
 
71
  let document: Html = Html::parse_document(
72
+ &Searx::fetch_html_from_upstream(self, &url, header_map, client).await?,
73
  );
74
 
75
  if let Some(no_result_msg) = self.parser.parse_for_no_results(&document).nth(1) {