neon_arch commited on
Commit
1893e85
β€’
2 Parent(s): bb65e4e b18db54

Merge pull request #125 from MilimTheTrueOne/rolling

Browse files
CONTRIBUTING.md CHANGED
@@ -14,7 +14,7 @@ Know how to fix or improve a github action?. Consider Submitting a Pull request
14
 
15
  ## Source Code
16
 
17
- You should know atleast one of the things below to start contributing:
18
 
19
  - Rust basics
20
  - Actix-web crate basics
 
14
 
15
  ## Source Code
16
 
17
+ You should know at least one of the things below to start contributing:
18
 
19
  - Rust basics
20
  - Actix-web crate basics
PULL_REQUEST_TEMPLATE.md CHANGED
@@ -16,7 +16,7 @@
16
 
17
  ## Author's checklist
18
 
19
- <!-- additional notes for reviewiers -->
20
 
21
  ## Related issues
22
 
 
16
 
17
  ## Author's checklist
18
 
19
+ <!-- additional notes for reviewers -->
20
 
21
  ## Related issues
22
 
README.md CHANGED
@@ -59,7 +59,7 @@
59
  - **Community**
60
  - [πŸ“Š System Requirements](#system-requirements-)
61
  - [πŸ—¨οΈ FAQ (Frequently Asked Questions)](#faq-frequently-asked-questions-)
62
- - [πŸ“£ More Contributers Wanted](#more-contributers-wanted-)
63
  - [πŸ’– Supporting Websurfx](#supporting-websurfx-)
64
  - [πŸ“˜ Documentation](#documentation-)
65
  - [πŸ›£οΈ Roadmap](#roadmap-)
@@ -165,7 +165,7 @@ Websurfx is based on Rust due to its memory safety features, which prevents vuln
165
 
166
  **[⬆️ Back to Top](#--)**
167
 
168
- # More Contributers Wanted πŸ“£
169
 
170
  We are looking for more willing contributors to help grow this project. For more information on how you can contribute, check out the [project board](https://github.com/neon-mmd/websurfx/projects?query=is%3Aopen) and the [CONTRIBUTING.md](CONTRIBUTING.md) file for guidelines and rules for making contributions.
171
 
 
59
  - **Community**
60
  - [πŸ“Š System Requirements](#system-requirements-)
61
  - [πŸ—¨οΈ FAQ (Frequently Asked Questions)](#faq-frequently-asked-questions-)
62
+ - [πŸ“£ More Contributors Wanted](#more-contributors-wanted-)
63
  - [πŸ’– Supporting Websurfx](#supporting-websurfx-)
64
  - [πŸ“˜ Documentation](#documentation-)
65
  - [πŸ›£οΈ Roadmap](#roadmap-)
 
165
 
166
  **[⬆️ Back to Top](#--)**
167
 
168
+ # More Contributors Wanted πŸ“£
169
 
170
  We are looking for more willing contributors to help grow this project. For more information on how you can contribute, check out the [project board](https://github.com/neon-mmd/websurfx/projects?query=is%3Aopen) and the [CONTRIBUTING.md](CONTRIBUTING.md) file for guidelines and rules for making contributions.
171
 
public/static/pagination.js CHANGED
@@ -30,8 +30,8 @@ function navigate_backward() {
30
  let page = parseInt(searchParams.get('page'));
31
 
32
  if (isNaN(page)) {
33
- page = 1;
34
- } else if (page > 1) {
35
  page--;
36
  }
37
 
 
30
  let page = parseInt(searchParams.get('page'));
31
 
32
  if (isNaN(page)) {
33
+ page = 0;
34
+ } else if (page > 0) {
35
  page--;
36
  }
37
 
src/config_parser/parser.rs CHANGED
@@ -24,46 +24,35 @@ pub struct Config {
24
  pub binding_ip_addr: String,
25
  pub style: Style,
26
  pub redis_connection_url: String,
27
- pub aggregator: AggreatorConfig,
28
  pub logging: bool,
29
  pub debug: bool,
30
  }
31
 
32
  /// Configuration options for the aggregator.
33
  #[derive(Clone)]
34
- pub struct AggreatorConfig {
35
  /// Whether to introduce a random delay before sending the request to the search engine.
36
  pub random_delay: bool,
37
  }
38
 
39
  impl Config {
40
  /// A function which parses the config.lua file and puts all the parsed options in the newly
41
- /// contructed Config struct and returns it.
42
  ///
43
  /// # Error
44
  ///
45
  /// Returns a lua parse error if parsing of the config.lua file fails or has a syntax error
46
- /// or io error if the config.lua file doesn't exists otherwise it returns a newly contructed
47
  /// Config struct with all the parsed config options from the parsed config file.
48
  pub fn parse() -> Result<Self, Box<dyn std::error::Error>> {
49
  Lua::new().context(|context| -> Result<Self, Box<dyn std::error::Error>> {
50
  let globals = context.globals();
51
 
52
  context
53
- .load(&fs::read_to_string(
54
- Config::handle_different_config_file_path()?,
55
- )?)
56
  .exec()?;
57
 
58
- let production_use = globals.get::<_, bool>("production_use")?;
59
- let aggregator_config = if production_use {
60
- AggreatorConfig { random_delay: true }
61
- } else {
62
- AggreatorConfig {
63
- random_delay: false,
64
- }
65
- };
66
-
67
  Ok(Config {
68
  port: globals.get::<_, u16>("port")?,
69
  binding_ip_addr: globals.get::<_, String>("binding_ip_addr")?,
@@ -72,7 +61,9 @@ impl Config {
72
  globals.get::<_, String>("colorscheme")?,
73
  ),
74
  redis_connection_url: globals.get::<_, String>("redis_connection_url")?,
75
- aggregator: aggregator_config,
 
 
76
  logging: globals.get::<_, bool>("logging")?,
77
  debug: globals.get::<_, bool>("debug")?,
78
  })
@@ -90,35 +81,37 @@ impl Config {
90
  /// one (3).
91
  /// 3. `websurfx/` (under project folder ( or codebase in other words)) if it is not present
92
  /// here then it returns an error as mentioned above.
93
- fn handle_different_config_file_path() -> Result<String, Box<dyn std::error::Error>> {
94
- if Path::new(
95
- format!(
96
- "{}/.config/{}/config.lua",
97
- std::env::var("HOME").unwrap(),
98
- COMMON_DIRECTORY_NAME
99
- )
100
- .as_str(),
101
- )
102
- .exists()
103
- {
104
- Ok(format!(
105
  "{}/.config/{}/{}",
106
  std::env::var("HOME").unwrap(),
107
  COMMON_DIRECTORY_NAME,
108
  CONFIG_FILE_NAME
109
- ))
110
- } else if Path::new(
111
- format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME).as_str(),
112
- )
113
- .exists()
114
- {
115
- Ok("/etc/xdg/websurfx/config.lua".to_string())
116
- } else if Path::new(format!("./{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME).as_str())
117
  .exists()
118
  {
119
- Ok("./websurfx/config.lua".to_string())
120
- } else {
121
- Err("Config file not found!!".to_string().into())
122
  }
 
 
 
 
 
 
 
 
 
123
  }
124
  }
 
24
  pub binding_ip_addr: String,
25
  pub style: Style,
26
  pub redis_connection_url: String,
27
+ pub aggregator: AggregatorConfig,
28
  pub logging: bool,
29
  pub debug: bool,
30
  }
31
 
32
  /// Configuration options for the aggregator.
33
  #[derive(Clone)]
34
+ pub struct AggregatorConfig {
35
  /// Whether to introduce a random delay before sending the request to the search engine.
36
  pub random_delay: bool,
37
  }
38
 
39
  impl Config {
40
  /// A function which parses the config.lua file and puts all the parsed options in the newly
41
+ /// constructed Config struct and returns it.
42
  ///
43
  /// # Error
44
  ///
45
  /// Returns a lua parse error if parsing of the config.lua file fails or has a syntax error
46
+ /// or io error if the config.lua file doesn't exists otherwise it returns a newly constructed
47
  /// Config struct with all the parsed config options from the parsed config file.
48
  pub fn parse() -> Result<Self, Box<dyn std::error::Error>> {
49
  Lua::new().context(|context| -> Result<Self, Box<dyn std::error::Error>> {
50
  let globals = context.globals();
51
 
52
  context
53
+ .load(&fs::read_to_string(Config::get_config_path()?)?)
 
 
54
  .exec()?;
55
 
 
 
 
 
 
 
 
 
 
56
  Ok(Config {
57
  port: globals.get::<_, u16>("port")?,
58
  binding_ip_addr: globals.get::<_, String>("binding_ip_addr")?,
 
61
  globals.get::<_, String>("colorscheme")?,
62
  ),
63
  redis_connection_url: globals.get::<_, String>("redis_connection_url")?,
64
+ aggregator: AggregatorConfig {
65
+ random_delay: globals.get::<_, bool>("production_use")?,
66
+ },
67
  logging: globals.get::<_, bool>("logging")?,
68
  debug: globals.get::<_, bool>("debug")?,
69
  })
 
81
  /// one (3).
82
  /// 3. `websurfx/` (under project folder ( or codebase in other words)) if it is not present
83
  /// here then it returns an error as mentioned above.
84
+ fn get_config_path() -> Result<String, Box<dyn std::error::Error>> {
85
+ // check user config
86
+
87
+ let path = format!(
88
+ "{}/.config/{}/config.lua",
89
+ std::env::var("HOME").unwrap(),
90
+ COMMON_DIRECTORY_NAME
91
+ );
92
+ if Path::new(path.as_str()).exists() {
93
+ return Ok(format!(
 
 
94
  "{}/.config/{}/{}",
95
  std::env::var("HOME").unwrap(),
96
  COMMON_DIRECTORY_NAME,
97
  CONFIG_FILE_NAME
98
+ ));
99
+ }
100
+
101
+ // look for config in /etc/xdg
102
+ if Path::new(format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME).as_str())
 
 
 
103
  .exists()
104
  {
105
+ return Ok("/etc/xdg/websurfx/config.lua".to_string());
 
 
106
  }
107
+
108
+ // use dev config
109
+ if Path::new(format!("./{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME).as_str()).exists()
110
+ {
111
+ return Ok("./websurfx/config.lua".to_string());
112
+ }
113
+
114
+ // if no of the configs above exist, return error
115
+ Err("Config file not found!!".to_string().into())
116
  }
117
  }
src/config_parser/parser_models.rs CHANGED
@@ -1,5 +1,5 @@
1
  //! This module provides public models for handling, storing and serializing parsed config file
2
- //! options from config.lua by grouping them togather.
3
 
4
  use serde::{Deserialize, Serialize};
5
 
 
1
  //! This module provides public models for handling, storing and serializing parsed config file
2
+ //! options from config.lua by grouping them together.
3
 
4
  use serde::{Deserialize, Serialize};
5
 
src/engines/duckduckgo.rs CHANGED
@@ -36,7 +36,7 @@ pub async fn results(
36
  user_agent: &str,
37
  ) -> Result<HashMap<String, RawSearchResult>, EngineError> {
38
  // Page number can be missing or empty string and so appropriate handling is required
39
- // so that upstream server recieves valid page number.
40
  let url: String = match page {
41
  1 => {
42
  format!("https://html.duckduckgo.com/html/?q={query}&s=&dc=&v=1&o=json&api=/d.js")
@@ -86,7 +86,7 @@ pub async fn results(
86
  let results: String = reqwest::Client::new()
87
  .get(url)
88
  .timeout(Duration::from_secs(5))
89
- .headers(header_map) // add spoofed headers to emulate human behaviour
90
  .send()
91
  .await
92
  .into_report()
 
36
  user_agent: &str,
37
  ) -> Result<HashMap<String, RawSearchResult>, EngineError> {
38
  // Page number can be missing or empty string and so appropriate handling is required
39
+ // so that upstream server receives valid page number.
40
  let url: String = match page {
41
  1 => {
42
  format!("https://html.duckduckgo.com/html/?q={query}&s=&dc=&v=1&o=json&api=/d.js")
 
86
  let results: String = reqwest::Client::new()
87
  .get(url)
88
  .timeout(Duration::from_secs(5))
89
+ .headers(header_map) // add spoofed headers to emulate human behavior
90
  .send()
91
  .await
92
  .into_report()
src/engines/engine_models.rs CHANGED
@@ -13,7 +13,7 @@ use std::fmt;
13
  /// search engines.
14
  /// * `UnexpectedError` - This variant handles all the errors which are unexpected or occur rarely
15
  /// and are errors mostly related to failure in initialization of HeaderMap, Selector errors and
16
- /// all other errors occuring within the code handling the `upstream search engines`.
17
  #[derive(Debug)]
18
  pub enum EngineError {
19
  EmptyResultSet,
 
13
  /// search engines.
14
  /// * `UnexpectedError` - This variant handles all the errors which are unexpected or occur rarely
15
  /// and are errors mostly related to failure in initialization of HeaderMap, Selector errors and
16
+ /// all other errors occurring within the code handling the `upstream search engines`.
17
  #[derive(Debug)]
18
  pub enum EngineError {
19
  EmptyResultSet,
src/search_results_handler/aggregator.rs CHANGED
@@ -17,7 +17,7 @@ use crate::engines::{duckduckgo, searx};
17
  /// then removes duplicate results and if two results are found to be from two or more engines
18
  /// then puts their names together to show the results are fetched from these upstream engines
19
  /// and then removes all data from the HashMap and puts into a struct of all results aggregated
20
- /// into a vector and also adds the query used into the struct this is neccessory because
21
  /// otherwise the search bar in search remains empty if searched from the query url
22
  ///
23
  /// # Example:
 
17
  /// then removes duplicate results and if two results are found to be from two or more engines
18
  /// then puts their names together to show the results are fetched from these upstream engines
19
  /// and then removes all data from the HashMap and puts into a struct of all results aggregated
20
+ /// into a vector and also adds the query used into the struct this is necessary because
21
  /// otherwise the search bar in search remains empty if searched from the query url
22
  ///
23
  /// # Example:
src/server/routes.rs CHANGED
@@ -1,5 +1,5 @@
1
  //! This module provides the functionality to handle different routes of the `websurfx`
2
- //! meta search engine website and provide approriate response to each route/page
3
  //! when requested.
4
 
5
  use std::fs::read_to_string;
@@ -82,40 +82,16 @@ pub async fn search(
82
  .insert_header(("location", "/"))
83
  .finish())
84
  } else {
85
- let page_url: String; // Declare the page_url variable without initializing it
86
-
87
- // ...
88
-
89
- let page = match params.page {
90
- Some(page_number) => {
91
- if page_number <= 1 {
92
- page_url = format!(
93
- "http://{}:{}/search?q={}&page={}",
94
- config.binding_ip_addr, config.port, query, 1
95
- );
96
- 1
97
- } else {
98
- page_url = format!(
99
- "http://{}:{}/search?q={}&page={}",
100
- config.binding_ip_addr, config.port, query, page_number
101
- );
102
-
103
- page_number
104
- }
105
- }
106
- None => {
107
- page_url = format!(
108
- "http://{}:{}{}&page={}",
109
- config.binding_ip_addr,
110
- config.port,
111
- req.uri(),
112
- 1
113
- );
114
-
115
- 1
116
- }
117
  };
118
 
 
 
 
 
 
119
  // fetch the cached results json.
120
  let cached_results_json = redis_cache.cached_results_json(&page_url);
121
  // check if fetched results was indeed fetched or it was an error and if so
 
1
  //! This module provides the functionality to handle different routes of the `websurfx`
2
+ //! meta search engine website and provide appropriate response to each route/page
3
  //! when requested.
4
 
5
  use std::fs::read_to_string;
 
82
  .insert_header(("location", "/"))
83
  .finish())
84
  } else {
85
+ let page = match &params.page {
86
+ Some(page) => *page,
87
+ None => 0,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
  };
89
 
90
+ let page_url = format!(
91
+ "http://{}:{}/search?q={}&page={}",
92
+ config.binding_ip_addr, config.port, query, page
93
+ );
94
+
95
  // fetch the cached results json.
96
  let cached_results_json = redis_cache.cached_results_json(&page_url);
97
  // check if fetched results was indeed fetched or it was an error and if so