Spaces:
Runtime error
Runtime error
Merge pull request #125 from MilimTheTrueOne/rolling
Browse files- CONTRIBUTING.md +1 -1
- PULL_REQUEST_TEMPLATE.md +1 -1
- README.md +2 -2
- public/static/pagination.js +2 -2
- src/config_parser/parser.rs +33 -40
- src/config_parser/parser_models.rs +1 -1
- src/engines/duckduckgo.rs +2 -2
- src/engines/engine_models.rs +1 -1
- src/search_results_handler/aggregator.rs +1 -1
- src/server/routes.rs +9 -33
CONTRIBUTING.md
CHANGED
@@ -14,7 +14,7 @@ Know how to fix or improve a github action?. Consider Submitting a Pull request
|
|
14 |
|
15 |
## Source Code
|
16 |
|
17 |
-
You should know
|
18 |
|
19 |
- Rust basics
|
20 |
- Actix-web crate basics
|
|
|
14 |
|
15 |
## Source Code
|
16 |
|
17 |
+
You should know at least one of the things below to start contributing:
|
18 |
|
19 |
- Rust basics
|
20 |
- Actix-web crate basics
|
PULL_REQUEST_TEMPLATE.md
CHANGED
@@ -16,7 +16,7 @@
|
|
16 |
|
17 |
## Author's checklist
|
18 |
|
19 |
-
<!-- additional notes for
|
20 |
|
21 |
## Related issues
|
22 |
|
|
|
16 |
|
17 |
## Author's checklist
|
18 |
|
19 |
+
<!-- additional notes for reviewers -->
|
20 |
|
21 |
## Related issues
|
22 |
|
README.md
CHANGED
@@ -59,7 +59,7 @@
|
|
59 |
- **Community**
|
60 |
- [π System Requirements](#system-requirements-)
|
61 |
- [π¨οΈ FAQ (Frequently Asked Questions)](#faq-frequently-asked-questions-)
|
62 |
-
- [π£ More
|
63 |
- [π Supporting Websurfx](#supporting-websurfx-)
|
64 |
- [π Documentation](#documentation-)
|
65 |
- [π£οΈ Roadmap](#roadmap-)
|
@@ -165,7 +165,7 @@ Websurfx is based on Rust due to its memory safety features, which prevents vuln
|
|
165 |
|
166 |
**[β¬οΈ Back to Top](#--)**
|
167 |
|
168 |
-
# More
|
169 |
|
170 |
We are looking for more willing contributors to help grow this project. For more information on how you can contribute, check out the [project board](https://github.com/neon-mmd/websurfx/projects?query=is%3Aopen) and the [CONTRIBUTING.md](CONTRIBUTING.md) file for guidelines and rules for making contributions.
|
171 |
|
|
|
59 |
- **Community**
|
60 |
- [π System Requirements](#system-requirements-)
|
61 |
- [π¨οΈ FAQ (Frequently Asked Questions)](#faq-frequently-asked-questions-)
|
62 |
+
- [π£ More Contributors Wanted](#more-contributors-wanted-)
|
63 |
- [π Supporting Websurfx](#supporting-websurfx-)
|
64 |
- [π Documentation](#documentation-)
|
65 |
- [π£οΈ Roadmap](#roadmap-)
|
|
|
165 |
|
166 |
**[β¬οΈ Back to Top](#--)**
|
167 |
|
168 |
+
# More Contributors Wanted π£
|
169 |
|
170 |
We are looking for more willing contributors to help grow this project. For more information on how you can contribute, check out the [project board](https://github.com/neon-mmd/websurfx/projects?query=is%3Aopen) and the [CONTRIBUTING.md](CONTRIBUTING.md) file for guidelines and rules for making contributions.
|
171 |
|
public/static/pagination.js
CHANGED
@@ -30,8 +30,8 @@ function navigate_backward() {
|
|
30 |
let page = parseInt(searchParams.get('page'));
|
31 |
|
32 |
if (isNaN(page)) {
|
33 |
-
page =
|
34 |
-
} else if (page >
|
35 |
page--;
|
36 |
}
|
37 |
|
|
|
30 |
let page = parseInt(searchParams.get('page'));
|
31 |
|
32 |
if (isNaN(page)) {
|
33 |
+
page = 0;
|
34 |
+
} else if (page > 0) {
|
35 |
page--;
|
36 |
}
|
37 |
|
src/config_parser/parser.rs
CHANGED
@@ -24,46 +24,35 @@ pub struct Config {
|
|
24 |
pub binding_ip_addr: String,
|
25 |
pub style: Style,
|
26 |
pub redis_connection_url: String,
|
27 |
-
pub aggregator:
|
28 |
pub logging: bool,
|
29 |
pub debug: bool,
|
30 |
}
|
31 |
|
32 |
/// Configuration options for the aggregator.
|
33 |
#[derive(Clone)]
|
34 |
-
pub struct
|
35 |
/// Whether to introduce a random delay before sending the request to the search engine.
|
36 |
pub random_delay: bool,
|
37 |
}
|
38 |
|
39 |
impl Config {
|
40 |
/// A function which parses the config.lua file and puts all the parsed options in the newly
|
41 |
-
///
|
42 |
///
|
43 |
/// # Error
|
44 |
///
|
45 |
/// Returns a lua parse error if parsing of the config.lua file fails or has a syntax error
|
46 |
-
/// or io error if the config.lua file doesn't exists otherwise it returns a newly
|
47 |
/// Config struct with all the parsed config options from the parsed config file.
|
48 |
pub fn parse() -> Result<Self, Box<dyn std::error::Error>> {
|
49 |
Lua::new().context(|context| -> Result<Self, Box<dyn std::error::Error>> {
|
50 |
let globals = context.globals();
|
51 |
|
52 |
context
|
53 |
-
.load(&fs::read_to_string(
|
54 |
-
Config::handle_different_config_file_path()?,
|
55 |
-
)?)
|
56 |
.exec()?;
|
57 |
|
58 |
-
let production_use = globals.get::<_, bool>("production_use")?;
|
59 |
-
let aggregator_config = if production_use {
|
60 |
-
AggreatorConfig { random_delay: true }
|
61 |
-
} else {
|
62 |
-
AggreatorConfig {
|
63 |
-
random_delay: false,
|
64 |
-
}
|
65 |
-
};
|
66 |
-
|
67 |
Ok(Config {
|
68 |
port: globals.get::<_, u16>("port")?,
|
69 |
binding_ip_addr: globals.get::<_, String>("binding_ip_addr")?,
|
@@ -72,7 +61,9 @@ impl Config {
|
|
72 |
globals.get::<_, String>("colorscheme")?,
|
73 |
),
|
74 |
redis_connection_url: globals.get::<_, String>("redis_connection_url")?,
|
75 |
-
aggregator:
|
|
|
|
|
76 |
logging: globals.get::<_, bool>("logging")?,
|
77 |
debug: globals.get::<_, bool>("debug")?,
|
78 |
})
|
@@ -90,35 +81,37 @@ impl Config {
|
|
90 |
/// one (3).
|
91 |
/// 3. `websurfx/` (under project folder ( or codebase in other words)) if it is not present
|
92 |
/// here then it returns an error as mentioned above.
|
93 |
-
fn
|
94 |
-
|
95 |
-
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
-
|
100 |
-
|
101 |
-
)
|
102 |
-
|
103 |
-
{
|
104 |
-
Ok(format!(
|
105 |
"{}/.config/{}/{}",
|
106 |
std::env::var("HOME").unwrap(),
|
107 |
COMMON_DIRECTORY_NAME,
|
108 |
CONFIG_FILE_NAME
|
109 |
-
))
|
110 |
-
}
|
111 |
-
|
112 |
-
|
113 |
-
.
|
114 |
-
{
|
115 |
-
Ok("/etc/xdg/websurfx/config.lua".to_string())
|
116 |
-
} else if Path::new(format!("./{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME).as_str())
|
117 |
.exists()
|
118 |
{
|
119 |
-
Ok("
|
120 |
-
} else {
|
121 |
-
Err("Config file not found!!".to_string().into())
|
122 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
123 |
}
|
124 |
}
|
|
|
24 |
pub binding_ip_addr: String,
|
25 |
pub style: Style,
|
26 |
pub redis_connection_url: String,
|
27 |
+
pub aggregator: AggregatorConfig,
|
28 |
pub logging: bool,
|
29 |
pub debug: bool,
|
30 |
}
|
31 |
|
32 |
/// Configuration options for the aggregator.
|
33 |
#[derive(Clone)]
|
34 |
+
pub struct AggregatorConfig {
|
35 |
/// Whether to introduce a random delay before sending the request to the search engine.
|
36 |
pub random_delay: bool,
|
37 |
}
|
38 |
|
39 |
impl Config {
|
40 |
/// A function which parses the config.lua file and puts all the parsed options in the newly
|
41 |
+
/// constructed Config struct and returns it.
|
42 |
///
|
43 |
/// # Error
|
44 |
///
|
45 |
/// Returns a lua parse error if parsing of the config.lua file fails or has a syntax error
|
46 |
+
/// or io error if the config.lua file doesn't exists otherwise it returns a newly constructed
|
47 |
/// Config struct with all the parsed config options from the parsed config file.
|
48 |
pub fn parse() -> Result<Self, Box<dyn std::error::Error>> {
|
49 |
Lua::new().context(|context| -> Result<Self, Box<dyn std::error::Error>> {
|
50 |
let globals = context.globals();
|
51 |
|
52 |
context
|
53 |
+
.load(&fs::read_to_string(Config::get_config_path()?)?)
|
|
|
|
|
54 |
.exec()?;
|
55 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
56 |
Ok(Config {
|
57 |
port: globals.get::<_, u16>("port")?,
|
58 |
binding_ip_addr: globals.get::<_, String>("binding_ip_addr")?,
|
|
|
61 |
globals.get::<_, String>("colorscheme")?,
|
62 |
),
|
63 |
redis_connection_url: globals.get::<_, String>("redis_connection_url")?,
|
64 |
+
aggregator: AggregatorConfig {
|
65 |
+
random_delay: globals.get::<_, bool>("production_use")?,
|
66 |
+
},
|
67 |
logging: globals.get::<_, bool>("logging")?,
|
68 |
debug: globals.get::<_, bool>("debug")?,
|
69 |
})
|
|
|
81 |
/// one (3).
|
82 |
/// 3. `websurfx/` (under project folder ( or codebase in other words)) if it is not present
|
83 |
/// here then it returns an error as mentioned above.
|
84 |
+
fn get_config_path() -> Result<String, Box<dyn std::error::Error>> {
|
85 |
+
// check user config
|
86 |
+
|
87 |
+
let path = format!(
|
88 |
+
"{}/.config/{}/config.lua",
|
89 |
+
std::env::var("HOME").unwrap(),
|
90 |
+
COMMON_DIRECTORY_NAME
|
91 |
+
);
|
92 |
+
if Path::new(path.as_str()).exists() {
|
93 |
+
return Ok(format!(
|
|
|
|
|
94 |
"{}/.config/{}/{}",
|
95 |
std::env::var("HOME").unwrap(),
|
96 |
COMMON_DIRECTORY_NAME,
|
97 |
CONFIG_FILE_NAME
|
98 |
+
));
|
99 |
+
}
|
100 |
+
|
101 |
+
// look for config in /etc/xdg
|
102 |
+
if Path::new(format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME).as_str())
|
|
|
|
|
|
|
103 |
.exists()
|
104 |
{
|
105 |
+
return Ok("/etc/xdg/websurfx/config.lua".to_string());
|
|
|
|
|
106 |
}
|
107 |
+
|
108 |
+
// use dev config
|
109 |
+
if Path::new(format!("./{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME).as_str()).exists()
|
110 |
+
{
|
111 |
+
return Ok("./websurfx/config.lua".to_string());
|
112 |
+
}
|
113 |
+
|
114 |
+
// if no of the configs above exist, return error
|
115 |
+
Err("Config file not found!!".to_string().into())
|
116 |
}
|
117 |
}
|
src/config_parser/parser_models.rs
CHANGED
@@ -1,5 +1,5 @@
|
|
1 |
//! This module provides public models for handling, storing and serializing parsed config file
|
2 |
-
//! options from config.lua by grouping them
|
3 |
|
4 |
use serde::{Deserialize, Serialize};
|
5 |
|
|
|
1 |
//! This module provides public models for handling, storing and serializing parsed config file
|
2 |
+
//! options from config.lua by grouping them together.
|
3 |
|
4 |
use serde::{Deserialize, Serialize};
|
5 |
|
src/engines/duckduckgo.rs
CHANGED
@@ -36,7 +36,7 @@ pub async fn results(
|
|
36 |
user_agent: &str,
|
37 |
) -> Result<HashMap<String, RawSearchResult>, EngineError> {
|
38 |
// Page number can be missing or empty string and so appropriate handling is required
|
39 |
-
// so that upstream server
|
40 |
let url: String = match page {
|
41 |
1 => {
|
42 |
format!("https://html.duckduckgo.com/html/?q={query}&s=&dc=&v=1&o=json&api=/d.js")
|
@@ -86,7 +86,7 @@ pub async fn results(
|
|
86 |
let results: String = reqwest::Client::new()
|
87 |
.get(url)
|
88 |
.timeout(Duration::from_secs(5))
|
89 |
-
.headers(header_map) // add spoofed headers to emulate human
|
90 |
.send()
|
91 |
.await
|
92 |
.into_report()
|
|
|
36 |
user_agent: &str,
|
37 |
) -> Result<HashMap<String, RawSearchResult>, EngineError> {
|
38 |
// Page number can be missing or empty string and so appropriate handling is required
|
39 |
+
// so that upstream server receives valid page number.
|
40 |
let url: String = match page {
|
41 |
1 => {
|
42 |
format!("https://html.duckduckgo.com/html/?q={query}&s=&dc=&v=1&o=json&api=/d.js")
|
|
|
86 |
let results: String = reqwest::Client::new()
|
87 |
.get(url)
|
88 |
.timeout(Duration::from_secs(5))
|
89 |
+
.headers(header_map) // add spoofed headers to emulate human behavior
|
90 |
.send()
|
91 |
.await
|
92 |
.into_report()
|
src/engines/engine_models.rs
CHANGED
@@ -13,7 +13,7 @@ use std::fmt;
|
|
13 |
/// search engines.
|
14 |
/// * `UnexpectedError` - This variant handles all the errors which are unexpected or occur rarely
|
15 |
/// and are errors mostly related to failure in initialization of HeaderMap, Selector errors and
|
16 |
-
/// all other errors
|
17 |
#[derive(Debug)]
|
18 |
pub enum EngineError {
|
19 |
EmptyResultSet,
|
|
|
13 |
/// search engines.
|
14 |
/// * `UnexpectedError` - This variant handles all the errors which are unexpected or occur rarely
|
15 |
/// and are errors mostly related to failure in initialization of HeaderMap, Selector errors and
|
16 |
+
/// all other errors occurring within the code handling the `upstream search engines`.
|
17 |
#[derive(Debug)]
|
18 |
pub enum EngineError {
|
19 |
EmptyResultSet,
|
src/search_results_handler/aggregator.rs
CHANGED
@@ -17,7 +17,7 @@ use crate::engines::{duckduckgo, searx};
|
|
17 |
/// then removes duplicate results and if two results are found to be from two or more engines
|
18 |
/// then puts their names together to show the results are fetched from these upstream engines
|
19 |
/// and then removes all data from the HashMap and puts into a struct of all results aggregated
|
20 |
-
/// into a vector and also adds the query used into the struct this is
|
21 |
/// otherwise the search bar in search remains empty if searched from the query url
|
22 |
///
|
23 |
/// # Example:
|
|
|
17 |
/// then removes duplicate results and if two results are found to be from two or more engines
|
18 |
/// then puts their names together to show the results are fetched from these upstream engines
|
19 |
/// and then removes all data from the HashMap and puts into a struct of all results aggregated
|
20 |
+
/// into a vector and also adds the query used into the struct this is necessary because
|
21 |
/// otherwise the search bar in search remains empty if searched from the query url
|
22 |
///
|
23 |
/// # Example:
|
src/server/routes.rs
CHANGED
@@ -1,5 +1,5 @@
|
|
1 |
//! This module provides the functionality to handle different routes of the `websurfx`
|
2 |
-
//! meta search engine website and provide
|
3 |
//! when requested.
|
4 |
|
5 |
use std::fs::read_to_string;
|
@@ -82,40 +82,16 @@ pub async fn search(
|
|
82 |
.insert_header(("location", "/"))
|
83 |
.finish())
|
84 |
} else {
|
85 |
-
let
|
86 |
-
|
87 |
-
|
88 |
-
|
89 |
-
let page = match params.page {
|
90 |
-
Some(page_number) => {
|
91 |
-
if page_number <= 1 {
|
92 |
-
page_url = format!(
|
93 |
-
"http://{}:{}/search?q={}&page={}",
|
94 |
-
config.binding_ip_addr, config.port, query, 1
|
95 |
-
);
|
96 |
-
1
|
97 |
-
} else {
|
98 |
-
page_url = format!(
|
99 |
-
"http://{}:{}/search?q={}&page={}",
|
100 |
-
config.binding_ip_addr, config.port, query, page_number
|
101 |
-
);
|
102 |
-
|
103 |
-
page_number
|
104 |
-
}
|
105 |
-
}
|
106 |
-
None => {
|
107 |
-
page_url = format!(
|
108 |
-
"http://{}:{}{}&page={}",
|
109 |
-
config.binding_ip_addr,
|
110 |
-
config.port,
|
111 |
-
req.uri(),
|
112 |
-
1
|
113 |
-
);
|
114 |
-
|
115 |
-
1
|
116 |
-
}
|
117 |
};
|
118 |
|
|
|
|
|
|
|
|
|
|
|
119 |
// fetch the cached results json.
|
120 |
let cached_results_json = redis_cache.cached_results_json(&page_url);
|
121 |
// check if fetched results was indeed fetched or it was an error and if so
|
|
|
1 |
//! This module provides the functionality to handle different routes of the `websurfx`
|
2 |
+
//! meta search engine website and provide appropriate response to each route/page
|
3 |
//! when requested.
|
4 |
|
5 |
use std::fs::read_to_string;
|
|
|
82 |
.insert_header(("location", "/"))
|
83 |
.finish())
|
84 |
} else {
|
85 |
+
let page = match ¶ms.page {
|
86 |
+
Some(page) => *page,
|
87 |
+
None => 0,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
88 |
};
|
89 |
|
90 |
+
let page_url = format!(
|
91 |
+
"http://{}:{}/search?q={}&page={}",
|
92 |
+
config.binding_ip_addr, config.port, query, page
|
93 |
+
);
|
94 |
+
|
95 |
// fetch the cached results json.
|
96 |
let cached_results_json = redis_cache.cached_results_json(&page_url);
|
97 |
// check if fetched results was indeed fetched or it was an error and if so
|