Merge pull request #80 from neon-mmd/feat-add-debug-opt
Browse files- Cargo.lock +8 -8
- Cargo.toml +1 -1
- src/bin/websurfx.rs +1 -1
- src/config_parser/parser.rs +2 -0
- src/search_results_handler/aggregator.rs +2 -1
- src/server/routes.rs +1 -1
- websurfx/config.lua +1 -0
Cargo.lock
CHANGED
@@ -125,7 +125,7 @@ dependencies = [
|
|
125 |
"actix-utils",
|
126 |
"futures-core",
|
127 |
"futures-util",
|
128 |
-
"mio 0.8.
|
129 |
"num_cpus",
|
130 |
"socket2",
|
131 |
"tokio 1.28.2",
|
@@ -1510,14 +1510,14 @@ dependencies = [
|
|
1510 |
|
1511 |
[[package]]
|
1512 |
name = "mio"
|
1513 |
-
version = "0.8.
|
1514 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1515 |
-
checksum = "
|
1516 |
dependencies = [
|
1517 |
"libc",
|
1518 |
"log",
|
1519 |
"wasi 0.11.0+wasi-snapshot-preview1",
|
1520 |
-
"windows-sys 0.
|
1521 |
]
|
1522 |
|
1523 |
[[package]]
|
@@ -1603,9 +1603,9 @@ dependencies = [
|
|
1603 |
|
1604 |
[[package]]
|
1605 |
name = "once_cell"
|
1606 |
-
version = "1.17.
|
1607 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1608 |
-
checksum = "
|
1609 |
|
1610 |
[[package]]
|
1611 |
name = "openssl"
|
@@ -2879,7 +2879,7 @@ dependencies = [
|
|
2879 |
"autocfg 1.1.0",
|
2880 |
"bytes 1.4.0",
|
2881 |
"libc",
|
2882 |
-
"mio 0.8.
|
2883 |
"num_cpus",
|
2884 |
"parking_lot 0.12.1",
|
2885 |
"pin-project-lite",
|
@@ -3316,7 +3316,7 @@ dependencies = [
|
|
3316 |
|
3317 |
[[package]]
|
3318 |
name = "websurfx"
|
3319 |
-
version = "0.
|
3320 |
dependencies = [
|
3321 |
"actix-files",
|
3322 |
"actix-web",
|
|
|
125 |
"actix-utils",
|
126 |
"futures-core",
|
127 |
"futures-util",
|
128 |
+
"mio 0.8.7",
|
129 |
"num_cpus",
|
130 |
"socket2",
|
131 |
"tokio 1.28.2",
|
|
|
1510 |
|
1511 |
[[package]]
|
1512 |
name = "mio"
|
1513 |
+
version = "0.8.7"
|
1514 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1515 |
+
checksum = "eebffdb73fe72e917997fad08bdbf31ac50b0fa91cec93e69a0662e4264d454c"
|
1516 |
dependencies = [
|
1517 |
"libc",
|
1518 |
"log",
|
1519 |
"wasi 0.11.0+wasi-snapshot-preview1",
|
1520 |
+
"windows-sys 0.48.0",
|
1521 |
]
|
1522 |
|
1523 |
[[package]]
|
|
|
1603 |
|
1604 |
[[package]]
|
1605 |
name = "once_cell"
|
1606 |
+
version = "1.17.2"
|
1607 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
1608 |
+
checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b"
|
1609 |
|
1610 |
[[package]]
|
1611 |
name = "openssl"
|
|
|
2879 |
"autocfg 1.1.0",
|
2880 |
"bytes 1.4.0",
|
2881 |
"libc",
|
2882 |
+
"mio 0.8.7",
|
2883 |
"num_cpus",
|
2884 |
"parking_lot 0.12.1",
|
2885 |
"pin-project-lite",
|
|
|
3316 |
|
3317 |
[[package]]
|
3318 |
name = "websurfx"
|
3319 |
+
version = "0.11.0"
|
3320 |
dependencies = [
|
3321 |
"actix-files",
|
3322 |
"actix-web",
|
Cargo.toml
CHANGED
@@ -1,6 +1,6 @@
|
|
1 |
[package]
|
2 |
name = "websurfx"
|
3 |
-
version = "0.
|
4 |
edition = "2021"
|
5 |
|
6 |
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
|
|
1 |
[package]
|
2 |
name = "websurfx"
|
3 |
+
version = "0.11.0"
|
4 |
edition = "2021"
|
5 |
|
6 |
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
src/bin/websurfx.rs
CHANGED
@@ -19,7 +19,7 @@ async fn main() -> std::io::Result<()> {
|
|
19 |
let config = Config::parse().unwrap();
|
20 |
|
21 |
// Initializing logging middleware with level set to default or info.
|
22 |
-
if config.logging {
|
23 |
use env_logger::Env;
|
24 |
env_logger::Builder::from_env(Env::default().default_filter_or("info")).init();
|
25 |
}
|
|
|
19 |
let config = Config::parse().unwrap();
|
20 |
|
21 |
// Initializing logging middleware with level set to default or info.
|
22 |
+
if config.logging || config.debug {
|
23 |
use env_logger::Env;
|
24 |
env_logger::Builder::from_env(Env::default().default_filter_or("info")).init();
|
25 |
}
|
src/config_parser/parser.rs
CHANGED
@@ -26,6 +26,7 @@ pub struct Config {
|
|
26 |
pub redis_connection_url: String,
|
27 |
pub aggregator: AggreatorConfig,
|
28 |
pub logging: bool,
|
|
|
29 |
}
|
30 |
|
31 |
/// Configuration options for the aggregator.
|
@@ -73,6 +74,7 @@ impl Config {
|
|
73 |
redis_connection_url: globals.get::<_, String>("redis_connection_url")?,
|
74 |
aggregator: aggregator_config,
|
75 |
logging: globals.get::<_, bool>("logging")?,
|
|
|
76 |
})
|
77 |
})
|
78 |
}
|
|
|
26 |
pub redis_connection_url: String,
|
27 |
pub aggregator: AggreatorConfig,
|
28 |
pub logging: bool,
|
29 |
+
pub debug: bool,
|
30 |
}
|
31 |
|
32 |
/// Configuration options for the aggregator.
|
|
|
74 |
redis_connection_url: globals.get::<_, String>("redis_connection_url")?,
|
75 |
aggregator: aggregator_config,
|
76 |
logging: globals.get::<_, bool>("logging")?,
|
77 |
+
debug: globals.get::<_, bool>("debug")?,
|
78 |
})
|
79 |
})
|
80 |
}
|
src/search_results_handler/aggregator.rs
CHANGED
@@ -40,12 +40,13 @@ pub async fn aggregate(
|
|
40 |
query: &str,
|
41 |
page: u32,
|
42 |
random_delay: bool,
|
|
|
43 |
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
44 |
let user_agent: String = random_user_agent();
|
45 |
let mut result_map: HashMap<String, RawSearchResult> = HashMap::new();
|
46 |
|
47 |
// Add a random delay before making the request.
|
48 |
-
if random_delay {
|
49 |
let mut rng = rand::thread_rng();
|
50 |
let delay_secs = rng.gen_range(1..10);
|
51 |
std::thread::sleep(Duration::from_secs(delay_secs));
|
|
|
40 |
query: &str,
|
41 |
page: u32,
|
42 |
random_delay: bool,
|
43 |
+
debug: bool,
|
44 |
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
45 |
let user_agent: String = random_user_agent();
|
46 |
let mut result_map: HashMap<String, RawSearchResult> = HashMap::new();
|
47 |
|
48 |
// Add a random delay before making the request.
|
49 |
+
if random_delay || !debug {
|
50 |
let mut rng = rand::thread_rng();
|
51 |
let delay_secs = rng.gen_range(1..10);
|
52 |
std::thread::sleep(Duration::from_secs(delay_secs));
|
src/server/routes.rs
CHANGED
@@ -128,7 +128,7 @@ pub async fn search(
|
|
128 |
}
|
129 |
Err(_) => {
|
130 |
let mut results_json: crate::search_results_handler::aggregation_models::SearchResults =
|
131 |
-
aggregate(query, page, config.aggregator.random_delay).await?;
|
132 |
results_json.add_style(config.style.clone());
|
133 |
redis_cache
|
134 |
.cache_results(serde_json::to_string(&results_json)?, &page_url)?;
|
|
|
128 |
}
|
129 |
Err(_) => {
|
130 |
let mut results_json: crate::search_results_handler::aggregation_models::SearchResults =
|
131 |
+
aggregate(query, page, config.aggregator.random_delay, config.debug).await?;
|
132 |
results_json.add_style(config.style.clone());
|
133 |
redis_cache
|
134 |
.cache_results(serde_json::to_string(&results_json)?, &page_url)?;
|
websurfx/config.lua
CHANGED
@@ -1,5 +1,6 @@
|
|
1 |
-- ### General ###
|
2 |
logging = true -- an option to enable or disable logs.
|
|
|
3 |
|
4 |
-- ### Server ###
|
5 |
port = "8080" -- port on which server should be launched
|
|
|
1 |
-- ### General ###
|
2 |
logging = true -- an option to enable or disable logs.
|
3 |
+
debug = false -- an option to enable or disable debug mode.
|
4 |
|
5 |
-- ### Server ###
|
6 |
port = "8080" -- port on which server should be launched
|