:sparkles: config option to configure the https adaptive window size for requests (#529)

* Added config option to enable the reqwest client adaptive window

* Change adaptive window config name

Co-authored-by: neon_arch <mustafadhuleb53@gmail.com>

* Modified documentation

* Trimmed down aggregate parameters

---------

Co-authored-by: neon_arch <mustafadhuleb53@gmail.com>

src/config/parser.rs

@@ -30,6 +30,8 @@ pub struct Config {
     pub logging: bool,
     /// It stores the option to whether enable or disable debug mode.
     pub debug: bool,
+    /// It toggles whether to use adaptive HTTP windows
+    pub adaptive_window: bool,
     /// It stores all the engine names that were enabled by the user.
     pub upstream_search_engines: HashMap<String, bool>,
     /// It stores the time (secs) which controls the server request timeout.
@@ -68,6 +70,7 @@ impl Config {
 
         let debug: bool = globals.get::<_, bool>("debug")?;
         let logging: bool = globals.get::<_, bool>("logging")?;
+        let adaptive_window: bool = globals.get::<_, bool>("adaptive_window")?;
 
         if !logging_initialized {
             set_logging_level(debug, logging);
@@ -125,6 +128,7 @@ impl Config {
             },
             logging,
             debug,
+            adaptive_window,
             upstream_search_engines: globals
                 .get::<_, HashMap<String, bool>>("upstream_search_engines")?,
             request_timeout: globals.get::<_, u8>("request_timeout")?,

src/results/aggregator.rs

@@ -2,6 +2,7 @@
 //! search engines and then removes duplicate results.
 
 use super::user_agent::random_user_agent;
+use crate::config::parser::Config;
 use crate::handler::{file_path, FileType};
 use crate::models::{
     aggregation_models::{EngineErrorInfo, SearchResult, SearchResults},
@@ -66,18 +67,17 @@ type FutureVec = Vec<JoinHandle<Result<HashMap<String, SearchResult>, Report<Eng
 pub async fn aggregate(
     query: &str,
     page: u32,
-    random_delay: bool,
-    debug: bool,
+    config: &Config,
     upstream_search_engines: &[EngineHandler],
-    request_timeout: u8,
     safe_search: u8,
 ) -> Result<SearchResults, Box<dyn std::error::Error>> {
     let client = CLIENT.get_or_init(|| {
         ClientBuilder::new()
-            .timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
+            .timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
             .https_only(true)
             .gzip(true)
             .brotli(true)
+            .http2_adaptive_window(config.adaptive_window)
             .build()
             .unwrap()
     });
@@ -85,7 +85,7 @@ pub async fn aggregate(
     let user_agent: &str = random_user_agent();
 
     // Add a random delay before making the request.
-    if random_delay || !debug {
+    if config.aggregator.random_delay || !config.debug {
         let nanos = SystemTime::now().duration_since(UNIX_EPOCH)?.subsec_nanos() as f32;
         let delay = ((nanos / 1_0000_0000 as f32).floor() as u64) + 1;
         tokio::time::sleep(Duration::from_secs(delay)).await;

src/server/routes/search.rs

@@ -209,14 +209,12 @@ async fn results(
                     aggregate(
                         query,
                         page,
-                        config.aggregator.random_delay,
-                        config.debug,
+                        config,
                         &search_settings
                             .engines
                             .iter()
                             .filter_map(|engine| EngineHandler::new(engine).ok())
                             .collect::<Vec<EngineHandler>>(),
-                        config.request_timeout,
                         safe_search_level,
                     )
                     .await?

websurfx/config.lua

@@ -14,6 +14,8 @@ rate_limiter = {
 	number_of_requests = 20, -- The number of request that are allowed within a provided time limit.
 	time_limit = 3, -- The time limit in which the quantity of requests that should be accepted.
 }
+-- Set whether the server will use an adaptive/dynamic HTTPS window size, see https://httpwg.org/specs/rfc9113.html#fc-principles
+https_adaptive_window_size = false
 
 -- ### Search ###
 -- Filter results based on different levels. The levels provided are: