Repository: Y2Z/monolith
Pull request #400 (2024-08-15 21:24:02)

add HTTP referer header for resource requests
to mimic more-browser-like-behavior on loading page resources. Sometimes their response depends on `referer` header.

e.g. I faced with the problem to load font data for page, because it was protected for "direct" loading, got `401`, but with the `referer` header it looks like request from browser.

rakhnin: >And please feel free to add yourself as one of the authors of the program, if you'd like some well-deserved open-source fame.

Nice to have it ;), thanks

Patch:
diff --git a/Cargo.toml b/Cargo.toml
index 40d100e..9b96223 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -7,6 +7,7 @@ authors = [
     "Emmanuel Delaborde <th3rac25@gmail.com>",
     "Emi Simpson <emi@alchemi.dev>",
     "rhysd <lin90162@yahoo.co.jp>",
+    "Andriy Rakhnin <a@rakhnin.com>",
 ]
 edition = "2021"
 description = "CLI tool for saving web pages as a single HTML file"
diff --git a/src/url.rs b/src/url.rs
index 2723f31..7946021 100644
--- a/src/url.rs
+++ b/src/url.rs
@@ -79,6 +79,17 @@ pub fn parse_data_url(url: &Url) -> (String, String, Vec<u8>) {
     (media_type, charset, blob)
 }
 
+pub fn get_referer_url(url: Url) -> Url {
+    let mut url = url.clone();
+    // Spec: https://httpwg.org/specs/rfc9110.html#field.referer
+    // Must not include the fragment and userinfo components of the URI
+    url.set_fragment(None);
+    url.set_username(&"").unwrap();
+    url.set_password(None).unwrap();
+
+    url
+}
+
 pub fn resolve_url(from: &Url, to: &str) -> Url {
     match Url::parse(&to) {
         Ok(parsed_url) => parsed_url,
diff --git a/src/utils.rs b/src/utils.rs
index 20fa32c..9eda28a 100644
--- a/src/utils.rs
+++ b/src/utils.rs
@@ -1,12 +1,12 @@
 use reqwest::blocking::Client;
-use reqwest::header::{HeaderMap, HeaderValue, CONTENT_TYPE, COOKIE};
+use reqwest::header::{HeaderMap, HeaderValue, CONTENT_TYPE, COOKIE, REFERER};
 use std::collections::HashMap;
 use std::fs;
 use std::path::{Path, PathBuf};
 use url::Url;
 
 use crate::opts::Options;
-use crate::url::{clean_url, parse_data_url};
+use crate::url::{clean_url, get_referer_url, parse_data_url};
 
 const ANSI_COLOR_RED: &'static str = "\x1b[31m";
 const ANSI_COLOR_RESET: &'static str = "\x1b[0m";
@@ -298,6 +298,13 @@ pub fn retrieve_asset(
                     }
                 }
             }
+            // Add referer header for page resource requests
+            if parent_url != url {
+                headers.insert(
+                    REFERER,
+                    HeaderValue::from_str(get_referer_url(parent_url.clone()).as_str()).unwrap(),
+                );
+            }
             match client.get(url.as_str()).headers(headers).send() {
                 Ok(response) => {
                     if !options.ignore_errors && response.status() != reqwest::StatusCode::OK {


Test patch:
diff --git a/tests/url/clean_url.rs b/tests/url/clean_url.rs
index 15a6818..7043f8c 100644
--- a/tests/url/clean_url.rs
+++ b/tests/url/clean_url.rs
@@ -46,7 +46,7 @@ mod passing {
     }
 
     #[test]
-    fn removesempty_fragment_and_keeps_empty_query() {
+    fn removes_empty_fragment_and_keeps_query() {
         assert_eq!(
             url::clean_url(Url::parse("https://somewhere.com/font.eot?a=b&#").unwrap()).as_str(),
             "https://somewhere.com/font.eot?a=b&"
diff --git a/tests/url/get_referer_url.rs b/tests/url/get_referer_url.rs
new file mode 100644
index 0000000..8fe4f66
--- /dev/null
+++ b/tests/url/get_referer_url.rs
@@ -0,0 +1,91 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use reqwest::Url;
+
+    use monolith::url;
+
+    #[test]
+    fn preserve_original() {
+        let original_url: Url = Url::parse("https://somewhere.com/font.eot#iefix").unwrap();
+        let referer_url: Url = url::get_referer_url(original_url.clone());
+        assert_eq!(referer_url.as_str(), "https://somewhere.com/font.eot");
+        assert_eq!(
+            original_url.as_str(),
+            "https://somewhere.com/font.eot#iefix"
+        );
+    }
+
+    #[test]
+    fn removes_fragment() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://somewhere.com/font.eot#iefix").unwrap())
+                .as_str(),
+            "https://somewhere.com/font.eot"
+        );
+    }
+
+    #[test]
+    fn removes_empty_fragment() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://somewhere.com/font.eot#").unwrap()).as_str(),
+            "https://somewhere.com/font.eot"
+        );
+    }
+
+    #[test]
+    fn removes_empty_fragment_and_keeps_empty_query() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://somewhere.com/font.eot?#").unwrap()).as_str(),
+            "https://somewhere.com/font.eot?"
+        );
+    }
+
+    #[test]
+    fn removes_empty_fragment_and_keeps_query() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://somewhere.com/font.eot?a=b&#").unwrap())
+                .as_str(),
+            "https://somewhere.com/font.eot?a=b&"
+        );
+    }
+
+    #[test]
+    fn removes_credentials() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://cookie:monster@gibson.lan/path").unwrap())
+                .as_str(),
+            "https://gibson.lan/path"
+        );
+    }
+
+    #[test]
+    fn removes_empty_credentials() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://@gibson.lan/path").unwrap()).as_str(),
+            "https://gibson.lan/path"
+        );
+    }
+
+    #[test]
+    fn removes_empty_username_credentials() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://:monster@gibson.lan/path").unwrap()).as_str(),
+            "https://gibson.lan/path"
+        );
+    }
+
+    #[test]
+    fn removes_empty_password_credentials() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://cookie@gibson.lan/path").unwrap()).as_str(),
+            "https://gibson.lan/path"
+        );
+    }
+}
diff --git a/tests/url/mod.rs b/tests/url/mod.rs
index 6864e78..380bd93 100644
--- a/tests/url/mod.rs
+++ b/tests/url/mod.rs
@@ -1,5 +1,6 @@
 mod clean_url;
 mod create_data_url;
+mod get_referer_url;
 mod is_url_and_has_protocol;
 mod parse_data_url;
 mod resolve_url;