Repository: Y2Z/monolith
Pull request #426 (2025-03-09 01:08:26)

Improved caching mechanism


Patch:
diff --git a/.github/workflows/ci-netbsd.yml b/.github/workflows/ci-netbsd.yml
index 977a0f0..dc39930 100644
--- a/.github/workflows/ci-netbsd.yml
+++ b/.github/workflows/ci-netbsd.yml
@@ -1,6 +1,6 @@
 # CI NetBSD GitHub Actions workflow for monolith
 
-name: CI
+name: CI (NetBSD)
 
 on:
   pull_request:
@@ -30,5 +30,5 @@ jobs:
         prepare: |
           /usr/sbin/pkg_add rust mktools gmake pkgconf cwrappers
         run: |
-          cargo build --all --locked --verbose --no-default-features
-          cargo test --all --locked --verbose --no-default-features
+          cargo build --all --locked --verbose --no-default-features --features cli
+          cargo test --all --locked --verbose --no-default-features --features cli
diff --git a/Cargo.lock b/Cargo.lock
index 1fb884a..713c258 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -370,9 +370,9 @@ dependencies = [
 
 [[package]]
 name = "fastrand"
-version = "2.1.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
+checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4"
 
 [[package]]
 name = "flate2"
@@ -697,9 +697,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.153"
+version = "0.2.167"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
+checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc"
 
 [[package]]
 name = "linux-raw-sys"
@@ -802,9 +802,11 @@ dependencies = [
  "markup5ever_rcdom",
  "openssl",
  "percent-encoding",
+ "redb",
  "regex",
  "reqwest",
  "sha2",
+ "tempfile",
  "url",
 ]
 
@@ -1141,6 +1143,15 @@ dependencies = [
  "getrandom",
 ]
 
+[[package]]
+name = "redb"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea0a72cd7140de9fc3e318823b883abf819c20d478ec89ce880466dc2ef263c6"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.1"
@@ -1229,9 +1240,9 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
 
 [[package]]
 name = "rustix"
-version = "0.38.34"
+version = "0.38.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
+checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6"
 dependencies = [
  "bitflags 2.5.0",
  "errno",
@@ -1439,12 +1450,13 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.10.1"
+version = "3.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
+checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c"
 dependencies = [
  "cfg-if",
  "fastrand",
+ "once_cell",
  "rustix",
  "windows-sys 0.52.0",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index 2f72913..9fc9bd3 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -22,24 +22,21 @@ include = [
 ]
 license = "CC0-1.0"
 
-[features]
-default = ["vendored-openssl"]
-# Compile and statically link a copy of OpenSSL.
-vendored-openssl = ["openssl/vendored"]
-
 [dependencies]
 atty = "0.2.14"  # Used for highlighting network errors
 base64 = "0.22.1"  # Used for integrity attributes
 chrono = "0.4.38"  # Used for formatting output timestamp
-clap = "3.2.25"  # Used for processing CLI arguments
+clap = { version = "3.2.25", optional = true }  # Used for processing CLI arguments
 cssparser = "0.34.0"  # Used for dealing with CSS
 encoding_rs = "0.8.34"  # Used for parsing and converting document charsets
 html5ever = "0.27.0"  # Used for all things DOM
 markup5ever_rcdom = "0.3.0"  # Used for manipulating DOM
 percent-encoding = "2.3.1"  # Used for encoding URLs
 sha2 = "0.10.8"  # Used for calculating checksums during integrity checks
+redb = "2.4.0"  # Used for on-disk caching of downloaded assets
+tempfile = { version = "3.14.0", optional = true }  # Used for on-disk caching of downloaded assets
 url = "2.5.2"  # Used for parsing URLs
-openssl = "0.10.64" # Used for static linking of the OpenSSL library
+openssl = "0.10.64"  # Used for static linking of the OpenSSL library
 
 # Used for parsing srcset and NOSCRIPT
 [dependencies.regex]
@@ -55,3 +52,8 @@ features = ["default-tls", "blocking", "gzip", "brotli", "deflate"]
 
 [dev-dependencies]
 assert_cmd = "2.0.16"
+
+[features]
+default = ["cli", "vendored-openssl"]
+cli = ["clap", "tempfile"]  # Build a CLI tool that includes main() function
+vendored-openssl = ["openssl/vendored"]  # Compile and statically link a copy of OpenSSL
diff --git a/src/cache.rs b/src/cache.rs
new file mode 100644
index 0000000..ae15e93
--- /dev/null
+++ b/src/cache.rs
@@ -0,0 +1,115 @@
+use std::collections::HashMap;
+use std::path::Path;
+
+use redb::{Database, Error, TableDefinition};
+
+pub struct CacheMetadataItem {
+    data: Option<Vec<u8>>, // Asset's blob; used for caching small files or if on-disk database isn't utilized
+    media_type: Option<String>, // MIME-type, things like "text/plain", "image/png"...
+    charset: Option<String>, // "UTF-8", "UTF-16"...
+}
+
+// #[derive(Debug)]
+pub struct Cache {
+    min_file_size: usize, // Only use database for assets larger than this size (in bytes), otherwise keep them in RAM
+    metadata: HashMap<String, CacheMetadataItem>, // Dictionary of metadata (and occasionally data [mostly for very small files])
+    db: Option<Database>, // Pointer to database instance; None if not yet initialized or if failed to initialize
+    db_ok: Option<bool>, // None by default, Some(true) if was able to initialize database, Some (false) if an error occured
+}
+
+const TABLE: TableDefinition<&str, &[u8]> = TableDefinition::new("_");
+
+impl Cache {
+    pub fn new(min_file_size: usize, db_file_path: Option<String>) -> Cache {
+        let mut cache = Cache {
+            min_file_size: min_file_size,
+            metadata: HashMap::new(),
+            db: None,
+            db_ok: None,
+        };
+
+        if db_file_path.is_some() {
+            // Attempt to initialize on-disk database
+            match Database::create(Path::new(&db_file_path.unwrap())) {
+                Ok(db) => {
+                    cache.db = Some(db);
+                    cache.db_ok = Some(true);
+                    cache
+                }
+                Err(..) => {
+                    cache.db_ok = Some(false);
+                    cache
+                }
+            }
+        } else {
+            cache.db_ok = Some(false);
+            cache
+        }
+    }
+
+    pub fn set(&mut self, key: &str, data: &Vec<u8>, media_type: String, charset: String) {
+        let mut cache_metadata_item: CacheMetadataItem = CacheMetadataItem {
+            data: if self.db_ok.is_some() && self.db_ok.unwrap() {
+                None
+            } else {
+                Some((&*data.to_owned()).to_vec())
+            },
+            media_type: Some(media_type.to_owned()),
+            charset: Some(charset),
+        };
+
+        if (self.db_ok.is_none() || !self.db_ok.unwrap()) || data.len() <= self.min_file_size {
+            cache_metadata_item.data = Some((&*data.to_owned()).to_vec());
+        } else {
+            match self.db.as_ref().unwrap().begin_write() {
+                Ok(write_txn) => {
+                    {
+                        let mut table = write_txn.open_table(TABLE).unwrap();
+                        table.insert(key, &*data.to_owned()).unwrap();
+                    }
+                    write_txn.commit().unwrap();
+                }
+                Err(..) => {
+                    // Fall back to caching everything in memory
+                    cache_metadata_item.data = Some((&*data.to_owned()).to_vec());
+                }
+            }
+        }
+
+        self.metadata
+            .insert((*key).to_string(), cache_metadata_item);
+    }
+
+    pub fn get(&self, key: &str) -> Result<(Vec<u8>, String, String), Error> {
+        if self.metadata.contains_key(key) {
+            let metadata_item = self.metadata.get(key).unwrap();
+
+            if metadata_item.data.is_some() {
+                return Ok((
+                    metadata_item.data.as_ref().unwrap().to_vec(),
+                    metadata_item.media_type.as_ref().expect("").to_string(),
+                    metadata_item.charset.as_ref().expect("").to_string(),
+                ));
+            } else {
+                if self.db_ok.is_some() && self.db_ok.unwrap() {
+                    let read_txn = self.db.as_ref().unwrap().begin_read()?;
+                    let table = read_txn.open_table(TABLE)?;
+                    let data = table.get(key)?;
+                    let bytes = data.unwrap();
+
+                    return Ok((
+                        bytes.value().to_vec(),
+                        metadata_item.media_type.as_ref().expect("").to_string(),
+                        metadata_item.charset.as_ref().expect("").to_string(),
+                    ));
+                }
+            }
+        }
+
+        Err(Error::TransactionInProgress) // XXX
+    }
+
+    pub fn contains_key(&self, key: &str) -> bool {
+        self.metadata.contains_key(key)
+    }
+}
diff --git a/src/css.rs b/src/css.rs
index db1c83e..b2dccba 100644
--- a/src/css.rs
+++ b/src/css.rs
@@ -2,9 +2,9 @@ use cssparser::{
     serialize_identifier, serialize_string, ParseError, Parser, ParserInput, SourcePosition, Token,
 };
 use reqwest::blocking::Client;
-use std::collections::HashMap;
 use url::Url;
 
+use crate::cache::Cache;
 use crate::opts::Options;
 use crate::url::{create_data_url, resolve_url, EMPTY_IMAGE_DATA_URL};
 use crate::utils::retrieve_asset;
@@ -31,7 +31,7 @@ const CSS_PROPS_WITH_IMAGE_URLS: &[&str] = &[
 ];
 
 pub fn embed_css(
-    cache: &mut HashMap<String, Vec<u8>>,
+    cache: &mut Cache,
     client: &Client,
     document_url: &Url,
     css: &str,
@@ -74,7 +74,7 @@ pub fn is_image_url_prop(prop_name: &str) -> bool {
 }
 
 pub fn process_css<'a>(
-    cache: &mut HashMap<String, Vec<u8>>,
+    cache: &mut Cache,
     client: &Client,
     document_url: &Url,
     parser: &mut Parser,
diff --git a/src/html.rs b/src/html.rs
index 2eb2e9d..04cd6e7 100644
--- a/src/html.rs
+++ b/src/html.rs
@@ -12,9 +12,9 @@ use regex::Regex;
 use reqwest::blocking::Client;
 use reqwest::Url;
 use sha2::{Digest, Sha256, Sha384, Sha512};
-use std::collections::HashMap;
 use std::default::Default;
 
+use crate::cache::Cache;
 use crate::css::embed_css;
 use crate::js::attr_is_event_handler;
 use crate::opts::Options;
@@ -151,7 +151,7 @@ pub fn create_metadata_tag(url: &Url) -> String {
 }
 
 pub fn embed_srcset(
-    cache: &mut HashMap<String, Vec<u8>>,
+    cache: &mut Cache,
     client: &Client,
     document_url: &Url,
     srcset: &str,
@@ -637,7 +637,7 @@ pub fn serialize_document(mut dom: RcDom, document_encoding: String, options: &O
 }
 
 pub fn retrieve_and_embed_asset(
-    cache: &mut HashMap<String, Vec<u8>>,
+    cache: &mut Cache,
     client: &Client,
     document_url: &Url,
     node: &Handle,
@@ -738,7 +738,7 @@ pub fn retrieve_and_embed_asset(
 }
 
 pub fn walk_and_embed_assets(
-    cache: &mut HashMap<String, Vec<u8>>,
+    cache: &mut Cache,
     client: &Client,
     document_url: &Url,
     node: &Handle,
diff --git a/src/lib.rs b/src/lib.rs
index 460d388..cbe7549 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,3 +1,4 @@
+pub mod cache;
 pub mod cookies;
 pub mod css;
 pub mod html;
diff --git a/src/main.rs b/src/main.rs
index 803485d..d57c019 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,15 +1,17 @@
-use encoding_rs::Encoding;
-use markup5ever_rcdom::RcDom;
-use reqwest::blocking::Client;
-use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
-use std::collections::HashMap;
 use std::fs;
 use std::io::{self, prelude::*, Error, Write};
 use std::path::Path;
 use std::process;
 use std::time::Duration;
+
+use encoding_rs::Encoding;
+use markup5ever_rcdom::RcDom;
+use reqwest::blocking::Client;
+use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
+use tempfile::Builder;
 use url::Url;
 
+use monolith::cache::Cache;
 use monolith::cookies::parse_cookie_file_contents;
 use monolith::html::{
     add_favicon, create_metadata_tag, get_base_url, get_charset, has_favicon, html_to_dom,
@@ -64,6 +66,8 @@ pub fn read_stdin() -> Vec<u8> {
     }
 }
 
+const CACHE_ASSET_FILE_SIZE_THRESHOLD: usize = 1024 * 50; // Minimum asset file size (in bytes)
+
 fn main() {
     let mut options = Options::from_args();
 
@@ -146,11 +150,6 @@ fn main() {
             Ok(str) => match parse_cookie_file_contents(&str) {
                 Ok(cookies) => {
                     options.cookies = cookies;
-                    // for c in &cookies {
-                    //     // if !cookie.is_expired() {
-                    //         // options.cookies.append(c);
-                    //     // }
-                    // }
                 }
                 Err(_) => {
                     eprintln!("Could not parse specified cookie file");
@@ -165,7 +164,15 @@ fn main() {
     }
 
     // Initialize client
-    let mut cache = HashMap::new();
+    let temp_file = Builder::new()
+        .prefix(".monolith-")
+        .keep(!true)
+        .tempfile()
+        .unwrap();
+    let mut cache = Cache::new(
+        CACHE_ASSET_FILE_SIZE_THRESHOLD,
+        Some(temp_file.path().display().to_string()),
+    );
     let mut header_map = HeaderMap::new();
     if let Some(user_agent) = &options.user_agent {
         header_map.insert(
@@ -353,4 +360,7 @@ fn main() {
 
     // Write result into STDOUT or file
     output.write(&result).expect("Could not write output");
+
+    // Remove temporary file used for storing cache's database
+    // drop(temp_file);
 }
diff --git a/src/url.rs b/src/url.rs
index 7946021..a418c32 100644
--- a/src/url.rs
+++ b/src/url.rs
@@ -4,8 +4,8 @@ use url::Url;
 
 use crate::utils::{detect_media_type, parse_content_type};
 
-pub const EMPTY_IMAGE_DATA_URL: &'static str = "data:image/png;base64,\
-iVBORw0KGgoAAAANSUhEUgAAAA0AAAANCAQAAADY4iz3AAAAEUlEQVR42mNkwAkYR6UolgIACvgADsuK6xYAAAAASUVORK5CYII=";
+pub const EMPTY_IMAGE_DATA_URL: &'static str = "data:image/png,\
+%89PNG%0D%0A%1A%0A%00%00%00%0DIHDR%00%00%00%0D%00%00%00%0D%08%04%00%00%00%D8%E2%2C%F7%00%00%00%11IDATx%DAcd%C0%09%18G%A5%28%96%02%00%0A%F8%00%0E%CB%8A%EB%16%00%00%00%00IEND%AEB%60%82";
 
 pub fn clean_url(url: Url) -> Url {
     let mut url = url.clone();
diff --git a/src/utils.rs b/src/utils.rs
index 1a5e052..cf50974 100644
--- a/src/utils.rs
+++ b/src/utils.rs
@@ -1,10 +1,10 @@
 use reqwest::blocking::Client;
 use reqwest::header::{HeaderMap, HeaderValue, CONTENT_TYPE, COOKIE, REFERER};
-use std::collections::HashMap;
 use std::fs;
 use std::path::{Path, PathBuf};
 use url::Url;
 
+use crate::cache::Cache;
 use crate::opts::Options;
 use crate::url::{clean_url, get_referer_url, parse_data_url};
 
@@ -181,7 +181,7 @@ pub fn parse_content_type(content_type: &str) -> (String, String, bool) {
 }
 
 pub fn retrieve_asset(
-    cache: &mut HashMap<String, Vec<u8>>,
+    cache: &mut Cache,
     client: &Client,
     parent_url: &Url,
     url: &Url,
@@ -270,10 +270,10 @@ pub fn retrieve_asset(
             }
 
             Ok((
-                cache.get(&cache_key).unwrap().to_vec(),
+                cache.get(&cache_key).unwrap().0.to_vec(),
                 url.clone(),
-                "".to_string(),
-                "".to_string(),
+                cache.get(&cache_key).unwrap().1,
+                cache.get(&cache_key).unwrap().2,
             ))
         } else {
             if let Some(domains) = &options.domains {
@@ -369,7 +369,7 @@ pub fn retrieve_asset(
                     }
 
                     // Add retrieved resource to cache
-                    cache.insert(new_cache_key, data.clone());
+                    cache.set(&new_cache_key, &data, media_type.clone(), charset.clone());
 
                     // Return
                     Ok((data, response_url, media_type, charset))


Test patch:
diff --git a/tests/css/embed_css.rs b/tests/css/embed_css.rs
index 490fe8b..16cd4c7 100644
--- a/tests/css/embed_css.rs
+++ b/tests/css/embed_css.rs
@@ -9,15 +9,15 @@
 mod passing {
     use reqwest::blocking::Client;
     use reqwest::Url;
-    use std::collections::HashMap;
 
+    use monolith::cache::Cache;
     use monolith::css;
     use monolith::opts::Options;
     use monolith::url::EMPTY_IMAGE_DATA_URL;
 
     #[test]
     fn empty_input() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let document_url: Url = Url::parse("data:,").unwrap();
         let options = Options::default();
@@ -30,7 +30,7 @@ mod passing {
 
     #[test]
     fn trim_if_empty() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
         let options = Options::default();
@@ -43,7 +43,7 @@ mod passing {
 
     #[test]
     fn style_exclude_unquoted_images() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
         let mut options = Options::default();
@@ -75,7 +75,7 @@ mod passing {
 
     #[test]
     fn style_exclude_single_quoted_images() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let document_url: Url = Url::parse("data:,").unwrap();
         let mut options = Options::default();
@@ -107,7 +107,7 @@ mod passing {
 
     #[test]
     fn style_block() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let document_url: Url = Url::parse("file:///").unwrap();
         let mut options = Options::default();
@@ -129,7 +129,7 @@ mod passing {
 
     #[test]
     fn attribute_selectors() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
         let mut options = Options::default();
@@ -173,7 +173,7 @@ mod passing {
 
     #[test]
     fn import_string() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
         let mut options = Options::default();
@@ -201,7 +201,7 @@ mod passing {
 
     #[test]
     fn hash_urls() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
         let mut options = Options::default();
@@ -225,7 +225,7 @@ mod passing {
 
     #[test]
     fn transform_percentages_and_degrees() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
         let mut options = Options::default();
@@ -247,7 +247,7 @@ mod passing {
 
     #[test]
     fn unusual_indents() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
         let mut options = Options::default();
@@ -271,7 +271,7 @@ mod passing {
 
     #[test]
     fn exclude_fonts() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
         let mut options = Options::default();
@@ -319,7 +319,7 @@ mod passing {
 
     #[test]
     fn content() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let document_url: Url = Url::parse("data:,").unwrap();
         let mut options = Options::default();
@@ -344,7 +344,7 @@ mod passing {
 
     #[test]
     fn ie_css_hack() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let document_url: Url = Url::parse("data:,").unwrap();
         let mut options = Options::default();
diff --git a/tests/html/embed_srcset.rs b/tests/html/embed_srcset.rs
index f382a6d..2c3360b 100644
--- a/tests/html/embed_srcset.rs
+++ b/tests/html/embed_srcset.rs
@@ -9,15 +9,15 @@
 mod passing {
     use reqwest::blocking::Client;
     use reqwest::Url;
-    use std::collections::HashMap;
 
+    use monolith::cache::Cache;
     use monolith::html;
     use monolith::opts::Options;
     use monolith::url::EMPTY_IMAGE_DATA_URL;
 
     #[test]
     fn small_medium_large() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let srcset_value = "small.png 1x, medium.png 1.5x, large.png 2x";
         let mut options = Options::default();
@@ -42,7 +42,7 @@ mod passing {
 
     #[test]
     fn small_medium_only_medium_has_scale() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let srcset_value = "small.png, medium.png 1.5x";
         let mut options = Options::default();
@@ -64,7 +64,7 @@ mod passing {
 
     #[test]
     fn commas_within_file_names() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let srcset_value = "small,s.png 1x, large,l.png 2x";
         let mut options = Options::default();
@@ -86,7 +86,7 @@ mod passing {
 
     #[test]
     fn tabs_and_newlines_after_commas() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let srcset_value = "small,s.png 1x,\nmedium,m.png 2x,\nlarge,l.png 3x";
         let mut options = Options::default();
@@ -111,7 +111,7 @@ mod passing {
 
     #[test]
     fn no_whitespace_after_commas() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let srcset_value = "small,s.png 1x,medium,m.png 2x,large,l.png 3x";
         let mut options = Options::default();
@@ -136,7 +136,7 @@ mod passing {
 
     #[test]
     fn last_without_descriptor() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let srcset_value = "small,s.png 1x, medium,m.png 2x, large,l.png";
         let mut options = Options::default();
@@ -171,15 +171,15 @@ mod passing {
 mod failing {
     use reqwest::blocking::Client;
     use reqwest::Url;
-    use std::collections::HashMap;
 
+    use monolith::cache::Cache;
     use monolith::html;
     use monolith::opts::Options;
     use monolith::url::EMPTY_IMAGE_DATA_URL;
 
     #[test]
     fn trailing_comma() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
         let srcset_value = "small.png 1x, large.png 2x,";
         let mut options = Options::default();
diff --git a/tests/html/walk_and_embed_assets.rs b/tests/html/walk_and_embed_assets.rs
index 25eb173..5a8eda7 100644
--- a/tests/html/walk_and_embed_assets.rs
+++ b/tests/html/walk_and_embed_assets.rs
@@ -10,16 +10,16 @@ mod passing {
     use html5ever::serialize::{serialize, SerializeOpts};
     use markup5ever_rcdom::SerializableHandle;
     use reqwest::blocking::Client;
-    use std::collections::HashMap;
     use url::Url;
 
+    use monolith::cache::Cache;
     use monolith::html;
     use monolith::opts::Options;
     use monolith::url::EMPTY_IMAGE_DATA_URL;
 
     #[test]
     fn basic() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
 
         let html: &str = "<div><P></P></div>";
         let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
@@ -51,7 +51,7 @@ mod passing {
         let html = "<div><P></P><iframe src=\"\"></iframe></div>";
         let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
         let url: Url = Url::parse("http://localhost").unwrap();
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
 
         let mut options = Options::default();
         options.silent = true;
@@ -79,7 +79,7 @@ mod passing {
         let html = "<frameset><frame src=\"\"></frameset>";
         let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
         let url: Url = Url::parse("http://localhost").unwrap();
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
 
         let mut options = Options::default();
         options.silent = true;
@@ -112,7 +112,7 @@ mod passing {
         ";
         let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
         let url: Url = Url::parse("http://localhost").unwrap();
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
 
         let mut options = Options::default();
         options.no_css = true;
@@ -153,7 +153,7 @@ mod passing {
                     <div><img src=\"http://localhost/assets/mono_lisa.png\" /></div>";
         let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
         let url: Url = Url::parse("http://localhost").unwrap();
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
 
         let mut options = Options::default();
         options.no_images = true;
@@ -195,7 +195,7 @@ mod passing {
             "<body background=\"no/such/image.png\" background=\"no/such/image2.png\"></body>";
         let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
         let url: Url = Url::parse("http://localhost").unwrap();
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
 
         let mut options = Options::default();
         options.no_images = true;
@@ -224,7 +224,7 @@ mod passing {
         let html = "<frameset><frame src=\"http://trackbook.com\"></frameset>";
         let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
         let url: Url = Url::parse("http://localhost").unwrap();
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
 
         let mut options = Options::default();
         options.no_frames = true;
@@ -261,7 +261,7 @@ mod passing {
         let html = "<iframe src=\"http://trackbook.com\"></iframe>";
         let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
         let url: Url = Url::parse("http://localhost").unwrap();
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
 
         let mut options = Options::default();
         options.no_frames = true;
@@ -302,7 +302,7 @@ mod passing {
         ";
         let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
         let url: Url = Url::parse("http://localhost").unwrap();
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
 
         let mut options = Options::default();
         options.no_js = true;
@@ -342,7 +342,7 @@ mod passing {
                     <link integrity=\"sha384-12345\" rel=\"something\" href=\"https://some-site.com/some-file.ext\" />";
         let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
         let url: Url = Url::parse("http://localhost").unwrap();
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
 
         let mut options = Options::default();
         options.silent = true;
@@ -382,7 +382,7 @@ mod passing {
         ";
         let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
         let url: Url = Url::parse("http://localhost").unwrap();
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
 
         let mut options = Options::default();
         options.no_css = true;
@@ -425,7 +425,7 @@ mod passing {
         ";
         let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
         let url: Url = Url::parse("http://localhost").unwrap();
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
 
         let mut options = Options::default();
         options.no_css = true;
@@ -474,7 +474,7 @@ mod passing {
         ";
         let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
         let url: Url = Url::parse("http://localhost").unwrap();
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
 
         let mut options = Options::default();
         options.no_css = true;
@@ -521,7 +521,7 @@ mod passing {
         </html>";
         let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
         let url: Url = Url::parse("http://localhost").unwrap();
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
 
         let mut options = Options::default();
         options.no_images = true;
@@ -562,7 +562,7 @@ mod passing {
         let html = "<script id=\"data\" type=\"application/json\">{\"mono\":\"lith\"}</script>";
         let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
         let url: Url = Url::parse("http://localhost").unwrap();
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
 
         let mut options = Options::default();
         options.silent = true;
diff --git a/tests/utils/retrieve_asset.rs b/tests/utils/retrieve_asset.rs
index 54a2488..f7cd8c8 100644
--- a/tests/utils/retrieve_asset.rs
+++ b/tests/utils/retrieve_asset.rs
@@ -9,16 +9,16 @@
 mod passing {
     use reqwest::blocking::Client;
     use reqwest::Url;
-    use std::collections::HashMap;
     use std::env;
 
+    use monolith::cache::Cache;
     use monolith::opts::Options;
     use monolith::url;
     use monolith::utils;
 
     #[test]
     fn read_data_url() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
 
         let mut options = Options::default();
@@ -48,7 +48,7 @@ mod passing {
 
     #[test]
     fn read_local_file_with_file_url_parent() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
 
         let mut options = Options::default();
@@ -102,14 +102,14 @@ mod passing {
 mod failing {
     use reqwest::blocking::Client;
     use reqwest::Url;
-    use std::collections::HashMap;
 
+    use monolith::cache::Cache;
     use monolith::opts::Options;
     use monolith::utils;
 
     #[test]
     fn read_local_file_with_data_url_parent() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
 
         let mut options = Options::default();
@@ -134,7 +134,7 @@ mod failing {
 
     #[test]
     fn read_local_file_with_https_parent() {
-        let cache = &mut HashMap::new();
+        let cache = &mut Cache::new(0, None);
         let client = Client::new();
 
         let mut options = Options::default();