Repository: epi052/feroxbuster (rust)

Issue:
[BUG] Double scan of the same directory (with and without trailing slash)  
**Describe the bug**
In some scans, all the directories found are scanned twice, one with and one without a trailing slash (/). This is maybe related to https://github.com/epi052/feroxbuster/issues/67.

**To Reproduce**
I've tried really hard to understand what are the reasons that make the bug happen or not, but I wasn't able to find the exact reason.
Steps to reproduce the behavior:
1. go into an empty directory for testing (`mkdir test && cd test`)
2. create an index.html file (`touch index.html`). This disables the directory listing.
3. launch a webserver in this directory (`python3 -m http.server 8000`)
4. start feroxbuster on the webserver (`feroxbuster -u http://127.0.0.1:8000`) and notice the issue.

```bash
$ feroxbuster -u http://127.0.0.1:8000 -w /opt/SecLists/Discovery/Web-Content/raft-medium-directories.txt

 ___  ___  __   __     __      __         __   ___
|__  |__  |__) |__) | /  `    /  \ \_/ | |  \ |__
|    |___ |  \ |  \ | \__,    \__/ / \ | |__/ |___
by Ben "epi" Risher 🤓                 ver: 2.7.1
───────────────────────────┬──────────────────────
 🎯  Target Url            │ http://127.0.0.1:8000
 🚀  Threads               │ 50
 📖  Wordlist              │ /opt/SecLists/Discovery/Web-Content/raft-medium-directories.txt
 👌  Status Codes          │ [200, 204, 301, 302, 307, 308, 401, 403, 405, 500]
 💥  Timeout (secs)        │ 7
 🦡  User-Agent            │ feroxbuster/2.7.1
 💉  Config File           │ /home/mh4ck/.config/feroxbuster/ferox-config.toml
 🏁  HTTP methods          │ [GET]
 🔃  Recursion Depth       │ 4
───────────────────────────┴──────────────────────
 🏁  Press [ENTER] to use the Scan Management Menu™
──────────────────────────────────────────────────
200      GET        0l        0w        0c http://127.0.0.1:8000/
[####################] - 20s    60000/60000   0s      found:1       errors:21     
[####################] - 18s    30000/30000   1687/s  http://127.0.0.1:8000 
[####################] - 20s    30000/30000   1519/s  http://127.0.0.1:8000/ 
```

**Expected behavior**
Each directory is scanned only once, without the dependency to a trailing slash (which doesn't change the results)

**Environment (please complete the following information):**
 - feroxbuster version: 2.7.1
 - OS : ArchLinux


Pull request:
661 fix double dir scan
fixes #661 

# Landing a Pull Request (PR)

Long form explanations of most of the items below can be found in the [CONTRIBUTING](https://github.com/epi052/feroxbuster/blob/master/CONTRIBUTING.md) guide.

## Branching checklist
- [ ] There is an issue associated with your PR (bug, feature, etc.. if not, create one)
- [ ] Your PR description references the associated issue (i.e. fixes #123456)
- [ ] Code is in its own branch
- [ ] Branch name is related to the PR contents
- [ ] PR targets main

## Static analysis checks
- [ ] All rust files are formatted using `cargo fmt`
- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic`
- [ ] All existing tests pass

## Documentation
- [ ] New code is documented using [doc comments](https://doc.rust-lang.org/stable/rust-by-example/meta/doc.html)
- [ ] Documentation about your PR is included in the `docs`, as needed. The docs live in a [separate repository](https://epi052.github.io/feroxbuster-docs/docs/). Update the appropriate pages at the links below.
  - [ ] update [example config file section](https://epi052.github.io/feroxbuster-docs/docs/configuration/ferox-config-toml/)
  - [ ] update [help output section](https://epi052.github.io/feroxbuster-docs/docs/configuration/command-line/)
  - [ ] add an [example](https://epi052.github.io/feroxbuster-docs/docs/examples/)
  - [ ] update [comparison table](https://epi052.github.io/feroxbuster-docs/docs/compare/)

## Additional Tests
- [ ] New code is unit tested
- [ ] New code is integration tested, as needed
- [ ] New tests pass


Patch:
diff --git a/src/scan_manager/scan.rs b/src/scan_manager/scan.rs
index ba8f4f6..6d416b2 100644
--- a/src/scan_manager/scan.rs
+++ b/src/scan_manager/scan.rs
@@ -32,6 +32,9 @@ pub struct FeroxScan {
     /// The URL that to be scanned
     pub(super) url: String,
 
+    /// A url used solely for comparison to other URLs
+    pub(super) normalized_url: String,
+
     /// The type of scan
     pub scan_type: ScanType,
 
@@ -79,6 +82,7 @@ impl Default for FeroxScan {
             num_requests: 0,
             scan_order: ScanOrder::Latest,
             url: String::new(),
+            normalized_url: String::new(),
             progress_bar: Mutex::new(None),
             scan_type: ScanType::File,
             output_level: Default::default(),
@@ -191,6 +195,7 @@ impl FeroxScan {
     ) -> Arc<Self> {
         Arc::new(Self {
             url: url.to_string(),
+            normalized_url: format!("{}/", url.trim_end_matches('/')),
             scan_type,
             scan_order,
             num_requests,
@@ -332,10 +337,11 @@ impl Serialize for FeroxScan {
     where
         S: Serializer,
     {
-        let mut state = serializer.serialize_struct("FeroxScan", 4)?;
+        let mut state = serializer.serialize_struct("FeroxScan", 6)?;
 
         state.serialize_field("id", &self.id)?;
         state.serialize_field("url", &self.url)?;
+        state.serialize_field("normalized_url", &self.normalized_url)?;
         state.serialize_field("scan_type", &self.scan_type)?;
         state.serialize_field("status", &self.status)?;
         state.serialize_field("num_requests", &self.num_requests)?;
@@ -387,6 +393,11 @@ impl<'de> Deserialize<'de> for FeroxScan {
                         scan.url = url.to_string();
                     }
                 }
+                "normalized_url" => {
+                    if let Some(normalized_url) = value.as_str() {
+                        scan.normalized_url = normalized_url.to_string();
+                    }
+                }
                 "num_requests" => {
                     if let Some(num_requests) = value.as_u64() {
                         scan.num_requests = num_requests;
@@ -480,6 +491,7 @@ mod tests {
         let scan = FeroxScan {
             id: "".to_string(),
             url: "".to_string(),
+            normalized_url: String::from("/"),
             scan_type: ScanType::Directory,
             scan_order: ScanOrder::Initial,
             num_requests: 0,
diff --git a/src/scan_manager/scan_container.rs b/src/scan_manager/scan_container.rs
index 84f0c34..33b23a3 100644
--- a/src/scan_manager/scan_container.rs
+++ b/src/scan_manager/scan_container.rs
@@ -213,8 +213,10 @@ impl FeroxScans {
     /// on the given URL
     pub fn contains(&self, url: &str) -> bool {
         if let Ok(scans) = self.scans.read() {
+            let normalized = format!("{}/", url.trim_end_matches('/'));
+
             for scan in scans.iter() {
-                if scan.url == url {
+                if scan.normalized_url == normalized {
                     return true;
                 }
             }
@@ -225,8 +227,10 @@ impl FeroxScans {
     /// Find and return a `FeroxScan` based on the given URL
     pub fn get_scan_by_url(&self, url: &str) -> Option<Arc<FeroxScan>> {
         if let Ok(guard) = self.scans.read() {
+            let normalized = format!("{}/", url.trim_end_matches('/'));
+
             for scan in guard.iter() {
-                if scan.url == url {
+                if scan.normalized_url == normalized {
                     return Some(scan.clone());
                 }
             }
@@ -589,7 +593,8 @@ impl FeroxScans {
     ///
     /// Also return a reference to the new `FeroxScan`
     pub fn add_directory_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc<FeroxScan>) {
-        self.add_scan(url, ScanType::Directory, scan_order)
+        let normalized = format!("{}/", url.trim_end_matches('/'));
+        self.add_scan(&normalized, ScanType::Directory, scan_order)
     }
 
     /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan


Test patch:
diff --git a/src/scan_manager/tests.rs b/src/scan_manager/tests.rs
index c9a0c00..eaa3f7d 100644
--- a/src/scan_manager/tests.rs
+++ b/src/scan_manager/tests.rs
@@ -277,7 +277,7 @@ fn ferox_scan_serialize() {
         None,
     );
     let fs_json = format!(
-        r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}"#,
+        r#"{{"id":"{}","url":"https://spiritanimal.com","normalized_url":"https://spiritanimal.com/","scan_type":"Directory","status":"NotStarted","num_requests":0}}"#,
         fs.id
     );
     assert_eq!(fs_json, serde_json::to_string(&*fs).unwrap());
@@ -296,7 +296,7 @@ fn ferox_scans_serialize() {
     );
     let ferox_scans = FeroxScans::default();
     let ferox_scans_json = format!(
-        r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}]"#,
+        r#"[{{"id":"{}","url":"https://spiritanimal.com","normalized_url":"https://spiritanimal.com/","scan_type":"Directory","status":"NotStarted","num_requests":0}}]"#,
         ferox_scan.id
     );
     ferox_scans.scans.write().unwrap().push(ferox_scan);
@@ -556,6 +556,7 @@ fn feroxscan_display() {
     let scan = FeroxScan {
         id: "".to_string(),
         url: String::from("http://localhost"),
+        normalized_url: String::from("http://localhost/"),
         scan_order: ScanOrder::Latest,
         scan_type: Default::default(),
         num_requests: 0,
@@ -600,6 +601,7 @@ async fn ferox_scan_abort() {
     let scan = FeroxScan {
         id: "".to_string(),
         url: String::from("http://localhost"),
+        normalized_url: String::from("http://localhost/"),
         scan_order: ScanOrder::Latest,
         scan_type: Default::default(),
         num_requests: 0,
diff --git a/tests/test_scan_manager.rs b/tests/test_scan_manager.rs
index ce41745..067684c 100644
--- a/tests/test_scan_manager.rs
+++ b/tests/test_scan_manager.rs
@@ -20,11 +20,13 @@ fn resume_scan_works() {
     // localhost:PORT/ <- complete
     // localhost:PORT/js <- will get scanned with /css and /stuff
     let complete_scan = format!(
-        r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","scan_type":"Directory","status":"Complete"}}"#,
-        srv.url("/")
+        r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","normalized_url":"{}","scan_type":"Directory","status":"Complete"}}"#,
+        srv.url("/"),
+        srv.url("/"),
     );
     let incomplete_scan = format!(
-        r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","scan_type":"Directory","status":"NotStarted"}}"#,
+        r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","normalized_url":"{}/","scan_type":"Directory","status":"NotStarted"}}"#,
+        srv.url("/js"),
         srv.url("/js")
     );
     let scans = format!(r#""scans":[{},{}]"#, complete_scan, incomplete_scan);