Repository: maurosoria/dirsearch

Issue:
Recursively scan directories that were found via web crawling
### What is the feature?

We should add directories/sub-directories that are contained in paths, found via crawling to the recursion queue no matter whether those paths are wildcard or not. Maybe an optional argument?

### What is the use case?

Sometimes we find files (paths to files) in the response that are no longer there, maybe they were deleted, but the folders/directories still exist and might contain something interesting

Hints from issue comments:
@Prady18: Okay just try 👽👽

Solution patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a0642d6..3b01f15 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: [3.9, 3.11]
         os: ['ubuntu-latest', 'windows-latest']
 
     steps:
@@ -24,7 +24,7 @@ jobs:
         pip install codespell flake8 -r requirements.txt
     - name: Test
       run: |
-        python3 dirsearch.py -w ./tests/static/wordlist.txt -u https://example.com -o "tmp_report.${extension}" -O json,xml,plain,csv,md,sqlite,html --force-recursive -R 3 --full-url -q
+        python3 dirsearch.py -w ./tests/static/wordlist.txt -u https://example.com -o "tmp_report.{extension}" -O json,xml,plain,csv,md,sqlite,html --force-recursive -R 3 --full-url -q
         python3 dirsearch.py -w ./tests/static/wordlist.txt -l ./tests/static/targets.txt --subdirs /,admin/ --exclude-extensions conf -q -L -f -i 200 --user-agent a --log tmp_log.log
         python3 dirsearch.py -w ./tests/static/wordlist.txt --nmap-report ./tests/static/nmap.xml --max-rate 2 -H K:V --random-agent --overwrite-extensions --no-color
         python3 dirsearch.py -w ./tests/static/wordlist.txt --raw ./tests/static/raw.txt --prefixes . --suffixes ~ --skip-on-status 404 -m POST -d test=1 --crawl --min-response-size 9
diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
index 5391c4f..ab2c275 100644
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -71,7 +71,6 @@
 - [at0m](https://github.com/atomiczsec/)
 - [junmoka](https://github.com/junmoka)
 - [Akshay Ravi](https://www.linkedin.com/in/c09yc47/)
-- [kosyan62](https://https://github.com/kosyan62)
 - [Maxence Zolnieurck](https://github.com/mxcezl)
 - [Giorgos Drosos](https://github.com/gdrosos)
 - [huyphan](https://github.com/huyphan)
diff --git a/README.md b/README.md
index e1dc407..f85cd6e 100644
--- a/README.md
+++ b/README.md
@@ -55,7 +55,7 @@ Table of Contents
 Installation & Usage
 ------------
 
-**Requirement: python 3.8 or higher**
+**Requirement: python 3.9 or higher**
 
 Choose one of these installation options:
 
diff --git a/config.ini b/config.ini
index afafe4e..d2ebc13 100644
--- a/config.ini
+++ b/config.ini
@@ -28,7 +28,7 @@ exit-on-error = False
 #skip-on-status = 429,999
 
 [dictionary]
-default-extensions = php,aspx,jsp,html,js
+default-extensions = php,asp,aspx,jsp,html,htm
 force-extensions = False
 overwrite-extensions = False
 lowercase = False
diff --git a/dirsearch.py b/dirsearch.py
index 05af81c..8323362 100755
--- a/dirsearch.py
+++ b/dirsearch.py
@@ -27,7 +27,7 @@ from lib.core.installation import check_dependencies, install_dependencies
 from lib.core.settings import OPTIONS_FILE
 from lib.parse.config import ConfigParser
 
-if sys.version_info < (3, 8):
+if sys.version_info < (3, 9):
     sys.stderr.write("Sorry, dirsearch requires Python 3.8 or higher\n")
     sys.exit(1)
 
diff --git a/lib/controller/controller.py b/lib/controller/controller.py
index 4a07042..56620b4 100755
--- a/lib/controller/controller.py
+++ b/lib/controller/controller.py
@@ -63,6 +63,7 @@ from lib.parse.rawrequest import parse_raw
 from lib.parse.url import clean_path, parse_path
 from lib.report.manager import ReportManager
 from lib.utils.common import lstrip_once
+from lib.utils.crawl import Crawler
 from lib.utils.file import FileUtils
 from lib.utils.schemedet import detect_scheme
 from lib.view.terminal import interface
@@ -269,7 +270,7 @@ class Controller:
 
                 if not self.old_session:
                     current_time = time.strftime("%H:%M:%S")
-                    msg = f"{NEW_LINE}[{current_time}] Starting: {current_directory}"
+                    msg = f"{NEW_LINE}[{current_time}] Scanning: {current_directory}"
 
                     interface.warning(msg)
 
@@ -400,6 +401,13 @@ class Controller:
             else:
                 self.requester.request(response.full_path, proxy=options["replay_proxy"])
 
+        if options["crawl"]:
+            for path in Crawler.crawl(response):
+                if not self.dictionary.is_valid(path):
+                    continue
+                path = lstrip_once(path, self.base_path)
+                self.dictionary.add_extra(path)
+
     def update_progress_bar(self, response: BaseResponse) -> None:
         jobs_count = (
             # Jobs left for unscanned targets
@@ -507,14 +515,13 @@ class Controller:
                         raise SkipTargetInterrupt(
                             "Runtime exceeded the maximum set by the user"
                         )
+                    time.sleep(0.5)
 
                 break
 
             except KeyboardInterrupt:
                 self.handle_pause()
 
-            time.sleep(0.3)
-
     def add_directory(self, path: str) -> None:
         """Add directory to the recursion queue"""
 
diff --git a/lib/core/dictionary.py b/lib/core/dictionary.py
index 1424de2..d4d7dd8 100755
--- a/lib/core/dictionary.py
+++ b/lib/core/dictionary.py
@@ -62,6 +62,9 @@ class Dictionary:
     def __init__(self, **kwargs: Any) -> None:
         self._index = 0
         self._items = self.generate(**kwargs)
+        # Items in self._extra will be cleared when self.reset() is called
+        self._extra_index = 0
+        self._extra = []
 
     @property
     def index(self) -> int:
@@ -69,23 +72,23 @@ class Dictionary:
 
     @locked
     def __next__(self) -> str:
-        try:
-            path = self._items[self._index]
-        except IndexError:
+        if len(self._extra) > self._extra_index:
+            self._extra_index += 1
+            return self._extra[self._extra_index - 1]
+        elif len(self._items) > self._index:
+            self._index += 1
+            return self._items[self._index - 1]
+        else:
             raise StopIteration
 
-        self._index += 1
-
-        return path
-
     def __contains__(self, item: str) -> bool:
         return item in self._items
 
     def __getstate__(self) -> tuple[list[str], int]:
-        return self._items, self._index
+        return self._items, self._index, self._extra, self._extra_index
 
     def __setstate__(self, state: tuple[list[str], int]) -> None:
-        self._items, self._index = state
+        self._items, self._index, self._extra, self._extra_index = state
 
     def __iter__(self) -> Iterator[str]:
         return iter(self._items)
@@ -209,5 +212,12 @@ class Dictionary:
 
         return True
 
+    def add_extra(self, path) -> None:
+        if path in self._items or path in self._extra:
+            return
+
+        self._extra.append(path)
+
     def reset(self) -> None:
-        self._index = 0
+        self._index = self._extra_index = 0
+        self._extra.clear()
diff --git a/lib/core/fuzzer.py b/lib/core/fuzzer.py
index 2ce4217..95e07b0 100755
--- a/lib/core/fuzzer.py
+++ b/lib/core/fuzzer.py
@@ -38,7 +38,6 @@ from lib.core.settings import (
 )
 from lib.parse.url import clean_path
 from lib.utils.common import get_readable_size, lstrip_once
-from lib.utils.crawl import Crawler
 
 
 class BaseFuzzer:
@@ -51,7 +50,6 @@ class BaseFuzzer:
         not_found_callbacks: tuple[Callable[[BaseResponse], Any], ...],
         error_callbacks: tuple[Callable[[RequestException], Any], ...],
     ) -> None:
-        self._scanned = set()
         self._requester = requester
         self._dictionary = dictionary
         self._base_path: str = ""
@@ -237,13 +235,8 @@ class Fuzzer(BaseFuzzer):
         self._quit_event.set()
         self.play()
 
-    def scan(self, path: str, scanners: Generator[Scanner, None, None]) -> None:
-        # Avoid scanned paths from being re-scanned
-        if path in self._scanned:
-            return
-        else:
-            self._scanned.add(path)
-
+    def scan(self, path: str) -> None:
+        scanners = self.get_scanners_for(path)
         response = self._requester.request(path)
 
         if self.is_excluded(response):
@@ -264,23 +257,13 @@ class Fuzzer(BaseFuzzer):
         except Exception as e:
             self.exc = e
 
-        if options["crawl"]:
-            logger.info(f'THREAD-{threading.get_ident()}: crawling "/{path}"')
-            for path_ in Crawler.crawl(response):
-                if self._dictionary.is_valid(path_):
-                    logger.info(
-                        f'THREAD-{threading.get_ident()}: found new path "/{path_}" in /{path}'
-                    )
-                    self.scan(path_, self.get_scanners_for(path_))
-
     def thread_proc(self) -> None:
         logger.info(f'THREAD-{threading.get_ident()} started"')
 
         while True:
             try:
                 path = next(self._dictionary)
-                scanners = self.get_scanners_for(path)
-                self.scan(self._base_path + path, scanners)
+                self.scan(self._base_path + path)
 
             except StopIteration:
                 break
@@ -397,13 +380,8 @@ class AsyncFuzzer(BaseFuzzer):
         for task in self._background_tasks:
             task.cancel()
 
-    async def scan(self, path: str, scanners: Generator) -> None:
-        # Avoid scanned paths from being re-scanned
-        if path in self._scanned:
-            return
-        else:
-            self._scanned.add(path)
-
+    async def scan(self, path: str) -> None:
+        scanners = self.get_scanners_for(path)
         response = await self._requester.request(path)
 
         if self.is_excluded(response):
@@ -424,16 +402,6 @@ class AsyncFuzzer(BaseFuzzer):
         except Exception as e:
             self.exc = e
 
-        if options["crawl"]:
-            task = asyncio.current_task()
-            logger.info(f'{task.get_name()}: crawling "/{path}"')
-            for path_ in Crawler.crawl(response):
-                if self._dictionary.is_valid(path_):
-                    logger.info(
-                        f'{task.get_name()}: found new path "/{path_}" in /{path}'
-                    )
-                    await self.scan(path_, self.get_scanners_for(path_))
-
     async def task_proc(self) -> None:
         async with self.sem:
             await self._play_event.wait()
diff --git a/lib/core/settings.py b/lib/core/settings.py
index 7d663b1..e242169 100755
--- a/lib/core/settings.py
+++ b/lib/core/settings.py
@@ -115,8 +115,6 @@ SOCKET_TIMEOUT = 6
 
 RATE_UPDATE_DELAY = 0.15
 
-MAX_MATCH_RATIO = 0.98
-
 ITER_CHUNK_SIZE = 1024 * 1024
 
 MAX_RESPONSE_SIZE = 80 * 1024 * 1024
diff --git a/lib/report/xml_report.py b/lib/report/xml_report.py
index 7a7ea1e..c540f4d 100755
--- a/lib/report/xml_report.py
+++ b/lib/report/xml_report.py
@@ -16,7 +16,7 @@
 #
 #  Author: Mauro Soria
 
-from defusedxml import ElementTree as ET
+from xml.etree import ElementTree as ET
 
 from lib.core.decorators import locked
 from lib.core.settings import (
diff --git a/lib/utils/diff.py b/lib/utils/diff.py
index f78f6d7..d0249df 100755
--- a/lib/utils/diff.py
+++ b/lib/utils/diff.py
@@ -19,7 +19,7 @@
 import difflib
 import re
 
-from lib.core.settings import MAX_MATCH_RATIO
+from lib.utils.common import lstrip_once
 
 
 class DynamicContentParser:
@@ -38,20 +38,29 @@ class DynamicContentParser:
         """
         DynamicContentParser.compare_to() workflow
 
-          1. Check if the wildcard response is static or not, if yes, compare 2 responses
-          2. If it's not static, get static patterns (splitting by space) in both responses
-            and check if they match
-          3. In some rare cases, checking static patterns fails, so make a final confirmation
-            if the similarity ratio of 2 responses is not high enough to prove they are the same
+          1. Check if the wildcard response is static or not, if yes, compare two responses.
+          2. If it's not static, get static patterns (split by space) and check if the response
+            has all of them.
+          3. In some cases, checking static patterns isn't reliable enough, so we check the similarity
+            ratio of the two responses.
         """
 
-        if self._is_static and content == self._base_content:
-            return True
+        if self._is_static:
+            return content == self._base_content
 
-        diff = self._differ.compare(self._base_content.split(), content.split())
-        static_patterns_are_matched = self._static_patterns == self.get_static_patterns(diff)
-        match_ratio = difflib.SequenceMatcher(None, self._base_content, content).ratio()
-        return static_patterns_are_matched or match_ratio > MAX_MATCH_RATIO
+        i = -1
+        splitted_content = content.split()
+        for pattern in self._static_patterns:
+            try:
+                i = splitted_content.index(pattern, i + 1)
+            except IndexError:
+                return False
+
+        # The number of static patterns is not big enough to say it's a reliable method
+        if len(self._static_patterns) < 20 and len(content.split()) > len(self._base_content.split()):
+            return difflib.SequenceMatcher(None, self._base_content, content).ratio() > 0.75
+
+        return True
 
     @staticmethod
     def get_static_patterns(patterns):
@@ -59,7 +68,7 @@ class DynamicContentParser:
         # ["  str1", "- str2", "+ str3", "  str4"]
         #
         # Get only stable patterns in the contents
-        return [pattern for pattern in patterns if pattern.startswith("  ")]
+        return [lstrip_once(pattern, "  ") for pattern in patterns if pattern.startswith("  ")]
 
 
 def generate_matching_regex(string1: str, string2: str) -> str:
diff --git a/setup.py b/setup.py
index bc7cfc1..99ec691 100755
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@ setuptools.setup(
     entry_points={"console_scripts": ["dirsearch=dirsearch.dirsearch:main"]},
     package_data={"dirsearch": ["*", "db/*"]},
     include_package_data=True,
-    python_requires=">=3.8",
+    python_requires=">=3.9",
     install_requires=get_dependencies(),
     classifiers=[
         "Programming Language :: Python",
@@ -39,7 +39,7 @@ setuptools.setup(
         "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
         "Operating System :: OS Independent",
         "Topic :: Security",
-        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
     ],
     keywords=["infosec", "bug bounty", "pentesting", "security"],
 )


Test patch:
diff --git a/tests/utils/test_diff.py b/tests/utils/test_diff.py
index 769b87f..c21e1a0 100755
--- a/tests/utils/test_diff.py
+++ b/tests/utils/test_diff.py
@@ -26,5 +26,6 @@ class TestDiff(TestCase):
         self.assertEqual(generate_matching_regex("add.php", "abc.php"), "^a.*\\.php$", "Matching regex isn't correct")
 
     def test_dynamic_content_parser(self):
-        self.assertEqual(DynamicContentParser("a b c", "a b d")._static_patterns, ["  a", "  b"], "Static patterns are not right")
-        self.assertTrue(DynamicContentParser("a b c", "a b d").compare_to("a b ef"))
+        self.assertEqual(DynamicContentParser("a b c", "a b d")._static_patterns, ["a", "b"], "Static patterns are not right")
+        self.assertTrue(DynamicContentParser("abc.php not found", "def.php not found").compare_to("nothing.php not found"))
+        self.assertTrue(DynamicContentParser("abc.php not found", "def.php not found").compare_to("zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz.php not found"))