Repository: sherlock-project/sherlock

Issue:
Display response times in verbose mode
## Feature Request

It would be nice to display the response time for each request. 

### Why?

- It would help us determine which sites cause the program to slow down. If a site is very slow and non-critical, we could remove it to speed up Sherlock.
- The more metrics the merrier! Has very little overhead, so why not :)

### User Experience

We would only display this information in verbose mode (`-v` flag). The information can be displayed in each output line as:

```txt
[+] [78 ms] Quora: https://www.quora.com/profile/nareddyt
```

Furthermore, these response times can be exported in the CSV.

### Implementation Details

We would just need to store the time reach request was created in the `net_info` or `results_site` dictionaries. Then when the request is done, we would note the end time.

Note there is some math involved in calculating times. It's not a simple `end-start` because the `result` for each response is extracted in the order of requests (not as soon as a request finishes). This is something I still need to think about...


Discussion:
Sounds good. Maybe then we could sort the sites by their response time and check the fastest ones first.
Sounds great @nareddyt  😊, Let me know if you going to give it a hand ☺️ 
Here is my initial prototype of this feature. I was able to make use of [requests-futures' built-in hooks API](https://github.com/ross/requests-futures#working-in-the-background) to compute the response times per request in a parallel fashion. This introduces very little overhead.

![image](https://user-images.githubusercontent.com/11142171/50747375-96749200-1201-11e9-9185-52e0e6838673.png)

For now, I will keep this feature simple and only display the response times in the console / csv. We can work on more advanced uses of these metrics (like the suggestion by @Czechball) once this initial feature is merged.

I'll send out a PR once I clean up my code and also thoroughly test it :)

Resolving patch:
diff --git a/sherlock.py b/sherlock.py
--- a/sherlock.py
+++ b/sherlock.py
@@ -13,11 +13,12 @@
 import sys
 import platform
 import re
-from argparse import ArgumentParser, RawDescriptionHelpFormatter
-from concurrent.futures import ThreadPoolExecutor
+from time import time
 
 import requests
-from colorama import Back, Fore, Style, init
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+from concurrent.futures import ThreadPoolExecutor
+from colorama import Fore, Style, init
 from requests_futures.sessions import FuturesSession
 from torrequest import TorRequest
 
@@ -28,28 +29,78 @@
 # TODO: fix tumblr
 
 
+class ElapsedFuturesSession(FuturesSession):
+    """
+    Extends FutureSession to add a response time metric to each request.
+
+    This is taken (almost) directly from here: https://github.com/ross/requests-futures#working-in-the-background
+    """
+
+    def request(self, method, url, hooks={}, *args, **kwargs):
+        start = time()
+
+        def timing(r, *args, **kwargs):
+            elapsed_sec = time() - start
+            r.elapsed = round(elapsed_sec * 1000)
+
+        try:
+            if isinstance(hooks['response'], (list, tuple)):
+                # needs to be first so we don't time other hooks execution
+                hooks['response'].insert(0, timing)
+            else:
+                hooks['response'] = [timing, hooks['response']]
+        except KeyError:
+            hooks['response'] = timing
+
+        return super(ElapsedFuturesSession, self).request(method, url, hooks=hooks, *args, **kwargs)
+
+
 def open_file(fname):
     return open(fname, "a")
 
+
 def write_to_file(url, f):
     f.write(url + "\n")
 
+
 def final_score(amount, f):
     f.write("Total: "+str(amount) + "\n")
 
-def print_error(err, errstr, var, debug=False):
+
+def print_error(err, errstr, var, verbose=False):
     print(Style.BRIGHT + Fore.WHITE + "[" +
           Fore.RED + "-" +
           Fore.WHITE + "]" +
           Fore.RED + f" {errstr}" +
-          Fore.YELLOW + f" {err if debug else var}")
+          Fore.YELLOW + f" {err if verbose else var}")
+
+
+def format_response_time(response_time, verbose):
+    return " [{} ms]".format(response_time) if verbose else ""
+
+
+def print_found(social_network, url, response_time, verbose=False):
+    print((Style.BRIGHT + Fore.WHITE + "[" +
+           Fore.GREEN + "+" +
+           Fore.WHITE + "]" +
+           format_response_time(response_time, verbose) +
+           Fore.GREEN + " {}:").format(social_network), url)
+
+
+def print_not_found(social_network, response_time, verbose=False):
+    print((Style.BRIGHT + Fore.WHITE + "[" +
+           Fore.RED + "-" +
+           Fore.WHITE + "]" +
+           format_response_time(response_time, verbose) +
+           Fore.GREEN + " {}:" +
+           Fore.YELLOW + " Not Found!").format(social_network))
 
 
 def get_response(request_future, error_type, social_network, verbose=False):
     try:
         rsp = request_future.result()
         if rsp.status_code:
-            return rsp, error_type
+            return rsp, error_type, rsp.elapsed
     except requests.exceptions.HTTPError as errh:
         print_error(errh, "HTTP Error:", social_network, verbose)
     except requests.exceptions.ConnectionError as errc:
@@ -58,7 +109,7 @@ def get_response(request_future, error_type, social_network, verbose=False):
         print_error(errt, "Timeout Error:", social_network, verbose)
     except requests.exceptions.RequestException as err:
         print_error(err, "Unknown error:", social_network, verbose)
-    return None, ""
+    return None, "", -1
 
 
 def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False):
@@ -119,8 +170,8 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False):
         underlying_request = TorRequest()
         underlying_session = underlying_request.session()
 
-    # Create multi-threaded session for all requests
-    session = FuturesSession(executor=executor, session=underlying_session)
+    # Create multi-threaded session for all requests. Use our custom FuturesSession that exposes response time
+    session = ElapsedFuturesSession(executor=executor, session=underlying_session)
 
     # Results from analysis of all sites
     results_total = {}
@@ -193,10 +244,10 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False):
 
         # Retrieve future and ensure it has finished
         future = net_info["request_future"]
-        r, error_type = get_response(request_future=future,
-                                     error_type=error_type,
-                                     social_network=social_network,
-                                     verbose=verbose)
+        r, error_type, response_time = get_response(request_future=future,
+                                                    error_type=error_type,
+                                                    social_network=social_network,
+                                                    verbose=verbose)
 
         # Attempt to get request information
         try:
@@ -212,59 +263,35 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False):
             error = net_info.get("errorMsg")
             # Checks if the error message is in the HTML
             if not error in r.text:
-
-                print((Style.BRIGHT + Fore.WHITE + "[" +
-                       Fore.GREEN + "+" +
-                       Fore.WHITE + "]" +
-                       Fore.GREEN + " {}:").format(social_network), url)
+                print_found(social_network, url, response_time, verbose)
                 write_to_file(url, f)
                 exists = "yes"
                 amount=amount+1
             else:
-                print((Style.BRIGHT + Fore.WHITE + "[" +
-                       Fore.RED + "-" +
-                       Fore.WHITE + "]" +
-                       Fore.GREEN + " {}:" +
-                       Fore.YELLOW + " Not Found!").format(social_network))
+                print_not_found(social_network, response_time, verbose)
                 exists = "no"
 
         elif error_type == "status_code":
             # Checks if the status code of the response is 2XX
             if not r.status_code >= 300 or r.status_code < 200:
-
-                print((Style.BRIGHT + Fore.WHITE + "[" +
-                       Fore.GREEN + "+" +
-                       Fore.WHITE + "]" +
-                       Fore.GREEN + " {}:").format(social_network), url)
+                print_found(social_network, url, response_time, verbose)
                 write_to_file(url, f)
                 exists = "yes"
                 amount=amount+1
             else:
-                print((Style.BRIGHT + Fore.WHITE + "[" +
-                       Fore.RED + "-" +
-                       Fore.WHITE + "]" +
-                       Fore.GREEN + " {}:" +
-                       Fore.YELLOW + " Not Found!").format(social_network))
+                print_not_found(social_network, response_time, verbose)
                 exists = "no"
 
         elif error_type == "response_url":
             error = net_info.get("errorUrl")
             # Checks if the redirect url is the same as the one defined in data.json
             if not error in r.url:
-
-                print((Style.BRIGHT + Fore.WHITE + "[" +
-                       Fore.GREEN + "+" +
-                       Fore.WHITE + "]" +
-                       Fore.GREEN + " {}:").format(social_network), url)
+                print_found(social_network, url, response_time, verbose)
                 write_to_file(url, f)
                 exists = "yes"
                 amount=amount+1
             else:
-                print((Style.BRIGHT + Fore.WHITE + "[" +
-                       Fore.RED + "-" +
-                       Fore.WHITE + "]" +
-                       Fore.GREEN + " {}:" +
-                       Fore.YELLOW + " Not Found!").format(social_network))
+                print_not_found(social_network, response_time, verbose)
                 exists = "no"
 
         elif error_type == "":
@@ -276,11 +303,12 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False):
             exists = "error"
 
         # Save exists flag
-        results_site['exists']        = exists
+        results_site['exists'] = exists
 
         # Save results from request
-        results_site['http_status']   = http_status
+        results_site['http_status'] = http_status
         results_site['response_text'] = response_text
+        results_site['response_time_ms'] = response_time
 
         # Add this site's results into final dictionary with all of the other results.
         results_total[social_network] = results_site
@@ -311,7 +339,7 @@ def main():
                        )
     parser.add_argument("--verbose", "-v", "-d", "--debug",
                         action="store_true",  dest="verbose", default=False,
-                        help="Display extra debugging information."
+                        help="Display extra debugging information and metrics."
                        )
     parser.add_argument("--quiet", "-q",
                         action="store_false", dest="verbose",
@@ -393,7 +421,8 @@ def main():
                                  'url_main',
                                  'url_user',
                                  'exists',
-                                 'http_status'
+                                 'http_status',
+                                 'response_time_ms'
                                 ]
                                )
                 for site in results:
@@ -402,9 +431,11 @@ def main():
                                      results[site]['url_main'],
                                      results[site]['url_user'],
                                      results[site]['exists'],
-                                     results[site]['http_status']
+                                     results[site]['http_status'],
+                                     results[site]['response_time_ms']
                                     ]
                                    )
 
+
 if __name__ == "__main__":
     main()
\ No newline at end of file


Test patch: