Repository: RocketMap/RocketMap

Issue:
Failure to connect to Webhook server has very cryptic log messages
The log when RM fails to connect to a webhook server does not mention that it is a webhook that fails to connect and does not mention the server in question.

## Expected Behavior
something along the lines of
`unable to connect to {WH_ip}:{port} Retries(4)`

`unable to connect to {WH_ip}:{port} Giving up`

## Current Behavior
Current log message:
Retrying (Retry(total=1, connect=None, read=None, redirect=None)) after connection broken by 'NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x1A205170>: Failed to establish a new connection: [Errno 10061] No connection could be made because the target machine actively refused it',)': /

## Possible Solution

## Steps to Reproduce (for bugs)
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
<!--- reproduce this bug. Include code to reproduce, if relevant -->
1. Set up a WH in config to ip/port that do not accept connections
2. Start scanning
3. See the message

## Context
Hours of sleep lost because of failing to understand that this message is from PokeAlarm stopped.

## Your Environment
North European enviroment, cold and harsh, 10-15 degrees C



Discussion:
👍 On it.
Surprised this finally got reported xD. Ontop of this iv noticed when one of two webhook servers is down it doesn't push to the webhook thats enabled. If you want to test it. mabye give it half an hour or so.
@fosJoddie Could you run #2304 (and approve if it works for you), then get it to log the retry exception details? It might not log it if the retries aren't logged in the generic handler, but I'm curious.
after running for 10 minutes with my Webhook server off, the only messages I get is the same

`
2017-09-22 00:24:58,730 [         Thread-23][connectionpool][ WARNING] Retrying (Retry(total=2, connect=None, read=None, redirect=None)) after connection broken by 'NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x0598FFB0>: Failed to establish a new connection: [Errno 10061] No connection could be made because the target machine actively refused it',)': /
`

It seems to never throw any critical log lines after the connection attempts has been exceeded.

@sebastienvercammen 

Resolving patch:
diff --git a/pogom/webhook.py b/pogom/webhook.py
--- a/pogom/webhook.py
+++ b/pogom/webhook.py
@@ -4,9 +4,11 @@
 import logging
 import requests
 import threading
+
 from queue import Empty
 from cachetools import LFUCache
 from timeit import default_timer
+
 from .utils import get_async_requests_session
 
 log = logging.getLogger(__name__)
@@ -17,7 +19,7 @@
 def send_to_webhooks(args, session, message_frame):
     if not args.webhooks:
         # What are you even doing here...
-        log.warning('Called send_to_webhook() without webhooks.')
+        log.critical('Called send_to_webhook() without webhooks.')
         return
 
     req_timeout = args.wh_timeout
@@ -26,15 +28,16 @@ def send_to_webhooks(args, session, message_frame):
         try:
             # Disable keep-alive and set streaming to True, so we can skip
             # the response content.
-            session.post(w, json=message_frame,
-                         timeout=(None, req_timeout),
-                         background_callback=__wh_completed,
-                         headers={'Connection': 'close'},
-                         stream=True)
+            future = session.post(w, json=message_frame,
+                                  timeout=(None, req_timeout),
+                                  background_callback=__wh_request_completed,
+                                  headers={'Connection': 'close'},
+                                  stream=True)
+            future.add_done_callback(__wh_future_completed)
         except requests.exceptions.ReadTimeout:
             log.exception('Response timeout on webhook endpoint %s.', w)
         except requests.exceptions.RequestException as e:
-            log.exception(e)
+            log.exception('Request exception on webhook: %s.', e)
 
 
 def wh_updater(args, queue, key_caches):
@@ -176,8 +179,21 @@ def wh_updater(args, queue, key_caches):
 
 
 # Helpers
-# Background handler for completed webhook requests.
-def __wh_completed(sess, resp):
+
+# Background handler for completed webhook futures.
+def __wh_future_completed(future):
+    # Handle any exceptions that might've occurred.
+    try:
+        exc = future.exception(timeout=0)
+
+        if exc:
+            log.exception("Something's wrong with your webhook: %s.", exc)
+    except Exception as ex:
+        log.exception('Unexpected exception in exception info: %s.', ex)
+
+
+# Background handler for completed webhook requests from requests lib.
+def __wh_request_completed(sess, resp):
     # Instantly close the response to release the connection back to the pool.
     resp.close()
 
diff --git a/runserver.py b/runserver.py
--- a/runserver.py
+++ b/runserver.py
@@ -8,6 +8,7 @@
 import re
 import ssl
 import json
+import requests
 
 from distutils.version import StrictVersion
 
@@ -482,11 +483,16 @@ def set_log_and_verbosity(log):
     logging.getLogger('werkzeug').setLevel(logging.ERROR)
     logging.getLogger('pogom.apiRequests').setLevel(logging.INFO)
 
+    # This sneaky one calls log.warning() on every retry.
+    urllib3_logger = logging.getLogger(requests.packages.urllib3.__package__)
+    urllib3_logger.setLevel(logging.ERROR)
+
     # Turn these back up if debugging.
     if args.verbose >= 2:
         logging.getLogger('pgoapi').setLevel(logging.DEBUG)
         logging.getLogger('pgoapi.pgoapi').setLevel(logging.DEBUG)
         logging.getLogger('requests').setLevel(logging.DEBUG)
+        urllib3_logger.setLevel(logging.INFO)
 
     if args.verbose >= 3:
         logging.getLogger('peewee').setLevel(logging.DEBUG)


Test patch: