Repository: googleapis/google-cloud-python

Issue:
Logging: Crash if using non default log level names with CloudLoggingHandler
#### Environment details

1. OS: Ubuntu 18.04.1 LTS (ubuntu-bionic)
2. Python version: 3.6.7
3. google-cloud-logging version: 1.10.0

#### Steps to reproduce

Use GCP logging with non default log level name. Crashes every time. See code below using the non default log level name 'ERR'.

#### Code example

```python
import logging

from google.cloud import logging as cloud_logging
from google.cloud.logging.handlers import CloudLoggingHandler

client = cloud_logging.Client()

handler = CloudLoggingHandler(client)
cloud_logger = logging.getLogger("cloudLogger")
cloud_logger.setLevel(logging.INFO)
cloud_logger.addHandler(handler)

cloud_logger.error("First bad news")  # See this in GCP console if the two lines below is not run

logging.addLevelName(logging.ERROR, "ERR")
cloud_logger.error("More bad news")  # <- Crash if this and the line above is run

```

#### Output
```
Program shutting down, attempting to send 2 queued log entries to Stackdriver Logging...
Waiting up to 5 seconds.
Failed to submit 2 logs.
Traceback (most recent call last):
  File "<my virtualenv>/lib/python3.6/site-packages/google/protobuf/json_format.py", line 691, in _ConvertScalarFieldValue
    number = int(value)
ValueError: invalid literal for int() with base 10: 'ERR'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<my virtualenv>/lib/python3.6/site-packages/google/protobuf/json_format.py", line 549, in _ConvertFieldValuePair
    setattr(message, field.name, _ConvertScalarFieldValue(value, field))
  File "<my virtualenv>/lib/python3.6/site-packages/google/protobuf/json_format.py", line 695, in _ConvertScalarFieldValue
    value, field.enum_type.full_name))
google.protobuf.json_format.ParseError: Invalid enum value ERR for enum type google.logging.type.LogSeverity.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<my virtualenv>/lib/python3.6/site-packages/google/cloud/logging/handlers/transports/background_thread.py", line 122, in _safely_commit_batch
    batch.commit()
  File "<my virtualenv>/lib/python3.6/site-packages/google/cloud/logging/logger.py", line 383, in commit
    client.logging_api.write_entries(entries, **kwargs)
  File "<my virtualenv>/lib/python3.6/site-packages/google/cloud/logging/_gapic.py", line 125, in write_entries
    entry_pbs = [_log_entry_mapping_to_pb(entry) for entry in entries]
  File "<my virtualenv>/lib/python3.6/site-packages/google/cloud/logging/_gapic.py", line 125, in <listcomp>
    entry_pbs = [_log_entry_mapping_to_pb(entry) for entry in entries]
  File "<my virtualenv>/lib/python3.6/site-packages/google/cloud/logging/_gapic.py", line 464, in _log_entry_mapping_to_pb
    ParseDict(mapping, entry_pb)
  File "<my virtualenv>/lib/python3.6/site-packages/google/protobuf/json_format.py", line 421, in ParseDict
    parser.ConvertMessage(js_dict, message)
  File "<my virtualenv>/lib/python3.6/site-packages/google/protobuf/json_format.py", line 452, in ConvertMessage
    self._ConvertFieldValuePair(value, message)
  File "<my virtualenv>/lib/python3.6/site-packages/google/protobuf/json_format.py", line 552, in _ConvertFieldValuePair
    raise ParseError('Failed to parse {0} field: {1}'.format(name, e))
google.protobuf.json_format.ParseError: Failed to parse severity field: Invalid enum value ERR for enum type google.logging.type.LogSeverity.
Sent all pending logs.

Process finished with exit code 0

```





Discussion:

Resolving patch:
diff --git a/logging/google/cloud/logging/_helpers.py b/logging/google/cloud/logging/_helpers.py
--- a/logging/google/cloud/logging/_helpers.py
+++ b/logging/google/cloud/logging/_helpers.py
@@ -14,6 +14,8 @@
 
 """Common logging helpers."""
 
+import logging
+
 import requests
 
 from google.cloud.logging.entries import LogEntry
@@ -21,6 +23,33 @@
 from google.cloud.logging.entries import StructEntry
 from google.cloud.logging.entries import TextEntry
 
+try:
+    from google.cloud.logging_v2.gapic.enums import LogSeverity
+except ImportError:  # pragma: NO COVER
+
+    class LogSeverity(object):
+        """Map severities for non-GAPIC usage."""
+
+        DEFAULT = 0
+        DEBUG = 100
+        INFO = 200
+        NOTICE = 300
+        WARNING = 400
+        ERROR = 500
+        CRITICAL = 600
+        ALERT = 700
+        EMERGENCY = 800
+
+
+_NORMALIZED_SEVERITIES = {
+    logging.CRITICAL: LogSeverity.CRITICAL,
+    logging.ERROR: LogSeverity.ERROR,
+    logging.WARNING: LogSeverity.WARNING,
+    logging.INFO: LogSeverity.INFO,
+    logging.DEBUG: LogSeverity.DEBUG,
+    logging.NOTSET: LogSeverity.DEFAULT,
+}
+
 METADATA_URL = "http://metadata.google.internal./computeMetadata/v1/"
 METADATA_HEADERS = {"Metadata-Flavor": "Google"}
 
@@ -82,3 +111,15 @@ def retrieve_metadata_server(metadata_key):
         pass
 
     return None
+
+
+def _normalize_severity(stdlib_level):
+    """Normalize a Python stdlib severity to LogSeverity enum.
+
+    :type stdlib_level: int
+    :param stdlib_level: 'levelno' from a :class:`logging.LogRecord`
+
+    :rtype: int
+    :returns: Corresponding Stackdriver severity.
+    """
+    return _NORMALIZED_SEVERITIES.get(stdlib_level, stdlib_level)
diff --git a/logging/google/cloud/logging/handlers/transports/background_thread.py b/logging/google/cloud/logging/handlers/transports/background_thread.py
--- a/logging/google/cloud/logging/handlers/transports/background_thread.py
+++ b/logging/google/cloud/logging/handlers/transports/background_thread.py
@@ -28,6 +28,7 @@
 
 from six.moves import queue
 
+from google.cloud.logging import _helpers
 from google.cloud.logging.handlers.transports.base import Transport
 
 _DEFAULT_GRACE_PERIOD = 5.0  # Seconds
@@ -254,17 +255,16 @@ def enqueue(
         :param span_id: (optional) span_id within the trace for the log entry.
                         Specify the trace parameter if span_id is set.
         """
-        self._queue.put_nowait(
-            {
-                "info": {"message": message, "python_logger": record.name},
-                "severity": record.levelname,
-                "resource": resource,
-                "labels": labels,
-                "trace": trace,
-                "span_id": span_id,
-                "timestamp": datetime.datetime.utcfromtimestamp(record.created),
-            }
-        )
+        queue_entry = {
+            "info": {"message": message, "python_logger": record.name},
+            "severity": _helpers._normalize_severity(record.levelno),
+            "resource": resource,
+            "labels": labels,
+            "trace": trace,
+            "span_id": span_id,
+            "timestamp": datetime.datetime.utcfromtimestamp(record.created),
+        }
+        self._queue.put_nowait(queue_entry)
 
     def flush(self):
         """Submit any pending log records."""
diff --git a/logging/google/cloud/logging/handlers/transports/sync.py b/logging/google/cloud/logging/handlers/transports/sync.py
--- a/logging/google/cloud/logging/handlers/transports/sync.py
+++ b/logging/google/cloud/logging/handlers/transports/sync.py
@@ -17,6 +17,7 @@
 Logs directly to the the Stackdriver Logging API with a synchronous call.
 """
 
+from google.cloud.logging import _helpers
 from google.cloud.logging.handlers.transports.base import Transport
 
 
@@ -50,7 +51,7 @@ def send(
         info = {"message": message, "python_logger": record.name}
         self.logger.log_struct(
             info,
-            severity=record.levelname,
+            severity=_helpers._normalize_severity(record.levelno),
             resource=resource,
             labels=labels,
             trace=trace,


Test patch:
diff --git a/logging/tests/system/test_system.py b/logging/tests/system/test_system.py
--- a/logging/tests/system/test_system.py
+++ b/logging/tests/system/test_system.py
@@ -105,13 +105,16 @@ def setUp(self):
         self._handlers_cache = logging.getLogger().handlers[:]
 
     def tearDown(self):
-        retry = RetryErrors((NotFound, TooManyRequests, RetryError), max_tries=9)
+        retry_not_found = RetryErrors((NotFound), max_tries=4)
+        retry_other = RetryErrors((TooManyRequests, RetryError))
         for doomed in self.to_delete:
             try:
-                retry(doomed.delete)()
+                retry_not_found(retry_other(doomed.delete))()
             except AttributeError:
                 client, dataset = doomed
-                retry(client.delete_dataset)(dataset)
+                retry_not_found(retry_other(client.delete_dataset))(dataset)
+            except NotFound:
+                pass
         logging.getLogger().handlers = self._handlers_cache[:]
 
     @staticmethod
diff --git a/logging/tests/unit/handlers/transports/test_background_thread.py b/logging/tests/unit/handlers/transports/test_background_thread.py
--- a/logging/tests/unit/handlers/transports/test_background_thread.py
+++ b/logging/tests/unit/handlers/transports/test_background_thread.py
@@ -264,11 +264,62 @@ def test__main_thread_terminated_did_not_join(self):
         self.assertFalse(worker.is_alive)
 
     @staticmethod
-    def _enqueue_record(worker, message):
-        record = logging.LogRecord(
-            "python_logger", logging.INFO, None, None, message, None, None
+    def _enqueue_record(worker, message, levelno=logging.INFO, **kw):
+        record = logging.LogRecord("testing", levelno, None, None, message, None, None)
+        worker.enqueue(record, message, **kw)
+
+    def test_enqueue_defaults(self):
+        import datetime
+        from google.cloud.logging._helpers import LogSeverity
+
+        worker = self._make_one(_Logger(self.NAME))
+        self.assertTrue(worker._queue.empty())
+        message = "TEST SEVERITY"
+
+        self._enqueue_record(worker, message)
+
+        entry = worker._queue.get_nowait()
+        expected_info = {"message": message, "python_logger": "testing"}
+        self.assertEqual(entry["info"], expected_info)
+        self.assertEqual(entry["severity"], LogSeverity.INFO)
+        self.assertIsNone(entry["resource"])
+        self.assertIsNone(entry["labels"])
+        self.assertIsNone(entry["trace"])
+        self.assertIsNone(entry["span_id"])
+        self.assertIsInstance(entry["timestamp"], datetime.datetime)
+
+    def test_enqueue_explicit(self):
+        import datetime
+        from google.cloud.logging._helpers import LogSeverity
+
+        worker = self._make_one(_Logger(self.NAME))
+        self.assertTrue(worker._queue.empty())
+        message = "TEST SEVERITY"
+        resource = object()
+        labels = {"foo": "bar"}
+        trace = "TRACE"
+        span_id = "SPAN_ID"
+
+        self._enqueue_record(
+            worker,
+            message,
+            levelno=logging.ERROR,
+            resource=resource,
+            labels=labels,
+            trace=trace,
+            span_id=span_id,
         )
-        worker.enqueue(record, message)
+
+        entry = worker._queue.get_nowait()
+
+        expected_info = {"message": message, "python_logger": "testing"}
+        self.assertEqual(entry["info"], expected_info)
+        self.assertEqual(entry["severity"], LogSeverity.ERROR)
+        self.assertIs(entry["resource"], resource)
+        self.assertIs(entry["labels"], labels)
+        self.assertIs(entry["trace"], trace)
+        self.assertIs(entry["span_id"], span_id)
+        self.assertIsInstance(entry["timestamp"], datetime.datetime)
 
     def test__thread_main(self):
         from google.cloud.logging.handlers.transports import background_thread
diff --git a/logging/tests/unit/handlers/transports/test_sync.py b/logging/tests/unit/handlers/transports/test_sync.py
--- a/logging/tests/unit/handlers/transports/test_sync.py
+++ b/logging/tests/unit/handlers/transports/test_sync.py
@@ -37,6 +37,7 @@ def test_ctor(self):
 
     def test_send(self):
         from google.cloud.logging.logger import _GLOBAL_RESOURCE
+        from google.cloud.logging._helpers import LogSeverity
 
         client = _Client(self.PROJECT)
 
@@ -50,7 +51,14 @@ def test_send(self):
 
         transport.send(record, message, _GLOBAL_RESOURCE)
         EXPECTED_STRUCT = {"message": message, "python_logger": python_logger_name}
-        EXPECTED_SENT = (EXPECTED_STRUCT, "INFO", _GLOBAL_RESOURCE, None, None, None)
+        EXPECTED_SENT = (
+            EXPECTED_STRUCT,
+            LogSeverity.INFO,
+            _GLOBAL_RESOURCE,
+            None,
+            None,
+            None,
+        )
         self.assertEqual(transport.logger.log_struct_called_with, EXPECTED_SENT)
 
 
diff --git a/logging/tests/unit/test__helpers.py b/logging/tests/unit/test__helpers.py
--- a/logging/tests/unit/test__helpers.py
+++ b/logging/tests/unit/test__helpers.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 
+import logging
 import unittest
 
 import mock
@@ -121,6 +122,47 @@ def test_request_exception(self):
         self.assertIsNone(metadata)
 
 
+class Test__normalize_severity(unittest.TestCase):
+    @staticmethod
+    def _stackdriver_severity():
+        from google.cloud.logging._helpers import LogSeverity
+
+        return LogSeverity
+
+    def _normalize_severity_helper(self, stdlib_level, enum_level):
+        from google.cloud.logging._helpers import _normalize_severity
+
+        self.assertEqual(_normalize_severity(stdlib_level), enum_level)
+
+    def test__normalize_severity_critical(self):
+        severity = self._stackdriver_severity()
+        self._normalize_severity_helper(logging.CRITICAL, severity.CRITICAL)
+
+    def test__normalize_severity_error(self):
+        severity = self._stackdriver_severity()
+        self._normalize_severity_helper(logging.ERROR, severity.ERROR)
+
+    def test__normalize_severity_warning(self):
+        severity = self._stackdriver_severity()
+        self._normalize_severity_helper(logging.WARNING, severity.WARNING)
+
+    def test__normalize_severity_info(self):
+        severity = self._stackdriver_severity()
+        self._normalize_severity_helper(logging.INFO, severity.INFO)
+
+    def test__normalize_severity_debug(self):
+        severity = self._stackdriver_severity()
+        self._normalize_severity_helper(logging.DEBUG, severity.DEBUG)
+
+    def test__normalize_severity_notset(self):
+        severity = self._stackdriver_severity()
+        self._normalize_severity_helper(logging.NOTSET, severity.DEFAULT)
+
+    def test__normalize_severity_non_standard(self):
+        unknown_level = 35
+        self._normalize_severity_helper(unknown_level, unknown_level)
+
+
 class EntryMock(object):
     def __init__(self):
         self.sentinel = object()