Repository: rollbar/rollbar-agent

Issue:
Support running agent on Python 3
The rollbar-agent does not run on Python 3 (see https://github.com/rollbar/rollbar-agent/issues/53).

Python 2 has been officially sunsetted and is no longer supported, does not receive security patches, etc (see https://www.python.org/doc/sunset-python-2/)

Comments:

Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..7f35aa1
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,31 @@
+name: Rollbar-Agent CI
+
+on:
+  push:
+    branches: [ master ]
+    tags: [ v* ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python-version: [2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9]
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade --force-reinstall setuptools
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+      - name: Run tests
+        run: python setup.py test
diff --git a/README.rst b/README.rst
index fd1a6ae..d3dde4f 100644
--- a/README.rst
+++ b/README.rst
@@ -36,7 +36,7 @@ Requirements
 rollbar-agent requires:
 
 - A unix-like system (tested on Fedora and Ubuntu Linux and Mac OS X)
-- Python 2.6+
+- Python 3 or 2.6+
 - requests 0.13.1+ (will be installed by pip or setup.py, below)
 - a Rollbar_ account
 
diff --git a/rollbar-agent b/rollbar-agent
index c014b67..79d1f73 100755
--- a/rollbar-agent
+++ b/rollbar-agent
@@ -4,9 +4,7 @@ rollbar-agent: agent to monitor log files and send notices to Rollbar
 """
 import ast
 import codecs
-import ConfigParser
 import copy
-import dbm
 import fnmatch
 import hashlib
 import json
@@ -22,6 +20,16 @@ import sys
 import threading
 import time
 
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+
+try:
+    import dbm.ndbm as dbm
+except ImportError:
+    import dbm
+
 import requests
 
 log = logging.getLogger(__name__)
@@ -51,11 +59,21 @@ LOG_LEVEL = {
 
 terminal_character_attribute_pattern = re.compile(r'\x1b\[[0-9;]*m')
 
+
+def iteritems(_dict):
+    return _dict.items() if not hasattr(_dict, 'iteritems') else _dict.iteritems()
+
+
+def itervalues(_dict):
+    return _dict.values() if not hasattr(_dict, 'itervalues') else _dict.itervalues()
+
+
 def clean_line(app_config, line):
     if app_config.get('filter_chr_attr_sequences'):
         return terminal_character_attribute_pattern.sub("", line)
     return line
 
+
 def parse_timestamp(format, s):
     try:
         ts = time.mktime(time.strptime(s, format))
@@ -140,7 +158,7 @@ def build_python_log_format_parser(format, datefmt):
     }
 
     regex_str = '^' + re.escape(format) + '$'
-    for key, val in known_keys.iteritems():
+    for key, val in iteritems(known_keys):
         search = re.escape(key)
         replacement = "(?P<%s>%s)" % val
         regex_str = regex_str.replace(search, replacement)
@@ -177,7 +195,7 @@ def datefmt_to_regex(datefmt):
         '%Z': r'[A-Z]{3}',  # time zone name
     }
 
-    for key, val in replacements.iteritems():
+    for key, val in iteritems(replacements):
         datefmt = datefmt.replace(key, val)
 
     # last: replace %% with %
@@ -211,7 +229,11 @@ class Processor(object):
         # do immediate http post
         # in the future, will do this with batches and single separate thread
 
-        if isinstance(payload, basestring):
+        if sys.version_info[0] >= 3:
+            payload_is_str = isinstance(payload, str)
+        else:
+            payload_is_str = isinstance(payload, basestring)
+        if payload_is_str:
             payload = payload.encode('utf8')
 
         if options.dry_run:
@@ -318,7 +340,7 @@ class LogFileProcessor(Processor):
 
             try:
                 scrub_regexes.append(re.compile(pattern))
-            except Exception, e:
+            except Exception as e:
                 log.warning("Could not compile regex pattern: %s" % pattern)
 
         for line in fp:
@@ -342,7 +364,7 @@ class LogFileProcessor(Processor):
             for regex in scrub_regexes:
                 try:
                     line = regex.sub('******', line)
-                except Exception, e:
+                except Exception as e:
                     log.warning("Could not use regex %s on line %s" % (regex, line))
 
             current_message['data'].append(line)
@@ -429,7 +451,7 @@ class ScannerThread(threading.Thread):
         self.config = config
 
         self.apps = {}
-        for app_name, app_config in config.iteritems():
+        for app_name, app_config in iteritems(config):
             if app_name.startswith('_'):
                 continue
             self.apps[app_name] = {
@@ -468,7 +490,7 @@ class ScannerThread(threading.Thread):
         self.save_state(state)
 
         try:
-            for app in self.apps.itervalues():
+            for app in itervalues(self.apps):
                 self.scan_app(app, apps_state)
         except:
             log.exception("Caught exception in ScannerThread.scan_all() loop")
@@ -486,7 +508,8 @@ class ScannerThread(threading.Thread):
                 # the 'c' flag will only create the file if it doesn't exist
                 # https://docs.python.org/2/library/anydbm.html#anydbm.open
                 return shelve.open(self.config['_global']['statefile'],
-                                   flag='n' if recreate else 'c')
+                                   flag='n' if recreate else 'c',
+                                   protocol=self.config['_global']['state_proto'])
             except dbm.error:
                 log.error("Could not open statefile for writing. "
                     "Perhaps the directory doesn't exist? "
@@ -575,7 +598,7 @@ def main_loop():
 
     # sleep until the thread is killed
     # have to sleep in a loop, instead of worker.join(), otherwise we'll never get the signals
-    while scanner.isAlive():
+    while scanner.is_alive():
         time.sleep(1)
 
     log.info("Shutdown complete")
@@ -613,6 +636,7 @@ def parse_config(filename):
 
     defaults = {
         'statefile': '/var/cache/rollbar-agent.state',
+        'state_proto': '0',
         'sleep_time': '10',
         'endpoint': DEFAULT_ENDPOINT,
         'timeout': str(DEFAULT_TIMEOUT),
@@ -642,6 +666,7 @@ def parse_config(filename):
         return val.lower() == 'true'
 
     parsers = {
+        'state_proto': to_int,
         'sleep_time': to_int,
         'timeout': to_int,
         'ext_whitelist': to_list,
@@ -655,7 +680,10 @@ def parse_config(filename):
         'filter_chr_attr_sequences': to_bool,
     }
 
-    cp = ConfigParser.SafeConfigParser(defaults)
+    if sys.version_info >= (3, 2):
+        cp = configparser.ConfigParser(defaults)
+    else:
+        cp = configparser.SafeConfigParser(defaults)
     cp.read([filename])
 
     config = {'_formats': {}}
@@ -676,10 +704,10 @@ def parse_config(filename):
             format = {'name': format_name}
 
             format_type = cp.get(section_name, 'type')
-            format_spec = cp.get(section_name, 'format', True)
+            format_spec = cp.get(section_name, 'format', raw=True)
             try:
-                format_datefmt = cp.get(section_name, 'datefmt', True)
-            except ConfigParser.NoOptionError:
+                format_datefmt = cp.get(section_name, 'datefmt', raw=True)
+            except configparser.NoOptionError:
                 format_datefmt = DEFAULT_DATEFMT
 
             if format_type != 'python':
@@ -693,7 +721,7 @@ def parse_config(filename):
 
     global_config = cp.defaults()
     config['_global'] = {}
-    for option_name, raw_value in global_config.iteritems():
+    for option_name, raw_value in iteritems(global_config):
         if option_name in parsers:
             value = parsers[option_name](raw_value)
         else:
@@ -706,17 +734,17 @@ def parse_config(filename):
 def validate_config(config):
     errors = []
     required_vars = ['params.access_token', 'targets']
-    for app_name, app_config in config.iteritems():
+    for app_name, app_config in iteritems(config):
         if app_name.startswith('_'):
             continue
         for var_name in required_vars:
             if not app_config.get('params.access_token'):
                 errors.append("app:%s is missing required var %s" % (app_name, var_name))
     if errors:
-        print "CONFIGURATION ERRORS"
+        print("CONFIGURATION ERRORS")
         for error in errors:
-            print error
-        print
+            print(error)
+        print()
         sys.exit(1)
 
 
diff --git a/rollbar-agent.conf b/rollbar-agent.conf
index 9d62602..e9ad861 100644
--- a/rollbar-agent.conf
+++ b/rollbar-agent.conf
@@ -1,6 +1,14 @@
 [DEFAULT]
 # global option - where the state is kept.
 statefile = /var/cache/rollbar-agent.state
+
+# state protocol version (based on pickle protocols)
+#
+# defaults to 0 for backward compatibility with state files generated by Python2
+# note that for different versions of Python3 use different default values,
+# however you don't need to change the default 0
+state_proto = 0
+
 # target time how long to sleep between runs (in seconds). will never be longer than this.
 sleep_time = 10
 


Test patch:
diff --git a/test.py b/test.py
index b9fa076..a34217f 100644
--- a/test.py
+++ b/test.py
@@ -1,7 +1,15 @@
+import sys
+import types
 import unittest
-import imp
 
-rollbar_agent = imp.load_source('rollbar-agent', './rollbar-agent')
+if sys.version_info >= (3, 4):
+    from importlib.machinery import SourceFileLoader
+    loader = SourceFileLoader('rollbar-agent', './rollbar-agent')
+    rollbar_agent = types.ModuleType(loader.name)
+    loader.exec_module(rollbar_agent)
+else:
+    import imp
+    rollbar_agent = imp.load_source('rollbar-agent', './rollbar-agent')
 
 
 class FakeScanner: