Repository: cloud-custodian/cloud-custodian

Issue:
ci - intermittent sqlite-related failures
I've had a couple CI builds run today associated with #4101 - tests succeed locally but fail intermittently in CI for one or more ubuntu-based python environments. I see similar failures for #4035  and #4090 . The failures are all related to sqlite, which appears to be missing from the build image sometimes. That causes test results to vary across builds.

I don't know if this is a transient issue or not. Is it worth having a script step for ubuntu-based builds to ensure that sqlite is available?


Discussion:
yeah this seems like a regression in the azure pipeline build image that is hitting us intermittently.
filed upstream as https://github.com/microsoft/azure-pipelines-image-generation/issues/996
till its resolved upstream, i think we need to address this directly via package install, i've put up a pr to do that.
unfortunately I'm not clear we can handle this in our pipeline yaml, I've tried a few different things re sqlite install, but its not working :-( I might be missing something though. I'm installing the standard deps for python on xenial/16.04 for sqlite3 and libsqlite3-0 but its not taking.
Dang :(. I wonder if some of the python versions in their hosted agent tools cache were built without sqlite support? I was thinking maybe it would need libsqlite3-dev or something, but by the time the pipeline is running maybe it's too late to recover without going to an ugly nuclear option like rebuilding python.
if this doesn't get resolved by eod friday, i'll bite the bullet and switch on ci. our days of dead ci in the past month is approaching a week, ie there have been other incidents this month. While pipelines has been great as a ci tool, its not tenable if its broken this often, as its holding up contributions and activity in the project.
That makes sense, understandable. I see that the attrs project worked around it by using deadsnakes python. Don't know if that's more or less attractive than switching ci. I guess that'd only cover 3.7 for us anyway, and for 2.7 we'd need to use the native ubuntu package instead of the one from agent tools? Not great options :(.

python-attrs/attrs#541
for now i've disabled 2.7/3.7 builds to unblock ci.
unfortunately changing ci isn't viable atm as capitalone has made the organization/repo permissions unusable for changes to ci.

Resolving patch:
diff --git a/tools/c7n_mailer/c7n_mailer/ldap_lookup.py b/tools/c7n_mailer/c7n_mailer/ldap_lookup.py
--- a/tools/c7n_mailer/c7n_mailer/ldap_lookup.py
+++ b/tools/c7n_mailer/c7n_mailer/ldap_lookup.py
@@ -16,7 +16,12 @@
 import re
 import redis
 
-import sqlite3
+try:
+    import sqlite3
+except ImportError:
+    have_sqlite = False
+else:
+    have_sqlite = True
 from ldap3 import Connection
 from ldap3.core.exceptions import LDAPSocketOpenError
 
@@ -42,6 +47,8 @@ def __init__(self, config, logger):
             redis_port = int(config.get('redis_port', 6379))
             self.caching = self.get_redis_connection(redis_host, redis_port)
         elif self.cache_engine == 'sqlite':
+            if not have_sqlite:
+                raise RuntimeError('No sqlite available: stackoverflow.com/q/44058239')
             self.caching = LocalSqlite(config.get('ldap_cache_file', '/var/tmp/ldap.cache'), logger)
 
     def get_redis_connection(self, redis_host, redis_port):


Test patch:
diff --git a/tools/c7n_azure/tests/test_logic_app.py b/tools/c7n_azure/tests/test_logic_app.py
--- a/tools/c7n_azure/tests/test_logic_app.py
+++ b/tools/c7n_azure/tests/test_logic_app.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import mock
+import re
 
 from azure_common import BaseTest, arm_template
 from jsonschema.exceptions import ValidationError
@@ -113,8 +114,8 @@ def test_get_callback(self, request_mock):
         regex = r'https://.*/workflows/.*/triggers/manual/paths/invoke' \
                 + r'\?api-version=.*triggers%2Fmanual%2Frun'
 
-        self.assertRegex(req1['url'], regex)
-        self.assertRegex(req2['url'], regex)
+        self.assertTrue(re.search(regex, req1['url']))
+        self.assertTrue(re.search(regex, req2['url']))
 
     def _get_manager(self):
         """The tests don't require real resource data,
diff --git a/tools/c7n_azure/tests/test_sqldatabase.py b/tools/c7n_azure/tests/test_sqldatabase.py
--- a/tools/c7n_azure/tests/test_sqldatabase.py
+++ b/tools/c7n_azure/tests/test_sqldatabase.py
@@ -313,8 +313,10 @@ def setUpClass(cls, *args, **kwargs):
 
     def tearDown(self, *args, **kwargs):
         super(ShortTermBackupRetentionPolicyActionTest, self).tearDown(*args, **kwargs)
+        args = list(self.retention_policy_context)
+        args.append(14)
         reverted_policy = ShortTermBackupRetentionPolicyActionTest.client.create_or_update(
-            *self.retention_policy_context, 14).result()
+            *args).result()
         self.assertEqual(reverted_policy.retention_days, 14)
 
     @arm_template('sqlserver.json')
@@ -389,10 +391,11 @@ def tearDown(self, *args, **kwargs):
         }
 
         super(LongTermBackupRetentionPolicyActionTest, self).tearDown(*args, **kwargs)
+
+        params = list(self.retention_policy_context)
+        params.append(default_long_term_policy)
         reverted_policy = LongTermBackupRetentionPolicyActionTest.client.create_or_update(
-            *self.retention_policy_context,
-            default_long_term_policy
-        ).result()
+            *params).result()
 
         self.assertEqual(reverted_policy.weekly_retention, 'P1M')
         self.assertEqual(reverted_policy.monthly_retention, 'P12M')
diff --git a/tools/c7n_mailer/tests/test_ldap.py b/tools/c7n_mailer/tests/test_ldap.py
--- a/tools/c7n_mailer/tests/test_ldap.py
+++ b/tools/c7n_mailer/tests/test_ldap.py
@@ -14,18 +14,48 @@
 
 import unittest
 
+import pytest
+import sys
+
 from common import get_ldap_lookup, PETER, BILL
+from c7n_mailer.ldap_lookup import have_sqlite
+
+
+def azure_pipelines_broken():
+    """Azure pipelines has multiple major versions of python that are broken.
+
+    The fixes for this will take a month since the regression was
+    first noted since they have to rebuild thier images and roll them
+    out which apparently takes a while (which makes me wonder what
+    they do with major CVES), and the work arounds are deleting their
+    python installations, downloading a new installation, and using
+    that.. wtf.
+
+    Note. This is the third major regression in pipelines in less than
+    30 days that has broken ci for days. So i choose not to mince words.
+
+    As an alternative while we look for alternatives, this checks for tests
+    known failing with broken python installations and skips them.\
+    """
+    return (sys.version_info.major, sys.version_info.minor) in ((2, 7), (3, 7))
+
+
+SKIP_REASON = "Azure Pipelines still broken"
 
 
 class MailerLdapTest(unittest.TestCase):
 
     def setUp(self):
+        if not have_sqlite:
+            return
         self.ldap_lookup = get_ldap_lookup(cache_engine='sqlite')
 
+    @pytest.mark.skipif(azure_pipelines_broken(), reason=SKIP_REASON)
     def test_sqlite_cached_get_mail(self):
         michael_bolton = self.ldap_lookup.caching.get('michael_bolton')
         self.assertEqual(michael_bolton.get('mail'), 'michael_bolton@initech.com')
 
+    @pytest.mark.skipif(azure_pipelines_broken(), reason=SKIP_REASON)
     def test_regex_requiring_underscore(self):
         self.ldap_lookup.uid_regex = '_'
         michael_bolton = self.ldap_lookup.get_metadata_from_uid('michael_bolton')
@@ -35,6 +65,7 @@ def test_regex_requiring_underscore(self):
         # since '123456' doesn't have an underscore, it should return {}
         self.assertEqual(milton, {})
 
+    @pytest.mark.skipif(azure_pipelines_broken(), reason=SKIP_REASON)
     def test_sqlite_cache_set_escaping(self):
         irish_guy = {
             'dn': 'uid=john_oconnor,cn=users,dc=initech,dc=com',
@@ -48,6 +79,7 @@ def test_sqlite_cache_set_escaping(self):
         get_result = self.ldap_lookup.caching.get(irish_guy['uid'])
         self.assertEqual(get_result, irish_guy)
 
+    @pytest.mark.skipif(azure_pipelines_broken(), reason=SKIP_REASON)
     def test_regex_requiring_6chars_and_only_digits(self):
         # now we'll do some tests requiring the uid to be 6 characters only and digits
         self.ldap_lookup.uid_regex = '^[0-9]{6}$'
@@ -55,14 +87,17 @@ def test_regex_requiring_6chars_and_only_digits(self):
         milton_email = milton.get('mail')
         self.assertEqual(milton_email, 'milton@initech.com')
 
+    @pytest.mark.skipif(azure_pipelines_broken(), reason=SKIP_REASON)
     def test_sqlite_cached_get_email_to_addr_without_manager(self):
         to_addr = self.ldap_lookup.get_email_to_addrs_from_uid('michael_bolton')
         self.assertEqual(to_addr, ['michael_bolton@initech.com'])
 
+    @pytest.mark.skipif(azure_pipelines_broken(), reason=SKIP_REASON)
     def test_sqlite_cached_get_email_to_addrs_with_manager(self):
         to_addr = self.ldap_lookup.get_email_to_addrs_from_uid('michael_bolton', manager=True)
         self.assertEqual(to_addr, ['michael_bolton@initech.com', 'milton@initech.com'])
 
+    @pytest.mark.skipif(azure_pipelines_broken(), reason=SKIP_REASON)
     def test_uid_ldap_lookup(self):
         ldap_result = self.ldap_lookup.get_metadata_from_uid('peter')
         self.assertEqual(ldap_result['mail'], PETER[1]['mail'][0])
@@ -72,14 +107,17 @@ def test_uid_ldap_lookup(self):
         self.assertEqual(cached_result['mail'], PETER[1]['mail'][0])
         self.assertEqual(cached_result['uid'], PETER[1]['uid'][0])
 
+    @pytest.mark.skipif(azure_pipelines_broken(), reason=SKIP_REASON)
     def test_dn_ldap_lookup(self):
         bill_metadata = self.ldap_lookup.get_metadata_from_dn(BILL[0])
         self.assertEqual(bill_metadata['mail'], BILL[1]['mail'][0])
 
+    @pytest.mark.skipif(azure_pipelines_broken(), reason=SKIP_REASON)
     def test_to_addr_with_ldap_query(self):
         to_addr = self.ldap_lookup.get_email_to_addrs_from_uid('peter', manager=True)
         self.assertEqual(to_addr, ['peter@initech.com', 'bill_lumberg@initech.com'])
 
+    @pytest.mark.skipif(azure_pipelines_broken(), reason=SKIP_REASON)
     def test_that_dn_and_uid_write_to_cache_on_manager_lookup(self):
         bill_metadata = self.ldap_lookup.get_metadata_from_dn(BILL[0])
         bill_metadata_dn_lookup_cache = self.ldap_lookup.caching.get(BILL[0])
@@ -87,6 +125,7 @@ def test_that_dn_and_uid_write_to_cache_on_manager_lookup(self):
         bill_metadata_uid_lookup_cache = self.ldap_lookup.caching.get(BILL[1]['uid'][0])
         self.assertEqual(bill_metadata, bill_metadata_uid_lookup_cache)
 
+    @pytest.mark.skipif(azure_pipelines_broken(), reason=SKIP_REASON)
     def test_that_dn_and_uid_write_to_cache_on_employee_lookup(self):
         peter_uid = PETER[1]['uid'][0]
         peter_metadata = self.ldap_lookup.get_metadata_from_uid(peter_uid)
@@ -95,6 +134,7 @@ def test_that_dn_and_uid_write_to_cache_on_employee_lookup(self):
         self.assertEqual(peter_metadata, peter_metadata_uid_lookup_cache)
         self.assertEqual(peter_metadata, peter_metadata_dn_lookup_cache)
 
+    @pytest.mark.skipif(azure_pipelines_broken(), reason=SKIP_REASON)
     def test_random_string_dont_hit_ldap_twice_uid_lookup(self):
         # if we query ldap and get no result, we should never query ldap again
         # for that result, we should query the cache and just return {}