Repository: mozilla/treeherder
Pull request #8563: use in memory cache in parallel with db cache, convert keys to be date specific
Author: login: jmaher
Created: 2025-03-11T20:56:20Z | Merged: 2025-03-27T15:14:14Z

I want to figure out how to test this.  Also some of this might be a little overkill.


Diff:
diff --git a/tests/etl/test_load_artifacts.py b/tests/etl/test_load_artifacts.py
index ade53bb082b..970b690c912 100644
--- a/tests/etl/test_load_artifacts.py
+++ b/tests/etl/test_load_artifacts.py
@@ -1,7 +1,12 @@
+import datetime
 import json
 
+import pytest
+from django.core.cache import caches
+from django.urls import reverse
+
 from treeherder.etl.artifact import store_job_artifacts
-from treeherder.model.error_summary import get_error_summary
+from treeherder.model.error_summary import MemDBCache, get_error_summary
 from treeherder.model.models import TextLogError
 
 
@@ -67,3 +72,225 @@ def test_load_non_ascii_textlog_errors(test_job):
     assert TextLogError.objects.count() == 2
     assert TextLogError.objects.get(line_number=1587).line == "07:51:28  WARNING - \U000000c3"
     assert TextLogError.objects.get(line_number=1588).line == "07:51:29  WARNING - <U+01D400>"
+
+
+def _add_job_summary(job, date=None, expected_new=False):
+    text_log_summary_artifact = {
+        "type": "json",
+        "name": "text_log_summary",
+        "blob": json.dumps(
+            {
+                "errors": [
+                    {
+                        "line": "TEST-UNEXPECTED-FAIL | mypath/mytest.js | unexpected value 3.14",
+                        "linenumber": 1587,
+                    },
+                ],
+            }
+        ),
+    }
+
+    # ensure a result='failed' to treat failure as a NEW_failure
+    job.result = "testfailed"
+    if date:
+        job.submit_time = datetime.datetime.strptime(date, "%Y-%m-%d")
+    job.save()
+
+    text_log_summary_artifact["job_guid"] = job.guid
+    store_job_artifacts([text_log_summary_artifact])
+
+    # ensure bug_suggestions data is stored and retrieved properly
+    tle_all = TextLogError.objects.all()
+    bug_suggestions = get_error_summary(job)
+    for suggestion in bug_suggestions:
+        tle = next(
+            t
+            for t in tle_all
+            if t.line_number == suggestion["line_number"]
+            and t.job.guid == text_log_summary_artifact["job_guid"]
+        )
+        assert suggestion["failure_new_in_rev"] == tle.new_failure
+        assert tle.new_failure is expected_new
+
+
+def _add_annotation(client, job, test_user):
+    client.force_authenticate(user=test_user)
+    resp = client.post(
+        reverse("note-list", kwargs={"project": job.repository.name}),
+        data={
+            "job_id": job.id,
+            "failure_classification_id": 2,
+            "who": test_user.email,
+            "text": "you look like a man-o-lantern",
+        },
+    )
+    content = json.loads(resp.content)
+    assert content["message"] == f"note stored for job {job.id}"
+
+
+@pytest.mark.django_db
+def test_new_failure_annotation(client, test_jobs, test_user):
+    db_cache = caches["db_cache"]
+    db_cache.clear()
+
+    # add first job, new failure, add second job, not new failure
+    _add_job_summary(test_jobs[0], expected_new=True)
+    _add_job_summary(test_jobs[1], expected_new=False)
+
+    # annotate job 1 as fixed_by_commit, this should remove references from new line cache
+    # NOTE: I am annotating both; this will reset the counter to allow NEW failures to show
+    # up in the future
+    _add_annotation(client, test_jobs[0], test_user)
+    _add_annotation(client, test_jobs[1], test_user)
+
+    # add a 3rd job, same error, but different job/push, verify not new
+    # ensure a result='failed' to treat failure as a NEW_failure
+    _add_job_summary(test_jobs[2], expected_new=True)
+
+
+@pytest.mark.django_db
+def test_new_failure_partial_annotation(client, test_jobs, test_user):
+    db_cache = caches["db_cache"]
+    db_cache.clear()
+
+    # add first job, new failure, add second job, not new failure
+    _add_job_summary(test_jobs[0], expected_new=True)
+    _add_job_summary(test_jobs[1], expected_new=False)
+
+    # annotate job 1 as fixed_by_commit, this should remove references from new line cache
+    # NOTE: only annotate 1, so future lines will not be NEW
+    #       if we change to a model where we clear the cache for a given annotation
+    #       this test case would be different
+    _add_annotation(client, test_jobs[0], test_user)
+
+    # add a 3rd job, same error, but different job/push, verify not new
+    # ensure a result='failed' to treat failure as a NEW_failure
+    _add_job_summary(test_jobs[2], expected_new=False)
+
+
+@pytest.mark.django_db
+def test_memcache_migration():
+    root = "th_test"
+
+    date = "2025-01-01"
+    date2 = "2025-02-01"
+    # add stuff to line_cache, write to memcache, verify not in db_cache
+    line_cache = {}
+    line_cache[date] = {
+        "new_lines": {"this is a trap": 31415926535},
+        "this is a trap": 1,
+    }
+
+    db_cache = caches["db_cache"]
+    db_cache.set(root, line_cache)
+
+    # ensure we can get the cache and it is a dict with key "date"
+    lcache = db_cache.get(root)
+    assert date in lcache
+
+    lcache = MemDBCache(root)
+
+    # this will migrate the old format to the new
+    lc = lcache.get_cache()
+    assert date in lc
+    assert lcache.get_cache_keys() == [f"{date}"]
+
+    # add an extra date
+    lc[date2] = lc[date]
+    lcache.write_cache(date2)
+    assert date2 in lc
+    assert lcache.get_cache_keys() == [date, date2]
+
+    # flush cache so we can reload
+    memcache = caches["default"]
+    memcache.delete(root)
+    assert memcache.get(root) is None
+
+    # reload cache
+    lc = lcache.get_cache()
+    assert date in lc
+    assert date2 in lc
+    assert lcache.get_cache_keys() == [date, date2]
+
+    # ensure we can wipe things out and it really clears stuff
+    db_cache.clear()
+    memcache.clear()
+    assert lcache.get_cache_keys() == []
+
+
+@pytest.mark.django_db
+def test_memcache_to_db_class():
+    root = "th_test"
+    lcache = MemDBCache(root)
+    line_cache = lcache.get_cache()
+
+    date = "2025-01-01"
+    # add stuff to line_cache, write to memcache, verify not in db_cache
+    line_cache[date] = {
+        "new_lines": {"this is a trap": 31415926535},
+        "this is a trap": 1,
+    }
+    assert caches["default"].get(f"{root}_{date}") is None
+
+    # write to memcache
+    lcache.update_cache(date, line_cache[date])
+    assert len(lcache.get_cache_keys()) == 0
+
+    # add key to db
+    lcache.add_cache_keys(date)
+    assert len(lcache.get_cache_keys()) == 1
+
+    db_data = caches["db_cache"].get(f"{root}_{date}")
+    assert db_data is None
+
+    # bulk write entire memcache -> db
+    lcache.write_cache()
+    db_data = caches["db_cache"].get(f"{root}_{date}")
+    assert db_data == line_cache[date]
+
+    date2 = "2025-01-02"
+    line_cache[date2] = {
+        "new_lines": {"this is a trap2": 31415926535},
+        "this is a trap2": 1,
+    }
+
+    # write only date2 to db
+    lcache.write_cache(date2, line_cache[date2])
+    assert len(lcache.get_cache_keys()) == 2
+    db_data = caches["db_cache"].get(f"{root}_{date2}")
+    assert db_data == line_cache[date2]
+
+    # expire cache
+    lcache.remove_cache_key(date)
+    assert len(lcache.get_cache_keys()) == 1
+
+    # ensure we only have date2
+    line_cache = lcache.get_cache()
+    assert len(line_cache.keys()) == 1
+
+    # remove it all
+    lcache.remove_cache_key(date2)
+    line_cache = lcache.get_cache()
+    assert len(line_cache.keys()) == 0
+
+
+@pytest.mark.django_db
+def test_memcache_new_date(client, test_jobs, test_user):
+    root = "mc_error_lines"
+
+    db_cache = caches["db_cache"]
+    db_cache.clear()
+
+    date = "2025-01-01"
+    line_cache = {}
+    line_cache[date] = {
+        "new_lines": {"this is a trap": 31415926535},
+        "this is a trap": 1,
+    }
+
+    db_cache = caches["db_cache"]
+    db_cache.set(root, line_cache)
+
+    # adding two jobs on different dates
+    _add_job_summary(test_jobs[0], date="2025-01-02", expected_new=True)
+    _add_job_summary(test_jobs[1], date="2025-01-03", expected_new=False)
diff --git a/treeherder/model/error_summary.py b/treeherder/model/error_summary.py
index 55fd243815e..2614b653648 100644
--- a/treeherder/model/error_summary.py
+++ b/treeherder/model/error_summary.py
@@ -28,6 +28,92 @@
 PREFIX_PATTERN = r"^(TEST-UNEXPECTED-\S+|PROCESS-CRASH)\s+\|\s+"
 
 
+class MemDBCache:
+    keyroot = ""
+
+    def __init__(self, keyroot):
+        self.keyroot = keyroot
+
+    def get_cache(self):
+        lcache = {}
+        keys = self.get_cache_keys()
+        # copy db cache to memory cache
+        if keys and not cache.get(f"{self.keyroot}_{keys[0]}"):
+            for k in keys:
+                self.update_cache(k, db_cache.get(f"{self.keyroot}_{k}"))
+
+        for d in [k.split("_")[-1] for k in keys]:
+            lcache[d] = cache.get(f"{self.keyroot}_{d}")
+        return lcache
+
+    def write_cache(self, key=None, value=None):
+        if key:
+            # new key, add to both cache and db_cache
+            self.update_cache(key, value)
+            self.update_db_cache(key, value)
+            self.add_cache_keys(key)
+        else:
+            # flush cache->db_cache; ensure that keys are removed/added
+            keys = self.get_cache_keys()
+            for key in keys:
+                self.update_db_cache(key, cache.get(f"{self.keyroot}_{key}"))
+            self.set_cache_keys(keys)
+
+    def update_cache(self, key, value, timeout=LINE_CACHE_TIMEOUT):
+        cache.set(f"{self.keyroot}_{key}", value, timeout)
+
+    def update_db_cache(self, key, value, timeout=LINE_CACHE_TIMEOUT):
+        db_cache.set(f"{self.keyroot}_{key}", value, timeout)
+
+    def get_cache_keys(self):
+        # this will return all keys (dates)
+        keys = db_cache.get(f"{self.keyroot}_keys")
+        if not keys:
+            # this is the first time we run this
+            # either fresh DB || migrating from old
+            if db_cache.get(f"{self.keyroot}"):
+                keys = self.migrate_from_old()
+            else:
+                keys = []
+            self.set_cache_keys(keys)
+        return keys
+
+    def add_cache_keys(self, key):
+        # this will return all keys (dates)
+        keys = self.get_cache_keys()
+        if not keys:
+            keys = []
+        keys.append(key)
+        self.set_cache_keys(keys)
+
+    def remove_cache_key(self, key):
+        # this will return all keys (dates)
+        keys = self.get_cache_keys()
+        keys.remove(key)
+        cache.delete(f"{self.keyroot}_{key}")
+        db_cache.delete(f"{self.keyroot}_{key}")
+        self.set_cache_keys(keys)
+
+    def set_cache_keys(self, keys):
+        db_cache.set(f"{self.keyroot}_keys", keys, LINE_CACHE_TIMEOUT)
+
+    def migrate_from_old(self):
+        keys = []
+        # get keyroot, data is {date: value, date2: value, ...}
+        data = db_cache.get(self.keyroot)
+        dates = [d for d in data if re.match(r"\d{4}-\d{2}-\d{2}", d)]
+        if len(dates) > 0:
+            # only clear db cache if we have data to work with
+            cache.clear()
+
+        for date in dates:
+            self.update_cache(f"{date}", data[date])
+            self.update_db_cache(f"{date}", data[date])
+            keys.append(date)
+
+        return keys
+
+
 def get_error_summary(job, queryset=None):
     """
     Create a list of bug suggestions for a job.
@@ -41,19 +127,22 @@ def get_error_summary(job, queryset=None):
         return cached_error_summary
 
     # add support for error line caching
-    line_cache_key = "mc_error_lines"
     if job.repository == "comm-central":
-        line_cache_key = "cc_error_lines"
-    line_cache = db_cache.get(line_cache_key)
-    if line_cache is None:
-        line_cache = {str(job.submit_time.date()): {}}
+        lcache = MemDBCache("cc_error_lines")
+    else:
+        lcache = MemDBCache("mc_error_lines")
+
+    date = str(job.submit_time.date())
+    line_cache = lcache.get_cache()
+    if date not in line_cache.keys():
+        lcache.write_cache(date, {})
     else:
-        dates = list(line_cache.keys())
+        dates = lcache.get_cache_keys()
         dates.sort()
         for d in dates:
             date_time = datetime.datetime.strptime(d, "%Y-%m-%d")
             if date_time <= (job.submit_time - datetime.timedelta(days=LINE_CACHE_TIMEOUT_DAYS)):
-                del line_cache[d]
+                lcache.remove_cache_key(d)
             else:
                 break
 
@@ -86,7 +175,9 @@ def get_error_summary(job, queryset=None):
         logger.error("error caching error_summary for job %s: %s", job.id, e, exc_info=True)
 
     try:
-        db_cache.set(line_cache_key, line_cache, LINE_CACHE_TIMEOUT)
+        lcache.update_cache(date, line_cache[date])
+        # TODO: consider reducing this, each date is ~5%, so it will be faster
+        lcache.update_db_cache(date, line_cache[date])
     except Exception as e:
         newrelic.agent.record_custom_event("error caching error_lines for job", job.id)
         logger.error("error caching error_lines for job %s: %s", job.id, e, exc_info=True)
diff --git a/treeherder/webapp/api/note.py b/treeherder/webapp/api/note.py
index fb2e1eca2de..5ef9a952ab8 100644
--- a/treeherder/webapp/api/note.py
+++ b/treeherder/webapp/api/note.py
@@ -1,6 +1,5 @@
 import logging
 
-from django.core.cache import caches
 from rest_framework import viewsets
 from rest_framework.decorators import action
 from rest_framework.exceptions import ParseError
@@ -8,7 +7,7 @@
 from rest_framework.status import HTTP_404_NOT_FOUND
 
 from treeherder.model.error_summary import (
-    LINE_CACHE_TIMEOUT,
+    MemDBCache,
     cache_clean_error_line,
     get_cleaned_line,
 )
@@ -79,9 +78,7 @@ def create(self, request, project):
         """
         current_job = Job.objects.get(repository__name=project, id=int(request.data["job_id"]))
         fc_id = int(request.data["failure_classification_id"])
-        revision = None
-        if "text" in request.data:
-            revision = request.data["text"]
+        revision = current_job.push.revision
         JobNote.objects.create(
             job=current_job,
             failure_classification_id=fc_id,
@@ -91,23 +88,36 @@ def create(self, request, project):
 
         if fc_id == 2:  # this is for fixed_by_commit (backout | follow_up_commit)
             # remove cached failure line counts
-            db_cache = caches["db_cache"]
-            line_cache_key = "error_lines"
-            line_cache = db_cache.get(line_cache_key)
+            if current_job.repository == "comm-central":
+                lcache = MemDBCache("cc_error_lines")
+            else:
+                lcache = MemDBCache("mc_error_lines")
+            line_cache = lcache.get_cache()
             date = current_job.submit_time.date().isoformat()
             if line_cache and date in line_cache.keys():
                 for err in TextLogError.objects.filter(job=current_job):
                     cache_clean_line = cache_clean_error_line(get_cleaned_line(err.line))
+
+                    # TODO: if annotating a FBC and we don't have new_failure, or failure
+                    # has already been annotated (cleaned), then we need to store new_failure
+                    # in the tle.  Here we need to take all known failures that match ccl
+                    # and annotate them as new_failure.  Theoretically these will
                     if cache_clean_line in line_cache[date].keys():
                         line_cache[date][cache_clean_line] -= 1
+                        if line_cache[date][cache_clean_line] <= 0:
+                            del line_cache[date][cache_clean_line]
+
                         if (
                             cache_clean_line in line_cache[date]["new_lines"].keys()
                             and revision
                             and line_cache[date]["new_lines"][cache_clean_line] == revision
                         ):
+                            # delete both the "new_lines" reference and the existing reference
+                            # this allows us to classify a future failure as new_failure!
                             del line_cache[date]["new_lines"][cache_clean_line]
                         try:
-                            db_cache.set(line_cache_key, line_cache, LINE_CACHE_TIMEOUT)
+                            lcache.update_cache(date, line_cache[date])
+                            lcache.update_db_cache(date, line_cache[date])
                         except Exception as e:
                             logger.error(
                                 "error caching error_lines for job %s: %s",


Reviews:
user: login: Archaeopteryx
body: Do you plan to migrate the old line cache?
state: CHANGES_REQUESTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T15:22:07Z
user: login: jmaher
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T19:00:52Z
user: login: Archaeopteryx
body: Approved but will push it to prototype instance before merging.
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-18T20:39:43Z
user: login: Archaeopteryx
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-20T21:13:31Z
user: login: Archaeopteryx
body: This has been pushed to prototype.
state: CHANGES_REQUESTED
author_association: COLLABORATOR
submitted_at: 2025-03-25T01:54:16Z
user: login: jmaher
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-25T14:19:37Z
user: login: Archaeopteryx
body: Pushed to prototype one minute ago.
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-25T20:02:12Z
user: login: jmaher
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-25T20:50:30Z

Review comments:
diff_hunk: @@ -28,6 +28,71 @@
 PREFIX_PATTERN = r"^(TEST-UNEXPECTED-\S+|PROCESS-CRASH)\s+\|\s+"
 
 
+class MemDBCache:
+    keyroot = ""
+
+    def __init__(self, keyroot):
+        self.keyroot = keyroot
+
+    def get_cache(self):
+        lcache = {}
+        keys = self.get_cache_keys()
+        # copy db cache to memory cache
+        if not keys or not cache.get(keys[0]):
path: treeherder/model/error_summary.py
user: login: Archaeopteryx
body: `not keys or` does nothing here. This should be `if keys and not cache.get(keys[0]):`
created_at: 2025-03-18T15:03:05Z
author_association: COLLABORATOR
diff_hunk: @@ -28,6 +28,71 @@
 PREFIX_PATTERN = r"^(TEST-UNEXPECTED-\S+|PROCESS-CRASH)\s+\|\s+"
 
 
+class MemDBCache:
+    keyroot = ""
+
+    def __init__(self, keyroot):
+        self.keyroot = keyroot
+
+    def get_cache(self):
+        lcache = {}
+        keys = self.get_cache_keys()
+        # copy db cache to memory cache
+        if not keys or not cache.get(keys[0]):
path: treeherder/model/error_summary.py
user: login: jmaher
body: thanks, this makes sense
created_at: 2025-03-18T19:00:52Z
author_association: COLLABORATOR
diff_hunk: @@ -72,6 +73,99 @@ def test_load_non_ascii_textlog_errors(test_job):
     assert TextLogError.objects.get(line_number=1588).line == "07:51:29  WARNING - <U+01D400>"
 
 
+def _add_job_summary(job, expected_new):
+    text_log_summary_artifact = {
+        "type": "json",
+        "name": "text_log_summary",
+        "blob": json.dumps(
+            {
+                "errors": [
+                    {
+                        "line": "TEST-UNEXPECTED-FAIL | mypath/mytest.js | unexpected value 3.14",
+                        "linenumber": 1587,
+                    },
+                ],
+            }
+        ),
+    }
+
+    # ensure a result='failed' to treat failure as a NEW_failure
+    job.result = "testfailed"
+    job.save()
+
+    text_log_summary_artifact["job_guid"] = job.guid
+    store_job_artifacts([text_log_summary_artifact])
+
+    # ensure bug_suggestions data is stored and retrieved properly
+    tle_all = TextLogError.objects.all()
+    bug_suggestions = get_error_summary(job)
+    for suggestion in bug_suggestions:
+        tle = next(t for t in tle_all if t.line_number == suggestion["line_number"])
path: tests/etl/test_load_artifacts.py
user: login: Archaeopteryx
body: `tle` in line 104 overwrites the one from line 103.
created_at: 2025-03-25T01:27:54Z
author_association: COLLABORATOR
diff_hunk: @@ -72,6 +73,99 @@ def test_load_non_ascii_textlog_errors(test_job):
     assert TextLogError.objects.get(line_number=1588).line == "07:51:29  WARNING - <U+01D400>"
 
 
+def _add_job_summary(job, expected_new):
+    text_log_summary_artifact = {
+        "type": "json",
+        "name": "text_log_summary",
+        "blob": json.dumps(
+            {
+                "errors": [
+                    {
+                        "line": "TEST-UNEXPECTED-FAIL | mypath/mytest.js | unexpected value 3.14",
+                        "linenumber": 1587,
+                    },
+                ],
+            }
+        ),
+    }
+
+    # ensure a result='failed' to treat failure as a NEW_failure
+    job.result = "testfailed"
+    job.save()
+
+    text_log_summary_artifact["job_guid"] = job.guid
+    store_job_artifacts([text_log_summary_artifact])
+
+    # ensure bug_suggestions data is stored and retrieved properly
+    tle_all = TextLogError.objects.all()
+    bug_suggestions = get_error_summary(job)
+    for suggestion in bug_suggestions:
+        tle = next(t for t in tle_all if t.line_number == suggestion["line_number"])
path: tests/etl/test_load_artifacts.py
user: login: jmaher
body: thanks!
created_at: 2025-03-25T14:19:37Z
author_association: COLLABORATOR
diff_hunk: @@ -73,7 +74,7 @@ def test_load_non_ascii_textlog_errors(test_job):
     assert TextLogError.objects.get(line_number=1588).line == "07:51:29  WARNING - <U+01D400>"
 
 
-def _add_job_summary(job, expected_new):
+def _add_job_summary(job, expected_new, date=None):
path: tests/etl/test_load_artifacts.py
user: login: Archaeopteryx
body: Second argument is output-ish while third one is input-ish. Please switch positions.
created_at: 2025-03-25T20:01:58Z
author_association: COLLABORATOR
diff_hunk: @@ -73,7 +74,7 @@ def test_load_non_ascii_textlog_errors(test_job):
     assert TextLogError.objects.get(line_number=1588).line == "07:51:29  WARNING - <U+01D400>"
 
 
-def _add_job_summary(job, expected_new):
+def _add_job_summary(job, expected_new, date=None):
path: tests/etl/test_load_artifacts.py
user: login: jmaher
body: easy fix, updated this last commit to address this!  thanks for the feedback and deploying to prototype.  
created_at: 2025-03-25T20:50:30Z
author_association: COLLABORATOR

PR comments:
user: login: jmaher
created_at: 2025-03-12T19:24:01Z
body: I have tests- ready for review.  
author_association: COLLABORATOR
user: login: jmaher
created_at: 2025-03-18T19:06:42Z
body: I have updated the code- when finding no `_keys`, we create an empty list and move forward.  Now when we get that we look for the old key, and then split existing data into keys; finally delete the old key format from the db.

this is simple and lightweight.  
author_association: COLLABORATOR
user: login: jmaher
created_at: 2025-03-18T19:07:40Z
body: hmm, somehow I got a bunch of additional files/changes mixed in with this;  no idea why
author_association: COLLABORATOR
user: login: jmaher
created_at: 2025-03-18T19:49:09Z
body: all ready for review!
author_association: COLLABORATOR
user: login: jmaher
created_at: 2025-03-21T23:42:45Z
body: with the latest commit to the stack a few things are solved:
1) db flush is properly handled
2) when annotating we actually decrement and allow for future failures to be new
3) for #2 there is a test
4) an issue with cache key names existed when using get_cache(), this is now consistent.
author_association: COLLABORATOR
user: login: Archaeopteryx
created_at: 2025-03-25T10:02:37Z
body: This broke bug suggestions on prototype:

```
{
  "textPayload": "Traceback (most recent call last):\n  File \"/usr/local/lib/python3.10/site-packages/celery/app/trace.py\", line 453, in trace_task\n    R = retval = fun(*args, **kwargs)\n  File \"/usr/local/lib/python3.10/site-packages/newrelic/hooks/application_celery.py\", line 123, in wrapper\n    return wrapped(*args, **kwargs)\n  File \"/usr/local/lib/python3.10/site-packages/celery/app/trace.py\", line 736, in __protected_call__\n    return self.run(*args, **kwargs)\n  File \"/app/treeherder/workers/task.py\", line 65, in inner\n    raise task_func.retry(exc=e, countdown=timeout, throw=False)\n  File \"/usr/local/lib/python3.10/site-packages/celery/app/task.py\", line 736, in retry\n    raise_with_context(exc)\n  File \"/app/treeherder/workers/task.py\", line 41, in inner\n    return f(*args, **kwargs)\n  File \"/app/treeherder/log_parser/tasks.py\", line 76, in parse_logs\n    raise first_exception\n  File \"/app/treeherder/log_parser/tasks.py\", line 58, in parse_logs\n    parser(job_log)\n  File \"/app/treeherder/log_parser/tasks.py\", line 112, in post_log_artifacts\n    store_job_artifacts(serialized_artifacts)\n  File \"/app/treeherder/etl/artifact.py\", line 95, in store_job_artifacts\n    store_text_log_summary_artifact(job, artifact)\n  File \"/app/treeherder/etl/artifact.py\", line 38, in store_text_log_summary_artifact\n    bugs = error_summary.get_error_summary(job, queryset=log_errors)\n  File \"/app/treeherder/model/error_summary.py\", line 161, in get_error_summary\n    summary, line_cache = bug_suggestions_line(\n  File \"/app/treeherder/model/error_summary.py\", line 220, in bug_suggestions_line\n    counter += line_cache[day].get(cache_clean_line, 0)\nAttributeError: 'NoneType' object has no attribute 'get'",
  "insertId": "w8081dikttev746m",
  "resource": {
    "type": "k8s_container",
    "labels": {
      "project_id": "moz-fx-treeherde-nonprod-34ec",
      "namespace_name": "prototype-treeherder",
      "location": "us-west1",
      "cluster_name": "treeherder-nonprod-v1",
      "container_name": "treeherder",
      "pod_name": "treeherder-log-parser-68dc55d96f-9bj2p"
    }
  },
  "timestamp": "2025-03-25T10:00:34.081113714Z",
  "severity": "ERROR",
  "labels": {
    "k8s-pod/fullname": "treeherder-log-parser",
    "k8s-pod/jenkins-build-id": "319",
    "k8s-pod/app_kubernetes_io/part-of": "treeherder",
    "k8s-pod/app_kubernetes_io/version": "1.0.0",
    "logging.gke.io/top_level_controller_type": "Deployment",
    "compute.googleapis.com/resource_name": "gke-treeherder-nonpro-default-pool-v4-88dddb70-7gzf",
    "k8s-pod/app_kubernetes_io/managed-by": "jenkins",
    "k8s-pod/app_kubernetes_io/instance": "prototype",
    "k8s-pod/pod-template-hash": "68dc55d96f",
    "k8s-pod/app_kubernetes_io/component": "app",
    "k8s-pod/app_kubernetes_io/name": "treeherder-log-parser",
    "logging.gke.io/top_level_controller_name": "treeherder-log-parser"
  },
  "logName": "projects/moz-fx-treeherde-nonprod-34ec/logs/stderr",
  "receiveTimestamp": "2025-03-25T10:00:34.898659602Z",
  "errorGroups": [
    {
      "id": "CPr0--i2kYWH8QE"
    }
  ]
}
```
author_association: COLLABORATOR
user: login: jmaher
created_at: 2025-03-25T14:21:32Z
body: my most recent commit does:
 * fixes the case where a migration created `mc_error_lines_mc_error_lines_<date>`
 * fixed issue with line_cache[date] == None on prototype (same as ^)
 * added test to reproduce failure on prototype and ensure this "migrated" state is reliable
author_association: COLLABORATOR
user: login: jmaher
created_at: 2025-03-25T20:56:38Z
body: I still see this error `AttributeError: 'NoneType' object has no attribute 'get'`, is it possible the code hasn't been deployed yet?  Maybe there are tests required or another service to restart?
author_association: COLLABORATOR
user: login: Archaeopteryx
created_at: 2025-03-25T21:48:24Z
body: Jenkins says the top of this patch stack got deployed shortly before 20:12 UTC. There have been issues with older pushes getting deployed in the past but this was fixed a few weeks ago.

The issue was some keys had duplicate root prefixes. I have deleted all caching data on prototype and the cache gets populated again. The spinner in the failure lines are persists because of an internal issue with `bug-job-map` from a migration out of sync with production. Lets keep it as it is to verify the cache code works correct, afterwards the prototype database can be reset to resolve the issue.
author_association: COLLABORATOR