Repository: jazzband/django-simple-history

Issue:
Add support for as_of_related on HistoryManager
As implemented in this fork: https://bitbucket.org/joaofrancese/django-simple-history/src/0d1630812d2e/simple_history/manager.py?at=default



Discussion:
Any news on this? Will it be incorporated?

Hi @jlachowski. As you can see, this issue was opened a very long time ago, and both @treyhunner and I have no need for it. There has been some discussion in #118 for a similar feature, and I can't think of any good technical reasons it couldn't be added.

Unfortunately I don't have the time to take point on adding this feature, and I don't have a need for it, so I wouldn't be able to exercise it in a real project. If you, or someone else in the #118 discussion has interest, I would be happy to help with getting the feature into django-simple-history.


Resolving patch:
diff --git a/simple_history/manager.py b/simple_history/manager.py
--- a/simple_history/manager.py
+++ b/simple_history/manager.py
@@ -8,6 +8,10 @@
     get_change_reason_from_object,
 )
 
+# when converting a historical record to an instance, this attribute is added
+# to the instance so that code can reverse the instance to its historical record
+SIMPLE_HISTORY_REVERSE_ATTR_NAME = "_history"
+
 
 class HistoricalQuerySet(QuerySet):
     """
@@ -22,6 +26,7 @@ class HistoricalQuerySet(QuerySet):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._as_instances = False
+        self._as_of = None
         self._pk_attr = self.model.instance_type._meta.pk.attname
 
     def as_instances(self):
@@ -89,6 +94,7 @@ def latest_of_each(self):
     def _clone(self):
         c = super()._clone()
         c._as_instances = self._as_instances
+        c._as_of = self._as_of
         c._pk_attr = self._pk_attr
         return c
 
@@ -108,6 +114,9 @@ def _instanceize(self):
             and isinstance(self._result_cache[0], self.model)
         ):
             self._result_cache = [item.instance for item in self._result_cache]
+            for item in self._result_cache:
+                historic = getattr(item, SIMPLE_HISTORY_REVERSE_ATTR_NAME)
+                setattr(historic, "_as_of", self._as_of)
 
 
 class HistoryDescriptor:
@@ -195,6 +204,8 @@ def as_of(self, date):
         """
         queryset = self.get_queryset().filter(history_date__lte=date)
         if not self.instance:
+            if isinstance(queryset, HistoricalQuerySet):
+                queryset._as_of = date
             queryset = queryset.latest_of_each().as_instances()
             return queryset
 
@@ -209,7 +220,10 @@ def as_of(self, date):
             raise self.instance.DoesNotExist(
                 "%s had already been deleted." % self.instance._meta.object_name
             )
-        return history_obj.instance
+        result = history_obj.instance
+        historic = getattr(result, SIMPLE_HISTORY_REVERSE_ATTR_NAME)
+        setattr(historic, "_as_of", date)
+        return result
 
     def bulk_history_create(
         self,
diff --git a/simple_history/models.py b/simple_history/models.py
--- a/simple_history/models.py
+++ b/simple_history/models.py
@@ -11,17 +11,25 @@
 from django.db import models
 from django.db.models import ManyToManyField
 from django.db.models.fields.proxy import OrderWrt
+from django.db.models.fields.related import ForeignKey
+from django.db.models.fields.related_descriptors import (
+    ForwardManyToOneDescriptor,
+    ReverseManyToOneDescriptor,
+    create_reverse_many_to_one_manager,
+)
+from django.db.models.query import QuerySet
 from django.forms.models import model_to_dict
 from django.urls import reverse
 from django.utils import timezone
 from django.utils.encoding import smart_str
+from django.utils.functional import cached_property
 from django.utils.text import format_lazy
 from django.utils.translation import gettext_lazy as _
 
 from simple_history import utils
 
 from . import exceptions
-from .manager import HistoryDescriptor
+from .manager import SIMPLE_HISTORY_REVERSE_ATTR_NAME, HistoryDescriptor
 from .signals import post_create_historical_record, pre_create_historical_record
 from .utils import get_change_reason_from_object
 
@@ -422,7 +430,10 @@ def get_instance(self):
                     pass
                 else:
                     attrs.update(values)
-            return model(**attrs)
+            result = model(**attrs)
+            # this is the only way external code could know an instance is historical
+            setattr(result, SIMPLE_HISTORY_REVERSE_ATTR_NAME, self)
+            return result
 
         def get_next_record(self):
             """
@@ -632,6 +643,108 @@ def transform_field(field):
         field.serialize = True
 
 
+class HistoricForwardManyToOneDescriptor(ForwardManyToOneDescriptor):
+    """
+    Overrides get_queryset to provide historic query support, should the
+    instance be historic (and therefore was generated by a timepoint query)
+    and the other side of the relation also uses a history manager.
+    """
+
+    def get_queryset(self, **hints) -> QuerySet:
+        instance = hints.get("instance")
+        if instance:
+            history = getattr(instance, SIMPLE_HISTORY_REVERSE_ATTR_NAME, None)
+            histmgr = getattr(
+                self.field.remote_field.model,
+                getattr(
+                    self.field.remote_field.model._meta,
+                    "simple_history_manager_attribute",
+                    "_notthere",
+                ),
+                None,
+            )
+            if history and histmgr:
+                return histmgr.as_of(history._as_of)
+        return super().get_queryset(**hints)
+
+
+class HistoricReverseManyToOneDescriptor(ReverseManyToOneDescriptor):
+    """
+    Overrides get_queryset to provide historic query support, should the
+    instance be historic (and therefore was generated by a timepoint query)
+    and the other side of the relation also uses a history manager.
+    """
+
+    @cached_property
+    def related_manager_cls(self):
+        related_model = self.rel.related_model
+
+        class HistoricRelationModelManager(related_model._default_manager.__class__):
+            def get_queryset(self):
+                try:
+                    return self.instance._prefetched_objects_cache[
+                        self.field.remote_field.get_cache_name()
+                    ]
+                except (AttributeError, KeyError):
+                    history = getattr(
+                        self.instance, SIMPLE_HISTORY_REVERSE_ATTR_NAME, None
+                    )
+                    histmgr = getattr(
+                        self.model,
+                        getattr(
+                            self.model._meta,
+                            "simple_history_manager_attribute",
+                            "_notthere",
+                        ),
+                        None,
+                    )
+                    if history and histmgr:
+                        queryset = histmgr.as_of(history._as_of)
+                    else:
+                        queryset = super().get_queryset()
+                    return self._apply_rel_filters(queryset)
+
+        return create_reverse_many_to_one_manager(
+            HistoricRelationModelManager, self.rel
+        )
+
+
+class HistoricForeignKey(ForeignKey):
+    """
+    Allows foreign keys to work properly from a historic instance.
+
+    If you use as_of queries to extract historical instances from
+    a model, and you have other models that are related by foreign
+    key and also historic, changing them to a HistoricForeignKey
+    field type will allow you to naturally cross the relationship
+    boundary at the same point in time as the origin instance.
+
+    A historic instance maintains an attribute ("_historic") when
+    it is historic, holding the historic record instance and the
+    timepoint used to query it ("_as_of").  HistoricForeignKey
+    looks for this and uses an as_of query against the related
+    object so the relationship is assessed at the same timepoint.
+    """
+
+    forward_related_accessor_class = HistoricForwardManyToOneDescriptor
+    related_accessor_class = HistoricReverseManyToOneDescriptor
+
+
+def is_historic(instance):
+    """
+    Returns True if the instance was acquired with an as_of timepoint.
+    """
+    return to_historic(instance) is not None
+
+
+def to_historic(instance):
+    """
+    Returns a historic model instance if the instance was acquired with
+    an as_of timepoint, or None.
+    """
+    return getattr(instance, SIMPLE_HISTORY_REVERSE_ATTR_NAME, None)
+
+
 class HistoricalObjectDescriptor:
     def __init__(self, model, fields_included):
         self.model = model


Test patch:
diff --git a/simple_history/tests/models.py b/simple_history/tests/models.py
--- a/simple_history/tests/models.py
+++ b/simple_history/tests/models.py
@@ -4,10 +4,12 @@
 from django.apps import apps
 from django.conf import settings
 from django.db import models
+from django.db.models.deletion import CASCADE
+from django.db.models.fields.related import ForeignKey
 from django.urls import reverse
 
 from simple_history import register
-from simple_history.models import HistoricalRecords
+from simple_history.models import HistoricalRecords, HistoricForeignKey
 
 from .custom_user.models import CustomUser as User
 from .external.models import AbstractExternal, AbstractExternal2, AbstractExternal3
@@ -788,3 +790,67 @@ class ModelWithMultipleNoDBIndex(models.Model):
         "Library", on_delete=models.CASCADE, null=True, related_name="+"
     )
     history = HistoricalRecords(no_db_index=["name", "fk", "other"])
+
+
+class TestOrganization(models.Model):
+    name = models.CharField(max_length=15, unique=True)
+
+
+class TestOrganizationWithHistory(models.Model):
+    name = models.CharField(max_length=15, unique=True)
+    history = HistoricalRecords()
+
+
+class TestParticipantToHistoricOrganization(models.Model):
+    """
+    Non-historic table foreign key to historic table.
+
+    In this case it should simply behave like ForeignKey because
+    the origin model (this one) cannot be historic, so foreign key
+    lookups are always "current".
+    """
+
+    name = models.CharField(max_length=15, unique=True)
+    organization = HistoricForeignKey(
+        TestOrganizationWithHistory, on_delete=CASCADE, related_name="participants"
+    )
+
+
+class TestHistoricParticipantToOrganization(models.Model):
+    """
+    Historic table foreign key to non-historic table.
+
+    In this case it should simply behave like ForeignKey because
+    the origin model (this one) can be historic but the target model
+    is not, so foreign key lookups are always "current".
+    """
+
+    name = models.CharField(max_length=15, unique=True)
+    organization = HistoricForeignKey(
+        TestOrganization, on_delete=CASCADE, related_name="participants"
+    )
+    history = HistoricalRecords()
+
+
+class TestHistoricParticipanToHistoricOrganization(models.Model):
+    """
+    Historic table foreign key to historic table.
+
+    In this case as_of queries on the origin model (this one)
+    or on the target model (the other one) will traverse the
+    foreign key relationship honoring the timepoint of the
+    original query.  This only happens when both tables involved
+    are historic.
+
+    NOTE: related_name has to be different than the one used in
+          TestParticipantToHistoricOrganization as they are
+          sharing the same target table.
+    """
+
+    name = models.CharField(max_length=15, unique=True)
+    organization = HistoricForeignKey(
+        TestOrganizationWithHistory,
+        on_delete=CASCADE,
+        related_name="historic_participants",
+    )
+    history = HistoricalRecords()
diff --git a/simple_history/tests/tests/test_manager.py b/simple_history/tests/tests/test_manager.py
--- a/simple_history/tests/tests/test_manager.py
+++ b/simple_history/tests/tests/test_manager.py
@@ -5,6 +5,8 @@
 from django.db import IntegrityError
 from django.test import TestCase, override_settings, skipUnlessDBFeature
 
+from simple_history.manager import SIMPLE_HISTORY_REVERSE_ATTR_NAME
+
 from ..models import Document, Poll, RankedDocument
 
 User = get_user_model()
@@ -69,12 +71,23 @@ def test_modified(self):
 
 class AsOfAdditionalTestCase(TestCase):
     def test_create_and_delete(self):
-        now = datetime.now()
         document = Document.objects.create()
+        now = datetime.now()
         document.delete()
-        for doc_change in Document.history.all():
-            doc_change.history_date = now
-            doc_change.save()
+
+        docs_as_of_now = Document.history.as_of(now)
+        doc = docs_as_of_now[0]
+        # as_of queries inject a property allowing callers
+        # to go from instance to historical instance
+        historic = getattr(doc, SIMPLE_HISTORY_REVERSE_ATTR_NAME)
+        self.assertIsNotNone(historic)
+        # as_of queries inject the time point of the original
+        # query into the historic record so callers can do magical
+        # things like chase historic foreign key relationships
+        # by patching forward and reverse one-to-one relationship
+        # processing (see issue 880)
+        self.assertEqual(historic._as_of, now)
+
         docs_as_of_tmw = Document.history.as_of(now + timedelta(days=1))
         with self.assertNumQueries(1):
             self.assertFalse(list(docs_as_of_tmw))
@@ -122,6 +135,12 @@ def test_as_of(self):
         ids = {item["id"] for item in queryset.values("id")}
         self.assertEqual(ids, {document1.id, document2.id})
 
+        # these records are historic
+        record = queryset[0]
+        historic = getattr(record, SIMPLE_HISTORY_REVERSE_ATTR_NAME)
+        self.assertIsInstance(historic, RankedDocument.history.model)
+        self.assertEqual(historic._as_of, t1)
+
         # at t2 we have one record left
         queryset = RankedDocument.history.as_of(t2)
         self.assertEqual(queryset.count(), 1)
diff --git a/simple_history/tests/tests/test_models.py b/simple_history/tests/tests/test_models.py
--- a/simple_history/tests/tests/test_models.py
+++ b/simple_history/tests/tests/test_models.py
@@ -13,10 +13,17 @@
 from django.db.models.fields.proxy import OrderWrt
 from django.test import TestCase, override_settings
 from django.urls import reverse
+from django.utils import timezone
 
 from simple_history import register
 from simple_history.exceptions import RelatedNameConflictError
-from simple_history.models import HistoricalRecords, ModelChange
+from simple_history.models import (
+    SIMPLE_HISTORY_REVERSE_ATTR_NAME,
+    HistoricalRecords,
+    ModelChange,
+    is_historic,
+    to_historic,
+)
 from simple_history.signals import pre_create_historical_record
 from simple_history.tests.custom_user.models import CustomUser
 from simple_history.tests.tests.utils import (
@@ -93,6 +100,11 @@
     State,
     Street,
     Temperature,
+    TestHistoricParticipanToHistoricOrganization,
+    TestHistoricParticipantToOrganization,
+    TestOrganization,
+    TestOrganizationWithHistory,
+    TestParticipantToHistoricOrganization,
     UnicodeVerboseName,
     UnicodeVerboseNamePlural,
     UserTextFieldChangeReasonModel,
@@ -372,6 +384,16 @@ def test_inheritance(self):
             },
         )
 
+    def test_reverse_historical(self):
+        """Tests how we can go from instance to historical record."""
+        document = Document.objects.create()
+        historic = document.history.all()[0]
+        instance = historic.instance
+        self.assertEqual(
+            getattr(instance, SIMPLE_HISTORY_REVERSE_ATTR_NAME).history_date,
+            historic.history_date,
+        )
+
     def test_specify_history_user(self):
         user1 = User.objects.create_user("user1", "1@example.com")
         user2 = User.objects.create_user("user2", "1@example.com")
@@ -1856,3 +1878,137 @@ def test_unique_field_index(self):
         # keeps index
         self.assertTrue(self.model._meta.get_field("name_keeps_index").db_index)
         self.assertTrue(self.history_model._meta.get_field("name_keeps_index").db_index)
+
+
+class HistoricForeignKeyTest(TestCase):
+    """
+    Tests chasing foreign keys across time points naturally with
+    HistoricForeignKey.
+    """
+
+    def test_non_historic_to_historic(self):
+        """
+        Non-historic table foreign key to historic table.
+
+        In this case it should simply behave like ForeignKey because
+        the origin model (this one) cannot be historic, so foreign key
+        lookups are always "current".
+        """
+        org = TestOrganizationWithHistory.objects.create(name="original")
+        part = TestParticipantToHistoricOrganization.objects.create(
+            name="part", organization=org
+        )
+        before_mod = timezone.now()
+        self.assertEqual(part.organization.id, org.id)
+        self.assertEqual(org.participants.count(), 1)
+        self.assertEqual(org.participants.all()[0], part)
+
+        historg = TestOrganizationWithHistory.history.as_of(before_mod).get(
+            name="original"
+        )
+        self.assertEqual(historg.participants.count(), 1)
+        self.assertEqual(historg.participants.all()[0], part)
+
+        self.assertEqual(org.history.count(), 1)
+        org.name = "modified"
+        org.save()
+        self.assertEqual(org.history.count(), 2)
+
+        # drop internal caches, re-select
+        part = TestParticipantToHistoricOrganization.objects.get(name="part")
+        self.assertEqual(part.organization.name, "modified")
+
+    def test_historic_to_non_historic(self):
+        """
+        Historic table foreign key to non-historic table.
+
+        In this case it should simply behave like ForeignKey because
+        the origin model (this one) can be historic but the target model
+        is not, so foreign key lookups are always "current".
+        """
+        org = TestOrganization.objects.create(name="org")
+        part = TestHistoricParticipantToOrganization.objects.create(
+            name="original", organization=org
+        )
+        self.assertEqual(part.organization.id, org.id)
+        self.assertEqual(org.participants.count(), 1)
+        self.assertEqual(org.participants.all()[0], part)
+
+        histpart = TestHistoricParticipantToOrganization.objects.get(name="original")
+        self.assertEqual(histpart.organization.id, org.id)
+
+    def test_historic_to_historic(self):
+        """
+        Historic table foreign key to historic table.
+
+        In this case as_of queries on the origin model (this one)
+        or on the target model (the other one) will traverse the
+        foreign key relationship honoring the timepoint of the
+        original query.  This only happens when both tables involved
+        are historic.
+
+        At t1 we have one org, one participant.
+        At t2 we have one org, two participants, however the org's name has changed.
+        At t3 we have one org, and one participant has left.
+        """
+        org = TestOrganizationWithHistory.objects.create(name="original")
+        p1 = TestHistoricParticipanToHistoricOrganization.objects.create(
+            name="p1", organization=org
+        )
+        t1_one_participant = timezone.now()
+        p2 = TestHistoricParticipanToHistoricOrganization.objects.create(
+            name="p2", organization=org
+        )
+        org.name = "modified"
+        org.save()
+        t2_two_participants = timezone.now()
+        p1.delete()
+        t3_one_participant = timezone.now()
+
+        # forward relationships - see how natural chasing timepoint relations is
+        p1t1 = TestHistoricParticipanToHistoricOrganization.history.as_of(
+            t1_one_participant
+        ).get(name="p1")
+        self.assertEqual(p1t1.organization, org)
+        self.assertEqual(p1t1.organization.name, "original")
+        p1t2 = TestHistoricParticipanToHistoricOrganization.history.as_of(
+            t2_two_participants
+        ).get(name="p1")
+        self.assertEqual(p1t2.organization, org)
+        self.assertEqual(p1t2.organization.name, "modified")
+        p2t2 = TestHistoricParticipanToHistoricOrganization.history.as_of(
+            t2_two_participants
+        ).get(name="p2")
+        self.assertEqual(p2t2.organization, org)
+        self.assertEqual(p2t2.organization.name, "modified")
+        p2t3 = TestHistoricParticipanToHistoricOrganization.history.as_of(
+            t3_one_participant
+        ).get(name="p2")
+        self.assertEqual(p2t3.organization, org)
+        self.assertEqual(p2t3.organization.name, "modified")
+
+        # reverse relationships
+        # at t1
+        ot1 = TestOrganizationWithHistory.history.as_of(t1_one_participant).all()[0]
+        self.assertEqual(ot1.historic_participants.count(), 1)
+        self.assertEqual(ot1.historic_participants.all()[0].name, p1.name)
+        # at t2
+        ot2 = TestOrganizationWithHistory.history.as_of(t2_two_participants).all()[0]
+        self.assertEqual(ot2.historic_participants.count(), 2)
+        self.assertIn(p1.name, [item.name for item in ot2.historic_participants.all()])
+        self.assertIn(p2.name, [item.name for item in ot2.historic_participants.all()])
+        # at t3
+        ot3 = TestOrganizationWithHistory.history.as_of(t3_one_participant).all()[0]
+        self.assertEqual(ot3.historic_participants.count(), 1)
+        self.assertEqual(ot3.historic_participants.all()[0].name, p2.name)
+        # current
+        self.assertEqual(org.historic_participants.count(), 1)
+        self.assertEqual(org.historic_participants.all()[0].name, p2.name)
+
+        self.assertTrue(is_historic(ot1))
+        self.assertFalse(is_historic(org))
+
+        self.assertIsInstance(
+            to_historic(ot1), TestOrganizationWithHistory.history.model
+        )
+        self.assertIsNone(to_historic(org))