Repository: rhayes777/PyAutoFit
Pull request #872 (2023-11-10 15:43:44)

feature/aggregator unification
More methods to make align the interfaces of the directory and database aggregators



Patch:
diff --git a/autofit/aggregator/aggregator.py b/autofit/aggregator/aggregator.py
index 82f2d37d3..987a57d34 100755
--- a/autofit/aggregator/aggregator.py
+++ b/autofit/aggregator/aggregator.py
@@ -194,6 +194,14 @@ class Aggregator:
 
         return cls(search_outputs, grid_search_outputs)
 
+    def add_directory(self, directory: Union[str, Path]):
+        """
+        Add a directory to the aggregator.
+        """
+        aggregator = Aggregator.from_directory(directory)
+        self.search_outputs.extend(aggregator.search_outputs)
+        self.grid_search_outputs.extend(aggregator.grid_search_outputs)
+
     def remove_unzipped(self):
         """
         Removes the unzipped output directory for each phase.
diff --git a/autofit/aggregator/file_output.py b/autofit/aggregator/file_output.py
index d2d2a63b1..145dee054 100644
--- a/autofit/aggregator/file_output.py
+++ b/autofit/aggregator/file_output.py
@@ -1,7 +1,7 @@
 import json
 from abc import abstractmethod, ABC
 from pathlib import Path
-from typing import Union
+from typing import Union, Optional, Dict
 
 import numpy as np
 
@@ -69,6 +69,12 @@ class JSONOutput(FileOutput):
         """
         return from_dict(self.dict)
 
+    def value_using_reference(self, reference: Optional[Dict] = None):
+        """
+        The object represented by the JSON
+        """
+        return from_dict(self.dict, reference=reference)
+
 
 class PickleOutput(FileOutput):
     @property
diff --git a/autofit/aggregator/search_output.py b/autofit/aggregator/search_output.py
index c18a17b4a..98e0ec092 100644
--- a/autofit/aggregator/search_output.py
+++ b/autofit/aggregator/search_output.py
@@ -4,11 +4,11 @@ import logging
 import pickle
 from os import path
 from pathlib import Path
-from typing import Generator, Tuple, Optional, List
+from typing import Generator, Tuple, Optional, List, cast
 
 import dill
-import numpy as np
 
+from autoconf import cached_property
 from autofit.non_linear.samples.pdf import SamplesPDF
 from autofit.aggregator.file_output import (
     JSONOutput,
@@ -18,6 +18,7 @@ from autofit.mapper.identifier import Identifier
 from autofit.non_linear.samples.sample import samples_from_iterator
 from autoconf.dictable import from_dict
 
+# noinspection PyProtectedMember
 original_create_file_handle = dill._dill._create_filehandle
 
 
@@ -41,8 +42,9 @@ dill._dill._create_filehandle = _create_file_handle
 
 
 class AbstractSearchOutput:
-    def __init__(self, directory: Path):
+    def __init__(self, directory: Path, reference: Optional[dict] = None):
         self.directory = directory
+        self._reference = reference
 
     @property
     def is_complete(self) -> bool:
@@ -76,21 +78,21 @@ class AbstractSearchOutput:
             outputs.append(FileOutput(name, file_path))
         return outputs
 
-    @property
-    def jsons(self):
+    @cached_property
+    def jsons(self) -> List[JSONOutput]:
         """
         The json files in the search output files directory
         """
-        return self._outputs(".json")
+        return cast(List[JSONOutput], self._outputs(".json"))
 
-    @property
+    @cached_property
     def arrays(self):
         """
         The csv files in the search output files directory
         """
         return self._outputs(".csv")
 
-    @property
+    @cached_property
     def pickles(self):
         """
         The pickle files in the search output files directory
@@ -105,38 +107,51 @@ class AbstractSearchOutput:
         return self._outputs(".fits")
 
     @property
-    def log_likelihood(self) -> float:
+    def max_log_likelihood(self) -> Optional[float]:
         """
         The log likelihood of the maximum log likelihood sample
         """
-        return self.samples.max_log_likelihood_sample.log_likelihood
+        try:
+            return self.samples.max_log_likelihood_sample.log_likelihood
+        except AttributeError:
+            return None
 
-    def __getattr__(self, item):
+    def __getattr__(self, name):
         """
         Attempt to load a pickle by the same name from the search output directory.
 
         dataset.pickle, meta_dataset.pickle etc.
         """
-        try:
-            with open(self.files_path / f"{item}.pickle", "rb") as f:
-                return pickle.load(f)
-        except FileNotFoundError:
-            pass
-        try:
-            with open(self.files_path / f"{item}.json") as f:
-                d = json.load(f)
-                if "type" in d:
-                    result = from_dict(d, reference=self._reference)
-                    if result is not None:
-                        return result
-                return d
-        except FileNotFoundError:
-            pass
-        try:
-            with open(self.files_path / f"{item}.csv") as f:
-                return np.loadtxt(f)
-        except (FileNotFoundError, ValueError):
-            pass
+        return self.value(name)
+
+    def value(self, name: str):
+        """
+        Load the value of some object in the files directory for the search.
+
+        This may be a pickle, json, csv or fits file.
+
+        If the JSON has a specified type it is parsed as that type. See dictable.py
+        in autoconf.
+
+        Returns None if the file does not exist.
+
+        Parameters
+        ----------
+        name
+            The name of the file to load without a file suffix.
+
+        Returns
+        -------
+        The loaded object
+        """
+        for item in self.jsons:
+            if item.name == name:
+                return item.value_using_reference(self._reference)
+        for item in self.pickles + self.arrays + self.hdus:
+            if item.name == name:
+                return item.value
+
+        return None
 
 
 class SearchOutput(AbstractSearchOutput):
@@ -144,6 +159,8 @@ class SearchOutput(AbstractSearchOutput):
     @DynamicAttrs
     """
 
+    is_grid_search = False
+
     def __init__(self, directory: Path, reference: dict = None):
         """
         Represents the output of a single search. Comprises a metadata file and other dataset files.
@@ -153,13 +170,12 @@ class SearchOutput(AbstractSearchOutput):
         directory
             The directory of the search
         """
-        super().__init__(directory)
+        super().__init__(directory, reference)
         self.__search = None
         self.__model = None
 
         self.directory = directory
 
-        self._reference = reference
         self.file_path = directory / "metadata"
 
         try:
@@ -174,14 +190,34 @@ class SearchOutput(AbstractSearchOutput):
 
     @property
     def instance(self):
+        """
+        The instance of the maximum log likelihood sample i.e. the instance
+        with the greatest likelihood.
+
+        None if samples cannot be loaded.
+        """
         try:
             return self.samples.max_log_likelihood()
         except (AttributeError, NotImplementedError):
             return None
 
     @property
-    def id(self):
-        return str(Identifier([self.search, self.model, self.search.unique_tag]))
+    def id(self) -> str:
+        """
+        The unique identifier of the search.
+
+        This is used as a directory name and as a database identifier.
+        """
+        return str(Identifier([self.search, self.model, self.unique_tag]))
+
+    @property
+    def model(self):
+        """
+        The model used by the search
+        """
+        if self.__model is None:
+            self.__model = self.value("model")
+        return self.__model
 
     @property
     def samples(self) -> SamplesPDF:
@@ -279,6 +315,24 @@ class SearchOutput(AbstractSearchOutput):
         """
         return [getattr(child, name) for child in self.child_analyses]
 
+    @property
+    def path_prefix(self):
+        return self.search.paths.path_prefix
+
+    @property
+    def name(self):
+        """
+        The name of the search
+        """
+        return self.search.name
+
+    @property
+    def unique_tag(self):
+        """
+        The unique tag of the search
+        """
+        return self.search.unique_tag
+
     def __str__(self):
         return self.text
 
@@ -287,6 +341,8 @@ class SearchOutput(AbstractSearchOutput):
 
 
 class GridSearchOutput(AbstractSearchOutput):
+    is_grid_search = True
+
     @property
     def unique_tag(self) -> str:
         """
@@ -296,7 +352,10 @@ class GridSearchOutput(AbstractSearchOutput):
             return f.read()
 
     @property
-    def id(self):
+    def id(self) -> str:
+        """
+        Use the unique tag of the grid search as an identifier.
+        """
         return self.unique_tag
 
 
@@ -304,7 +363,7 @@ class GridSearch:
     def __init__(
         self,
         grid_search_output: GridSearchOutput,
-        search_outputs: List[SearchOutput],
+        children: List[SearchOutput],
     ):
         """
         Represents the output of a grid search. Comprises overall information from the grid search
@@ -314,11 +373,18 @@ class GridSearch:
         ----------
         grid_search_output
             The output of the grid search
-        search_outputs
+        children
             The outputs of each individual search performed as part of the grid search
         """
         self.grid_search_output = grid_search_output
-        self.search_outputs = search_outputs
+        self.children = children
+
+    @property
+    def best_fit(self) -> SearchOutput:
+        """
+        The output for the search in the grid search that had the greatest log likelihood
+        """
+        return max(self.children, key=lambda x: x.instance.log_likelihood)
 
     def __getattr__(self, item):
         return getattr(self.grid_search_output, item)
diff --git a/autofit/database/aggregator/scrape.py b/autofit/database/aggregator/scrape.py
index fd009bac2..8edeee5a1 100644
--- a/autofit/database/aggregator/scrape.py
+++ b/autofit/database/aggregator/scrape.py
@@ -64,9 +64,9 @@ class Scraper:
         for item in self.aggregator:
             logger.info(
                 f"Creating fit for: "
-                f"{item.search.paths.path_prefix} "
-                f"{item.search.unique_tag} "
-                f"{item.search.name} "
+                f"{item.path_prefix} "
+                f"{item.unique_tag} "
+                f"{item.name} "
                 f"{item.id} "
             )
 
@@ -76,13 +76,13 @@ class Scraper:
             except sa.orm.exc.NoResultFound:
                 fit = m.Fit(
                     id=item.id,
-                    name=item.search.name,
-                    unique_tag=item.search.unique_tag,
+                    name=item.name,
+                    unique_tag=item.unique_tag,
                     model=item.model,
                     instance=item.instance,
                     is_complete=item.is_complete,
                     info=item.info,
-                    max_log_likelihood=item.log_likelihood,
+                    max_log_likelihood=item.max_log_likelihood,
                     parent_id=item.parent_identifier,
                 )
 
@@ -118,7 +118,7 @@ class Scraper:
 
             _add_files(grid_search, item)
 
-            for search in item.search_outputs:
+            for search in item.children:
                 fit = self._retrieve_model_fit(search)
                 grid_search.children.append(fit)
             yield grid_search


Test patch:
diff --git a/test_autofit/aggregator/test_scrape.py b/test_autofit/aggregator/test_scrape.py
index 178cd8510..bea3b7615 100644
--- a/test_autofit/aggregator/test_scrape.py
+++ b/test_autofit/aggregator/test_scrape.py
@@ -8,6 +8,8 @@ class MockFit:
     def __init__(self):
         self.jsons = {}
 
+    id = "id"
+
     def set_json(self, name, json):
         self.jsons[name] = json
 
diff --git a/test_autofit/non_linear/grid/test_paths/test_aggregator.py b/test_autofit/non_linear/grid/test_paths/test_aggregator.py
index e89ebd7be..91d7db6d3 100644
--- a/test_autofit/non_linear/grid/test_paths/test_aggregator.py
+++ b/test_autofit/non_linear/grid/test_paths/test_aggregator.py
@@ -20,8 +20,8 @@ def make_grid_search(aggregator):
 
 
 def test_correspondence(grid_search, aggregator):
-    assert len(grid_search.search_outputs) == 100
-    assert grid_search.search_outputs[0] is list(aggregator)[0]
+    assert len(grid_search.children) == 100
+    assert grid_search.children[0] is list(aggregator)[0]
 
 
 def test_attributes(grid_search):