Repository: DOV-Vlaanderen/pydov

Issue:
Custom fields not populated if source fields not explicitly requested
<!-- You can ask questions about the DOV webservices or about the `pydov` package. If you have a question about the `pydov` Python package, please use following template. -->

* PyDOV version: master
* Python version: 3.10
* Operating System: Ubuntu

### Description

The custom fields `pkey_boring` and `pkey_sondering` in the interpretaties types are not populated if their source fields `Type_proef` and `Proeffiche` are not explicitly requested in the query too.

### What I Did

```python
from pydov.search.interpretaties import InformeleStratigrafieSearch

s = InformeleStratigrafieSearch()
df = s.search(max_features=1, return_fields=[
              'pkey_interpretatie', 'pkey_boring', 'pkey_sondering'])

print(df.iloc[0].transpose())
```
```
[000/001] .
pkey_interpretatie    https://www.dov.vlaanderen.be/data/interpretat...
pkey_boring                                                         NaN
pkey_sondering                                                      NaN
Name: 0, dtype: object
```

We expect either `pkey_boring` or `pkey_sondering` to be filled in.

Discussion:

Patch:
diff --git a/pydov/search/abstract.py b/pydov/search/abstract.py
index 486c649..1e5868e 100644
--- a/pydov/search/abstract.py
+++ b/pydov/search/abstract.py
@@ -768,7 +768,12 @@ class AbstractSearch(AbstractCommon):
                                        for i in self._map_df_wfs_source
                                        if i in return_fields])
 
+        extra_custom_fields = set()
+        for custom_field in self._type.get_fields(source=('custom',)).values():
+            extra_custom_fields.update(custom_field.requires_fields())
+
         wfs_property_names.extend(extra_wfs_fields)
+        wfs_property_names.extend(list(extra_custom_fields))
         wfs_property_names = list(set(wfs_property_names))
 
         if sort_by is not None:
diff --git a/pydov/types/abstract.py b/pydov/types/abstract.py
index 0a13cb6..593e82b 100644
--- a/pydov/types/abstract.py
+++ b/pydov/types/abstract.py
@@ -373,12 +373,6 @@ class AbstractDovType(AbstractTypeCommon):
             An instance of this class populated with the data from the WFS
             element.
 
-        Raises
-        ------
-        NotImplementedError
-            This is an abstract method that should be implemented in a
-            subclass.
-
         """
         instance = cls(feature.findtext(
             './{{{}}}{}'.format(namespace, cls.pkey_fieldname)))
@@ -391,6 +385,9 @@ class AbstractDovType(AbstractTypeCommon):
                 returntype=field.get('type', None)
             )
 
+        for field in cls.get_fields(source=('custom',)).values():
+            instance.data[field['name']] = field.calculate(instance) or np.nan
+
         return instance
 
     @classmethod
diff --git a/pydov/types/fields.py b/pydov/types/fields.py
index b803635..1dd39c0 100644
--- a/pydov/types/fields.py
+++ b/pydov/types/fields.py
@@ -125,8 +125,9 @@ class XmlField(AbstractField):
             self.__setitem__('xsd_type', xsd_type.typename)
 
 
-class _CustomField(AbstractField):
-    """Class for a custom field, created explicitly in pydov."""
+class _CustomWfsField(AbstractField):
+    """Class for a custom field, created explicitly in pydov from other WFS
+    fields."""
 
     def __init__(self, name, datatype, definition='', notnull=False):
         """Initialise a custom field.
@@ -144,6 +145,43 @@ class _CustomField(AbstractField):
             True if this field is always present (mandatory), False otherwise.
 
         """
-        super(_CustomField, self).__init__(name, 'custom', datatype)
+        super(_CustomWfsField, self).__init__(name, 'custom', datatype)
         self.__setitem__('definition', definition)
         self.__setitem__('notnull', notnull)
+
+    def requires_fields(self):
+        """Get a list of WFS fields that are required by (the calculation of)
+        this custom field.
+
+        Returns
+        -------
+        list of str
+            List of WFS fieldnames that is required by this custom field.
+
+        Raises
+        ------
+        NotImplementedError
+            Implement this in a subclass.
+        """
+        raise NotImplementedError
+
+    def calculate(self, instance):
+        """Calculate the value of this custom field for the given instance.
+
+        Parameters
+        ----------
+        instance : AbstractDovType
+            Instance of the corresponding type, containing all WFS values in
+            its data dictionary.
+
+        Returns
+        -------
+        Value to be used for this custom field for this instance. Its datatype
+        should match the one set in the initialisation of the custom field.
+
+        Raises
+        ------
+        NotImplementedError
+            Implement this in a subclass.
+        """
+        raise NotImplementedError
diff --git a/pydov/types/interpretaties.py b/pydov/types/interpretaties.py
index 68a47e0..8b0217d 100644
--- a/pydov/types/interpretaties.py
+++ b/pydov/types/interpretaties.py
@@ -1,13 +1,54 @@
 # -*- coding: utf-8 -*-
 """Module containing the DOV data types for interpretations, including
 subtypes."""
-import numpy as np
 
 from pydov.types.abstract import AbstractDovSubType, AbstractDovType
-from pydov.types.fields import WfsField, XmlField, XsdType, _CustomField
+from pydov.types.fields import WfsField, XmlField, XsdType, _CustomWfsField
 from pydov.util.dovutil import build_dov_url
 
 
+class PkeyBoringField(_CustomWfsField):
+    """Custom field to populate pkey_boring in case the interpretation is
+    linked with a Boring.
+    """
+
+    def __init__(self):
+        super().__init__(name='pkey_boring',
+                         definition='URL die verwijst naar de gegevens van de '
+                         'boring waaraan deze interpretatie '
+                         'gekoppeld is (indien gekoppeld aan een '
+                         'boring).',
+                         datatype='string')
+
+    def requires_fields(self):
+        return ['Type_proef', 'Proeffiche']
+
+    def calculate(self, instance):
+        if instance.data['Type_proef'] == 'Boring':
+            return instance.data['Proeffiche']
+
+
+class PkeySonderingField(_CustomWfsField):
+    """Custom field to populate pkey_sondering in case the interpretation is
+    linked with a Sondering.
+    """
+
+    def __init__(self):
+        super().__init__(name='pkey_sondering',
+                         definition='URL die verwijst naar de gegevens van de '
+                         'sondering waaraan deze interpretatie '
+                         'gekoppeld is (indien gekoppeld '
+                         'aan een sondering).',
+                         datatype='string')
+
+    def requires_fields(self):
+        return ['Type_proef', 'Proeffiche']
+
+    def calculate(self, instance):
+        if instance.data['Type_proef'] == 'Sondering':
+            return instance.data['Proeffiche']
+
+
 class AbstractCommonInterpretatie(AbstractDovType):
     """Abstract base class for interpretations that can be linked to
     boreholes or cone penetration tests."""
@@ -15,18 +56,8 @@ class AbstractCommonInterpretatie(AbstractDovType):
     fields = [
         WfsField(name='pkey_interpretatie',
                  source_field='Interpretatiefiche', datatype='string'),
-        _CustomField(name='pkey_boring',
-                     definition='URL die verwijst naar de gegevens van de '
-                                'boring waaraan deze informele stratigrafie '
-                                'gekoppeld is (indien gekoppeld aan een '
-                                'boring).',
-                     datatype='string'),
-        _CustomField(name='pkey_sondering',
-                     definition='URL die verwijst naar de gegevens van de '
-                                'sondering waaraan deze informele '
-                                'stratigrafie gekoppeld is (indien gekoppeld '
-                                'aan een sondering).',
-                     datatype='string'),
+        PkeyBoringField(),
+        PkeySonderingField(),
         WfsField(name='betrouwbaarheid_interpretatie',
                  source_field='Betrouwbaarheid', datatype='string'),
         WfsField(name='x', source_field='X_mL72', datatype='float'),
@@ -50,52 +81,6 @@ class AbstractCommonInterpretatie(AbstractDovType):
         """
         super().__init__('interpretatie', pkey)
 
-    @classmethod
-    def from_wfs_element(cls, feature, namespace):
-        instance = cls(
-            feature.findtext('./{{{}}}{}'.format(
-                namespace, cls.pkey_fieldname)))
-
-        typeproef = cls._parse(
-            func=feature.findtext,
-            xpath='Type_proef',
-            namespace=namespace,
-            returntype='string'
-        )
-
-        if typeproef == 'Boring':
-            instance.data['pkey_boring'] = cls._parse(
-                func=feature.findtext,
-                xpath='Proeffiche',
-                namespace=namespace,
-                returntype='string'
-            )
-            instance.data['pkey_sondering'] = np.nan
-        elif typeproef == 'Sondering':
-            instance.data['pkey_sondering'] = cls._parse(
-                func=feature.findtext,
-                xpath='Proeffiche',
-                namespace=namespace,
-                returntype='string'
-            )
-            instance.data['pkey_boring'] = np.nan
-        else:
-            instance.data['pkey_boring'] = np.nan
-            instance.data['pkey_sondering'] = np.nan
-
-        for field in cls.get_fields(source=('wfs',)).values():
-            if field['name'] in ['pkey_boring', 'pkey_sondering']:
-                continue
-
-            instance.data[field['name']] = cls._parse(
-                func=feature.findtext,
-                xpath=field['sourcefield'],
-                namespace=namespace,
-                returntype=field.get('type', None)
-            )
-
-        return instance
-
 
 class AbstractBoringInterpretatie(AbstractDovType):
     """Abstract base class for interpretations that are linked to boreholes


Test patch:
diff --git a/tests/test_search_customfield.py b/tests/test_search_customfield.py
new file mode 100644
index 0000000..c79f695
--- /dev/null
+++ b/tests/test_search_customfield.py
@@ -0,0 +1,37 @@
+import pandas as pd
+import pytest
+from pydov.search.interpretaties import InformeleStratigrafieSearch
+
+from tests.abstract import ServiceCheck
+
+
+class TestSearchCustomWfsField(object):
+    @pytest.mark.online
+    @pytest.mark.skipif(not ServiceCheck.service_ok(),
+                        reason="DOV service is unreachable")
+    def test_search_resolve_customfield(self):
+        """Test the search method without explicitly requesting the
+            'Type_proef' and 'Proeffiche' fields.
+
+            Test whether the output dataframe includes values for pkey_boring or
+            pkey_sondering.
+
+            Parameters
+            ----------
+            mp_get_schema : pytest.fixture
+                Monkeypatch the call to a remote OWSLib schema.
+            mp_remote_describefeaturetype : pytest.fixture
+                Monkeypatch the call to a remote DescribeFeatureType .
+            mp_remote_wfs_feature : pytest.fixture
+                Monkeypatch the call to get WFS features.
+            mp_dov_xml : pytest.fixture
+                Monkeypatch the call to get the remote XML data.
+
+            """
+        df = InformeleStratigrafieSearch().search(
+            max_features=1,
+            return_fields=('pkey_interpretatie', 'pkey_boring',
+                           'pkey_sondering'))
+
+        assert (not pd.isnull(df.pkey_boring[0])) or (
+            not pd.isnull(df.pkey_sondering[0]))