Repository: skypyproject/skypy
Pull request #267 (2020-09-08 14:17:14)

config files can have nested function calls
## Description

This PR makes it possible for config files to have functions with arguments that contain functions themselves.

This requires a different syntax `$variable` or `$table.column` for references.

```yaml
mystring: 'hello world'
list_range_len_mystring: !list
  - !range
    - !len
      - $mystring
```

## Checklist
- [x] Follow the [Contributor Guidelines](https://github.com/skypyproject/skypy/blob/master/CONTRIBUTING.md)
- [x] Write unit tests
- [x] Write documentation strings
- [x] Assign someone from your working team to review this pull request
- [x] Assign someone from the infrastructure team to review this pull request


Patch:
diff --git a/examples/abundance_matching.yml b/examples/abundance_matching.yml
index 9ed0355..117d94a 100644
--- a/examples/abundance_matching.yml
+++ b/examples/abundance_matching.yml
@@ -3,14 +3,14 @@ A_s: 2.1982e-09
 n_s: 0.969453
 redshift: 1
 growth_function: !skypy.power_spectrum.growth_function
-  redshift: !redshift
-  cosmology: !cosmology
+  redshift: $redshift
+  cosmology: $cosmology
 wavenumber: !numpy.logspace [-4, 2, 100]
 power_spectrum: !skypy.power_spectrum.eisenstein_hu
-  wavenumber: !wavenumber
-  A_s: !A_s
-  n_s: !n_s
-  cosmology: !cosmology
+  wavenumber: $wavenumber
+  A_s: $A_s
+  n_s: $n_s
+  cosmology: $cosmology
 tables:
   halos:
     mass: !skypy.halo.mass.press_schechter
@@ -18,19 +18,19 @@ tables:
       m_max: 1.0E+12
       resolution: 1000
       size: 1000
-      wavenumber: !wavenumber
-      power_spectrum: !power_spectrum
-      growth_function: !growth_function
-      cosmology: !cosmology
+      wavenumber: $wavenumber
+      power_spectrum: $power_spectrum
+      growth_function: $growth_function
+      cosmology: $cosmology
   galaxies:
     luminosity: !skypy.galaxy.luminosity.herbel_luminosities
       alpha: -0.5
       a_m: -0.610
       b_m: -20.416
       size: 100
-      redshift: !redshift
+      redshift: $redshift
   abundance_matching:
     .init: !skypy.halo.abundance_matching.vale_ostriker
       .depends: [halos.complete, galaxies.complete]
-      halos: !halos
-      galaxies: !galaxies
+      halos: $halos
+      galaxies: $galaxies
diff --git a/examples/gravitational_wave_rates.yml b/examples/gravitational_wave_rates.yml
index 49c92ae..4346f5f 100644
--- a/examples/gravitational_wave_rates.yml
+++ b/examples/gravitational_wave_rates.yml
@@ -8,21 +8,21 @@ tables:
       a_m: -0.610
       b_m: -20.416
       size: 1000
-      cosmology: !cosmology
+      cosmology: $cosmology
     luminosity: !skypy.galaxy.luminosity.herbel_luminosities
       alpha: -0.5
       a_m: -0.610
       b_m: -20.416
-      redshift: !merger_rates.redshift
+      redshift: $merger_rates.redshift
     ns_ns_rate: !skypy.gravitational_wave.b_band_merger_rate
       population: NS-NS
       optimism: low
-      luminosity: !merger_rates.luminosity
+      luminosity: $merger_rates.luminosity
     ns_bh_rate: !skypy.gravitational_wave.b_band_merger_rate
       population: NS-BH
       optimism: low
-      luminosity: !merger_rates.luminosity
+      luminosity: $merger_rates.luminosity
     bh_bh_rate: !skypy.gravitational_wave.b_band_merger_rate
       population: BH-BH
       optimism: low
-      luminosity: !merger_rates.luminosity
+      luminosity: $merger_rates.luminosity
diff --git a/examples/mccl_galaxies.yml b/examples/mccl_galaxies.yml
index 1b4979d..ae274f4 100644
--- a/examples/mccl_galaxies.yml
+++ b/examples/mccl_galaxies.yml
@@ -12,30 +12,30 @@ tables:
       a_m: -0.610
       b_m: -20.416
       size: 141
-      cosmology: !cosmology
+      cosmology: $cosmology
     luminosity: !skypy.galaxy.luminosity.herbel_luminosities
       alpha: -0.5
       a_m: -0.610
       b_m: -20.416
-      redshift: !red_galaxies.redshift
+      redshift: $red_galaxies.redshift
     magnitude: !skypy.utils.astronomy.absolute_magnitude_from_luminosity
-      luminosity: !red_galaxies.luminosity
+      luminosity: $red_galaxies.luminosity
       zeropoint: -4.73
     spectral_coefficients: !skypy.galaxy.spectrum.dirichlet_coefficients
       alpha0: [2.461, 2.358, 2.568, 2.268, 2.402]
       alpha1: [2.410, 2.340, 2.200, 2.540, 2.464]
-      redshift: !red_galaxies.redshift
+      redshift: $red_galaxies.redshift
     physical_size: !skypy.galaxy.size.early_type_lognormal
-      magnitude: !red_galaxies.magnitude
+      magnitude: $red_galaxies.magnitude
       a: 0.60
       b: -4.63
       M0: -20.52
       sigma1: 0.48
       sigma2: 0.25
     angular_size: !skypy.galaxy.size.angular_size
-      physical_size: !red_galaxies.physical_size
-      redshift: !red_galaxies.redshift
-      cosmology: !cosmology
+      physical_size: $red_galaxies.physical_size
+      redshift: $red_galaxies.redshift
+      cosmology: $cosmology
     ellipticity: !skypy.galaxy.ellipticity.beta_ellipticity
       e_ratio: 0.45
       e_sum: 3.5
@@ -48,21 +48,21 @@ tables:
       a_m: -0.417
       b_m: -20.591
       size: 63
-      cosmology: !cosmology
+      cosmology: $cosmology
     luminosity: !skypy.galaxy.luminosity.herbel_luminosities
       alpha: -1.3
       a_m: -0.417
       b_m: -20.591
-      redshift: !blue_galaxies.redshift
+      redshift: $blue_galaxies.redshift
     magnitude: !skypy.utils.astronomy.absolute_magnitude_from_luminosity
-      luminosity: !blue_galaxies.luminosity
+      luminosity: $blue_galaxies.luminosity
       zeropoint: -4.73
     spectral_coefficients: !skypy.galaxy.spectrum.dirichlet_coefficients
       alpha0: [2.079, 3.524, 1.917, 1.992, 2.536]
       alpha1: [2.265, 3.862, 1.921, 1.685, 2.480]
-      redshift: !blue_galaxies.redshift
+      redshift: $blue_galaxies.redshift
     physical_size: !skypy.galaxy.size.late_type_lognormal
-      magnitude: !blue_galaxies.magnitude
+      magnitude: $blue_galaxies.magnitude
       alpha: 0.21
       beta: 0.53
       gamma: -1.31
@@ -70,9 +70,9 @@ tables:
       sigma1: 0.48
       sigma2: 0.25
     angular_size: !skypy.galaxy.size.angular_size
-      physical_size: !blue_galaxies.physical_size
-      redshift: !blue_galaxies.redshift
-      cosmology: !cosmology
+      physical_size: $blue_galaxies.physical_size
+      redshift: $blue_galaxies.redshift
+      cosmology: $cosmology
     ellipticity: !skypy.galaxy.ellipticity.beta_ellipticity
       e_ratio: 0.45
       e_sum: 3.5
diff --git a/skypy/pipeline/driver.py b/skypy/pipeline/driver.py
index c408202..82c79e0 100644
--- a/skypy/pipeline/driver.py
+++ b/skypy/pipeline/driver.py
@@ -141,9 +141,14 @@ class SkyPyDriver:
         tuples specify function calls `(function name, function args)`
         '''
 
-        # check for plain value
+        # check if not function
         if not isinstance(value, tuple):
-            return value
+            # check for reference
+            if isinstance(value, str) and value[0] == '$':
+                return self[value[1:]]
+            else:
+                # plain value
+                return value
 
         # value is tuple (function name, [function args])
         name = value[0]
@@ -179,13 +184,10 @@ class SkyPyDriver:
     def get_args(self, args):
         '''parse function arguments
 
-        tuples specify references to container: `(field name,)`
+        strings beginning with `$` are references to other fields
         '''
 
-        if isinstance(args, tuple):
-            # get reference
-            return self[args[0]]
-        elif isinstance(args, dict):
+        if isinstance(args, dict):
             # recurse kwargs
             return {k: self.get_args(v) for k, v in args.items()}
         elif isinstance(args, list):
@@ -193,17 +195,20 @@ class SkyPyDriver:
             return [self.get_args(a) for a in args]
         else:
             # return value
-            return args
+            return self.get_value(args)
 
     def get_deps(self, args):
         '''get dependencies from function args
 
-        returns a list of all dependencies found
+        returns a list of all references found
         '''
 
-        if isinstance(args, tuple):
+        if isinstance(args, str) and args[0] == '$':
             # reference
-            return [args[0]]
+            return [args[1:]]
+        elif isinstance(args, tuple):
+            # recurse on function arguments
+            return self.get_deps(args[1]) if len(args) > 1 else []
         elif isinstance(args, dict):
             # get explicit dependencies
             deps = args.pop('.depends', [])


Test patch:
diff --git a/skypy/pipeline/tests/data/test_config.yml b/skypy/pipeline/tests/data/test_config.yml
index 115802c..db7bef8 100644
--- a/skypy/pipeline/tests/data/test_config.yml
+++ b/skypy/pipeline/tests/data/test_config.yml
@@ -11,10 +11,10 @@ tables:
     test_column_1: !numpy.random.uniform
       size: 10
     test_column_2: !numpy.random.uniform
-      low: !test_table_1.test_column_1
+      low: $test_table_1.test_column_1
     test_column_3: !numpy.random.uniform
-      - !test_table_1.test_column_1  # <- low
-      - !test_table_1.test_column_2  # <- high
+      - $test_table_1.test_column_1  # <- low
+      - $test_table_1.test_column_2  # <- high
   test_table_2:
     .init: !astropy.table.QTable
       .depends: test_table_1.complete
diff --git a/skypy/pipeline/tests/test_driver.py b/skypy/pipeline/tests/test_driver.py
index 395adbc..64649eb 100644
--- a/skypy/pipeline/tests/test_driver.py
+++ b/skypy/pipeline/tests/test_driver.py
@@ -26,7 +26,7 @@ def test_driver():
                   'column1': ('numpy.random.uniform', {
                       'size': size}),
                   'column2': ('numpy.random.uniform', {
-                      'low': ('test_table.column1',)}),
+                      'low': '$test_table.column1'}),
                   'column3': ('list', [string])}}}
 
     driver = SkyPyDriver()
@@ -63,13 +63,13 @@ def test_driver():
 
     # Check for failure if 'column1' requires itself creating a cyclic
     # dependency graph
-    config['tables']['test_table']['column1'] = ('list', ('test_table.column1',))
+    config['tables']['test_table']['column1'] = ('list', '$test_table.column1')
     with pytest.raises(networkx.NetworkXUnfeasible):
         SkyPyDriver().execute(config)
 
     # Check for failure if 'column1' and 'column2' both require each other
     # creating a cyclic dependency graph
-    config['tables']['test_table']['column1'] = ('list', ('test_table.column2',))
+    config['tables']['test_table']['column1'] = ('list', '$test_table.column2')
     with pytest.raises(networkx.NetworkXUnfeasible):
         SkyPyDriver().execute(config)
 
@@ -100,14 +100,16 @@ def test_driver():
 
     # Check variables intialised by function
     config = {'test_func': ('list', 'hello world'),
-              'len_of_test_func': ('len', ('test_func',)),
+              'len_of_test_func': ('len', '$test_func'),
               'nested_references': ('sum', [
-                [('test_func',), [' '], ('test_func',)], []])}
+                ['$test_func', [' '], '$test_func'], []]),
+              'nested_functions': ('list', ('range', ('len', '$test_func')))}
     driver = SkyPyDriver()
     driver.execute(config)
     assert driver.test_func == list('hello world')
     assert driver.len_of_test_func == len('hello world')
     assert driver.nested_references == list('hello world hello world')
+    assert driver.nested_functions == list(range(len('hello world')))
 
 
 def teardown_module(module):