Repository: pypest/pyemu

Issue:
Pf.build_pst fails when using longnames=False plus dataframe of pilot-points
From the discussions section,

When using a dataframe of predefined pilot-points with longnames=False in PstFrom, the template files are created fine but the build will either (1) fail completely if only pilot-points are in the setup, or (2) continue if the setup has parameters in addition to pilot-points but it won't swap the long names for short names in the pilot-point template files. 

From the error that is reported when the build fails completely (i.e., when you only have pilot-points) it looks like no entry is being made in the full_par_dat dataframe for the tpl_filename associated with the pilot-points when they are added via a dataframe. I guess this explains why the build continues without failing if parameters other than the pilot-points are included. 

You should be able to recreate the error by simply using longnames=False, adding pilot-points from a dataframe and then building Pst.

Hope that helps.

Discussion:

Patch:
diff --git a/pyemu/utils/pst_from.py b/pyemu/utils/pst_from.py
index d113cebd..fda95db4 100644
--- a/pyemu/utils/pst_from.py
+++ b/pyemu/utils/pst_from.py
@@ -186,6 +186,8 @@ class PstFrom(object):
             On windows, beware setting this much smaller than 50 because of the overhead associated with
             spawning the pool.  This value is added to the call to `apply_list_and_array_pars`. Default is 50
         echo (`bool`): flag to echo logger messages to the screen.  Default is True
+        pp_solve_num_threads (`int`): number of threads to use for the pyemu very-slow kriging solve for
+            pilot-point type parameters.  Default is 10.
 
     Note:
         This is the way...
@@ -213,6 +215,7 @@ class PstFrom(object):
         tpl_subfolder=None,
         chunk_len=50,
         echo=True,
+        pp_solve_num_threads=10
     ):
         self.original_d = Path(original_d)
         self.new_d = Path(new_d)
@@ -231,6 +234,7 @@ class PstFrom(object):
             start_datetime = _get_datetime_from_str(start_datetime)
         self.start_datetime = start_datetime
         self.geostruct = None
+        self.pp_solve_num_threads = int(pp_solve_num_threads)
         self.par_struct_dict = {}
         # self.par_struct_dict_l = {}
 
@@ -2461,6 +2465,7 @@ class PstFrom(object):
                                 "'zone' col not found in pp dataframe, adding generic zone"
                             )
                             pp_locs.loc[:, "zone"] = 1
+
                         elif zone_array is not None:
                             # check that all the zones in the pp df are in the zone array
                             missing = []
@@ -2586,6 +2591,8 @@ class PstFrom(object):
                         tpl_filename,
                         pnb,
                     )
+                    df["tpl_filename"] = tpl_filename
+                    df["pp_filename"] = pp_filename
                 df.loc[:, "pargp"] = pargp
                 df.set_index("parnme", drop=False, inplace=True)
                 # df includes most of the par info for par_dfs and also for
@@ -2654,7 +2661,7 @@ class PstFrom(object):
                             spatial_reference,
                             var_filename=var_filename,
                             zone_array=zone_array,
-                            num_threads=10,
+                            num_threads=self.pp_solve_num_threads,
                         )
                         ok_pp.to_grid_factors_file(fac_filename)
                     else:
@@ -2675,7 +2682,7 @@ class PstFrom(object):
                                 ok_pp.calc_factors(
                                     node_df.x,
                                     node_df.y,
-                                    num_threads=1,
+                                    num_threads=self.pp_solve_num_threads,
                                     pt_zone=zone,
                                     idx_vals=node_df.node.astype(int),
                                 )
@@ -2687,7 +2694,7 @@ class PstFrom(object):
                             for node, (x, y) in spatial_reference.items():
                                 data.append([node, x, y])
                             node_df = pd.DataFrame(data, columns=["node", "x", "y"])
-                            ok_pp.calc_factors(node_df.x, node_df.y, num_threads=10)
+                            ok_pp.calc_factors(node_df.x, node_df.y, num_threads=self.pp_solve_num_threads)
                             ok_pp.to_grid_factors_file(
                                 fac_filename, ncol=node_df.shape[0]
                             )


Test patch:
diff --git a/autotest/pst_from_tests.py b/autotest/pst_from_tests.py
index 93c7827b..acb4dd0d 100644
--- a/autotest/pst_from_tests.py
+++ b/autotest/pst_from_tests.py
@@ -4451,7 +4451,8 @@ def shortname_conversion_test(tmp_path):
                      remove_existing=True,
                      longnames=False,
                      zero_based=False,
-                     spatial_reference=sr)
+                     spatial_reference=sr,
+                     pp_solve_num_threads=1)
 
         v = pyemu.geostats.ExpVario(contribution=1.0, a=1000)
         gr_gs = pyemu.geostats.GeoStruct(variograms=v)
@@ -4467,6 +4468,19 @@ def shortname_conversion_test(tmp_path):
                           par_type="constant", par_name_base="cpar",
                           pargp="cpargp", upper_bound=0.1, lower_bound=10.0,
                           geostruct=gr_gs)
+        pf.add_parameters(filenames=parfiles,
+                          par_type="pilotpoints", par_name_base="pppar",
+                          pargp="pppargp", upper_bound=0.1, lower_bound=10.0,
+                          geostruct=gr_gs)
+
+        pp_locs = pyemu.pp_utils.setup_pilotpoints_grid(sr=sr,prefix_dict={0:"pps_1"})
+        pp_locs = pp_locs.loc[:,["name","x","y","zone","parval1"]]
+        pf.add_parameters(filenames=parfiles,
+                          par_type="pilotpoints", par_name_base="pppar2",
+                          pargp="pppargp2", upper_bound=0.1, lower_bound=10.0,
+                          geostruct=gr_gs,pp_space=pp_locs)
+
+
 
         pf.add_observations(
             "obsfile1",
@@ -4517,7 +4531,7 @@ def shortname_conversion_test(tmp_path):
             with open(os.path.join(pf.new_d, tpl), "rt") as f:
                 parin = set(rex.findall(f.read()))
             par = par - parin
-        assert len(par) == 0, f"{len(par)} pars not found in tplfiles: {par[:100]}..."
+        assert len(par) == 0, f"{len(par)} pars not found in tplfiles: {par}..."
         # test update/rebuild
         pf.add_observations(
             "obsfile3",
@@ -4555,6 +4569,7 @@ def shortname_conversion_test(tmp_path):
     except Exception as e:
         os.chdir(bd)
         raise Exception(str(e))
+
     os.chdir(bd)
 
 
@@ -5399,6 +5414,7 @@ def test_array_fmt_pst_from(tmp_path):
 
 
 if __name__ == "__main__":
+    shortname_conversion_test('.')
     #mf6_freyberg_pp_locs_test()
     # invest()
     #freyberg_test(os.path.abspath("."))
@@ -5427,7 +5443,7 @@ if __name__ == "__main__":
     # tpf.test_add_list_parameters()
     # # pstfrom_profile()
     # mf6_freyberg_arr_obs_and_headerless_test()
-    usg_freyberg_test(".")
+    #usg_freyberg_test(".")
     #vertex_grid_test()
     #direct_quickfull_test()
     #list_float_int_index_test()