Merge branch 'main' into name_equals

examples/uc/generic_gradient_uc_test_table1.slurm

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+#SBATCH -N 6
+#SBATCH -J 50scen_gradient
+#SBATCH -t 00:20:00
+#SBATCH -p pbatch
+#SBATCH --mail-type=ALL
+#SBATCH -A mpisppy
+
+export MPICH_ASYNC_PROGRESS=1
+source ${HOME}/venvs/mpisppy/bin/activate
+cd ${HOME}/mpi-sppy/examples/uc
+
+SOLVERNAME="gurobi_persistent"
+
+# IMPT: run limited number of iterations (2) to get the uc_cyl_nonants.npy xhat file"
+
+# the command line below executes UC with cost-based, gradient rho, and dynamic primal/dual-based updates
+srun -n 150 python -u -m mpi4py ../../mpisppy/generic_cylinders.py --module-name uc_funcs --bundles-per-rank=0 --max-iterations=100 --default-rho=1 --xhatshuffle --ph-ob --num-scens=50 --max-solver-threads=2 --mipgaps-json=phmipgaps.json --solver-name=${SOLVERNAME} --xhatpath uc_cyl_nonants_table1.npy --rel-gap 0.00001 --abs-gap=1 --intra-hub-conv-thresh=-1 --grad-rho-setter --grad-order-stat 0.5 --grad-dynamic-primal-crit --grad-dynamic-dual-crit
+
+#--grad-rho-setter --grad-order-stat 0.5 --grad-dynamic-primal-crit
+
+# other ...
+
+# uses uc_funcs rho-setter
+# --use-cost-based-rho
+
+# --grad-dynamic-primal-crit
+# --grad-dynamic-dual-crit
+# --grad-dynamic-primal-thresh
+# --grad-dynamic-dual-thresh
+
+####mpiexec --oversubscribe -np 1 python -m mpi4py gradient_uc_cylinders.py --bundles-per-rank=0 --max-iterations=20 --default-rho=1 --num-scens=5 --max-solver-threads=2 --lagrangian-iter0-mipgap=1e-7 --ph-mipgaps-json=phmipgaps.json --solver-name=${SOLVERNAME} --xhatpath uc_cyl_nonants.npy --rel-gap 0.000001 --display-progress --grad-rho-setter --grad-order-stat 0.5 --grad-display-rho
+
+# ? do we need ph_ob_rho_rescale_factors_json", ph_ob_gradient_rho", ??
+
+# --fwph
+echo
+
+# --rho-setter --order-stat 0.5
+# --display-progress

examples/uc/gradient_uc_test_table1.slurm

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+#SBATCH -N 6
+#SBATCH -J 50scen_gradient
+#SBATCH -t 00:20:00
+#SBATCH -p pbatch
+#SBATCH --mail-type=ALL
+#SBATCH -A mpisppy
+
+export MPICH_ASYNC_PROGRESS=1
+source ${HOME}/venvs/mpisppy/bin/activate
+cd ${HOME}/mpi-sppy/examples/uc
+
+SOLVERNAME="gurobi_persistent"
+
+# IMPT: run limited number of iterations (2) to get the uc_cyl_nonants.npy xhat file"
+
+echo "Now run the cylinders"
+# the command line below executes UC with cost-based, gradient rho, and dynamic primal/dual-based updates
+srun -n 150 python -u -m mpi4py gradient_uc_cylinders.py --bundles-per-rank=0 --max-iterations=100 --default-rho=1 --xhatshuffle --ph-ob --num-scens=50 --max-solver-threads=2 --ph-mipgaps-json=phmipgaps.json --solver-name=${SOLVERNAME} --xhatpath uc_cyl_nonants_table1.npy --rel-gap 0.00001 --abs-gap=1 --intra-hub-conv-thresh=-1 --grad-rho-setter --grad-order-stat 0.5 --grad-dynamic-primal-crit --grad-dynamic-dual-crit
+
+#--grad-rho-setter --grad-order-stat 0.5 --grad-dynamic-primal-crit
+
+# other ...
+
+# uses uc_funcs rho-setter
+# --use-cost-based-rho
+
+# --grad-dynamic-primal-crit
+# --grad-dynamic-dual-crit
+# --grad-dynamic-primal-thresh
+# --grad-dynamic-dual-thresh
+
+####mpiexec --oversubscribe -np 1 python -m mpi4py gradient_uc_cylinders.py --bundles-per-rank=0 --max-iterations=20 --default-rho=1 --num-scens=5 --max-solver-threads=2 --lagrangian-iter0-mipgap=1e-7 --ph-mipgaps-json=phmipgaps.json --solver-name=${SOLVERNAME} --xhatpath uc_cyl_nonants.npy --rel-gap 0.000001 --display-progress --grad-rho-setter --grad-order-stat 0.5 --grad-display-rho
+
+# ? do we need ph_ob_rho_rescale_factors_json", ph_ob_gradient_rho", ??
+
+# --fwph
+echo
+
+# --rho-setter --order-stat 0.5
+# --display-progress

mpisppy/tests/test_gradient_rho.py

@@ -190,7 +190,7 @@ def test_compute_rho(self):
         self.cfg.grad_cost_file_in = self.cfg.grad_cost_file_out
         self.rho_object = find_rho.Find_Rho(self.ph_object, self.cfg)
         rho = self.rho_object.compute_rho(indep_denom=True)
-        self.assertAlmostEqual(rho['DevotedAcreage[CORN0]'], 6.982758620689654)
+        self.assertAlmostEqual(rho['DevotedAcreage[CORN0]'], 6.982758620689655)
         rho = self.rho_object.compute_rho()
         self.assertAlmostEqual(rho['DevotedAcreage[CORN0]'], 8.163805471726114)
 
@@ -270,7 +270,7 @@ def test_grad_extensions(self):
         self.cfg.xhatpath = './examples/rho_test_data/farmer_cyl_nonants.npy'
         self.cfg.max_iterations = 4
         self.ph_object = self._run_ph_farmer()
-        self.assertAlmostEqual(self.ph_object.conv, 2.12869680, places=1)
+        self.assertAlmostEqual(self.ph_object.conv, 2.128968965420187, places=1)
 
 
 if __name__ == '__main__':

mpisppy/utils/config.py

@@ -786,7 +786,7 @@ def dynamic_gradient_args(self): # AKA adaptive
         self.add_to_config("grad_dynamic_primal_thresh",
                            description="primal threshold for diff during gradient calcs",
                            domain=float,
-                           default=0.001)
+                           default=0.1)
 
         self.add_to_config("grad_dynamic_dual_thresh",
                            description="dual threshold for abs norm during gradient calcs",

mpisppy/utils/find_rho.py

@@ -17,6 +17,7 @@
 import typing
 import copy
 import time
+from sortedcollections import OrderedSet
 
 import mpisppy.log
 from mpisppy import MPI
@@ -165,9 +166,7 @@ def compute_rho(self, indep_denom=False):
            rhos (dict): dict {variable name: list of rhos for this variable}
         """
         local_snames = self.ph_object.local_scenario_names
-        vnames = []  # TBD: why not just use c.keys()
-        for (sname, vname) in self.c.keys():
-            if vname not in vnames: vnames.append(vname)
+        vnames = OrderedSet(vname for (sname, vname) in self.c.keys())
         k0, s0 = list(self.ph_object.local_scenarios.items())[0]
         # TBD: drop vname_to_idx and use the map already provided
         vname_to_idx = {var.name : ndn_i[1] for ndn_i, var in s0._mpisppy_data.nonant_indices.items()}
@@ -249,6 +248,9 @@ def write_grad_rho(self):
                     for (vname, rho) in rho_data.items():
                         writer.writerow([vname, rho])
 
+        comm = self.ph_object.comms['ROOT']                    
+        comm.Barrier()                        
+
 
 class Set_Rho():
     """ Interface to set the computed rhos in PH.

mpisppy/utils/gradient.py

@@ -197,8 +197,9 @@ def write_grad_rho(self):
                      for (vname, rho) in rho_data.items():
                          writer.writerow([vname, rho_data[vname]])
 
+             # barrier added to avoid race-conditions involving other ranks reading.
              comm = self.ph_object.comms['ROOT']                    
-             comm.Barrier()                         
+             comm.Barrier()
 
 ###################################################################################
 

requirements.txt

@@ -1,5 +1,6 @@
 numpy
 scipy
 pandas
+sortedcollections
 mpi4py>=3.0.3
 pyomo>=6.2

setup.py

@@ -30,6 +30,7 @@
     packages=packages,
     python_requires='>=3.8',
     install_requires=[
+        'sortedcollections',
         'numpy<2',
         'scipy',
         'pyomo>=6.4',