Merge branch 'development' into sdc_newton_robustness

AMReX-Astro · Oct 8, 2023 · aa90c9d · aa90c9d
2 parents 4ad1016 + 09d4627
commit aa90c9d
Show file tree

Hide file tree

Showing 35 changed files with 579 additions and 1,756 deletions.
diff --git a/.github/workflows/good_defines.txt b/.github/workflows/good_defines.txt
@@ -5,7 +5,6 @@ AMREX_USE_CUDA
 AMREX_USE_GPU
 AMREX_USE_OMP
 AUX_THERMO
-AUX_UPDATE
 BL_FORT_USE_LOWERCASE
 BL_FORT_USE_UNDERSCORE
 BL_FORT_USE_UPPERCASE

diff --git a/CHANGES.md b/CHANGES.md
@@ -1,3 +1,9 @@
+# 23.11
+
+  * The compile option USE_AUX_UPDATE has been removed. If you want to
+    manually update the auxiliary parameters, you can use an external
+    source term or you can use the problem post-timestep hook. (#2614)
+
 # 23.10
 
   * True-SDC no longer evolves density as part of the reaction system

diff --git a/Docs/source/FlowChart.rst b/Docs/source/FlowChart.rst
@@ -176,17 +176,6 @@ of each step.
 
       Only Strang+CTU and simplified-SDC support retries.
 
-#. [AUX_UPDATE] *Auxiliary quantity evolution*
-
-   Auxiliary variables in Castro are those that obey a continuity
-   equation (with optional sources) that are passed into the EOS, but
-   not subjected to the constraint on mass fractions (summing to one).
-
-   The advection and source terms are already dealt with in the
-   main hydrodynamics advance (above step). A user-supplied routine
-   ca_auxupdate can be provided here to further update these
-   quantities.
-
 #. [POINTMASS] *Point mass*
 
    If ``castro.point_mass_fix_solution`` is set, then we

diff --git a/Docs/source/build_system.rst b/Docs/source/build_system.rst
@@ -177,13 +177,6 @@ Hydrodynamics and Source Term Parameters
 Simulation Flow Parameters
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-  * ``USE_AUX_UPDATE``: some networks define auxiliary quantities, which in general
-    Castro will advect, but not otherwise change.  If we set ``USE_AUX_UPDATE=TRUE``
-    then Castro will call a user-supplied routine ``advance_aux()`` that can
-    change the auxiliary quantities.
-
-    .. index:: USE_AUX_UPDATE
-
   * ``USE_POST_SIM``: if this is defined, then Castro will call the user-defined 
     routine ``problem_post_simulation()`` after the full evolution of the problem
     has ended.

diff --git a/Exec/Make.Castro b/Exec/Make.Castro
@@ -236,10 +236,6 @@ ifeq ($(USE_SHOCK_VAR), TRUE)
   DEFINES += -DSHOCK_VAR
 endif
 
-ifeq ($(USE_AUX_UPDATE), TRUE)
-  DEFINES += -DAUX_UPDATE
-endif
-
 ifeq ($(USE_POST_SIM), TRUE)
   DEFINES += -DDO_PROBLEM_POST_SIMULATION
 endif

diff --git a/Exec/gravity_tests/hse_convergence_general/_prob_params b/Exec/gravity_tests/hse_convergence_general/_prob_params
@@ -2,8 +2,6 @@ X_min                   real              1.e-4_rt                  y
 
 cutoff_density          real              500.e0_rt                 y
 
-dx_model                real              10.0_rt                   y
-
 T_hi                    real              5.e8_rt                   y
 
 T_star                  real              1.e8_rt                   y

diff --git a/Exec/gravity_tests/hse_convergence_general/inputs_1d.128.hse_test b/Exec/gravity_tests/hse_convergence_general/inputs_1d.128.hse_test
@@ -75,8 +75,6 @@ amr.plot_per         = 1.e-5      # number of seconds between plotfiles
 amr.derive_plot_vars = ALL
 
 # PROBLEM PARAMETERS
-problem.dx_model = 20.e0
-
 problem.dens_base = 3.43e6
 
 problem.T_star = 1.e8

diff --git a/Exec/gravity_tests/hse_convergence_general/inputs_1d.256.hse_test b/Exec/gravity_tests/hse_convergence_general/inputs_1d.256.hse_test
@@ -75,8 +75,6 @@ amr.plot_per         = 1.e-5      # number of seconds between plotfiles
 amr.derive_plot_vars = ALL
 
 # PROBLEM PARAMETERS
-problem.dx_model = 20.e0
-
 problem.dens_base = 3.43e6
 
 problem.T_star = 1.e8

diff --git a/Exec/gravity_tests/hse_convergence_general/inputs_1d.512.hse_test b/Exec/gravity_tests/hse_convergence_general/inputs_1d.512.hse_test
@@ -75,8 +75,6 @@ amr.plot_per         = 1.e-5      # number of seconds between plotfiles
 amr.derive_plot_vars = ALL
 
 # PROBLEM PARAMETERS
-problem.dx_model = 20.e0
-
 problem.dens_base = 3.43e6
 
 problem.T_star = 1.e8

diff --git a/Exec/gravity_tests/hse_convergence_general/problem_initialize.H b/Exec/gravity_tests/hse_convergence_general/problem_initialize.H
@@ -7,6 +7,7 @@
 #include <initial_model.H>
 #include <network.H>
 #include <ambient.H>
+#include <global.H>
 
 AMREX_INLINE
 void problem_initialize ()
@@ -133,9 +134,14 @@ void problem_initialize ()
         amrex::Error("ERROR: fuel mass fractions don't sum to 1");
     }
 
+    // we use the fine grid dx for the model resolution
+    auto fine_geom = global::the_amr_ptr->Geom(global::the_amr_ptr->maxLevel());
+
+    auto dx = fine_geom.CellSizeArray();
+    auto dx_model = dx[AMREX_SPACEDIM-1];
 
     int nx_model = static_cast<int>((probhi[AMREX_SPACEDIM-1] -
-                                     problo[AMREX_SPACEDIM-1]) / problem::dx_model);
+                                     problo[AMREX_SPACEDIM-1]) / dx_model);
 
     int ng = 4;
 
@@ -152,7 +158,7 @@ void problem_initialize ()
     model_params.low_density_cutoff = problem::low_density_cutoff;
 
     generate_initial_model(nx_model + ng,
-                           problo[AMREX_SPACEDIM-1] - ng*problem::dx_model,
+                           problo[AMREX_SPACEDIM-1] - ng * dx_model,
                            probhi[AMREX_SPACEDIM-1],
                            model_params);
 

diff --git a/Exec/hydro_tests/double_bubble/_prob_params b/Exec/hydro_tests/double_bubble/_prob_params
@@ -22,8 +22,6 @@ single                  integer       0               y
 # at the lower boundary, do we extrapolate the vertical momentum or set it to zero
 boundary_type           integer       1               y
 
-dx_model                real          0.00390625      y
-
 left_bubble_x_center    real          0.0_rt
 
 right_bubble_x_center   real          0.0_rt
diff --git a/Exec/hydro_tests/double_bubble/inputs_2d b/Exec/hydro_tests/double_bubble/inputs_2d
@@ -86,8 +86,6 @@ problem.pert_width = 0.025e0
 
 problem.do_isentropic = 1
 
-problem.dx_model = 0.00390625
-
 # EOS
 eos.species_a_name = "A"
 eos.species_a_gamma = 1.666e0

diff --git a/Exec/hydro_tests/double_bubble/inputs_2d.single b/Exec/hydro_tests/double_bubble/inputs_2d.single
@@ -88,8 +88,6 @@ problem.do_isentropic = 1
 
 problem.single = 1
 
-problem.dx_model = 0.005859375
-
 # EOS
 eos.species_a_name = "A"
 eos.species_a_gamma = 1.666e0

diff --git a/Exec/hydro_tests/double_bubble/inputs_2d.single.equal b/Exec/hydro_tests/double_bubble/inputs_2d.single.equal
@@ -88,8 +88,6 @@ problem.do_isentropic = 1
 
 problem.single = 1
 
-problem.dx_model = 0.005859375
-
 # EOS
 eos.species_a_name = "A"
 eos.species_a_gamma = 1.666e0

diff --git a/Exec/hydro_tests/double_bubble/inputs_2d.test b/Exec/hydro_tests/double_bubble/inputs_2d.test
@@ -86,8 +86,6 @@ problem.pert_width = 0.025e0
 
 problem.do_isentropic = 1
 
-problem.dx_model = 0.00390625
-
 # EOS
 eos.species_a_name = "A"
 eos.species_a_gamma = 1.666e0

diff --git a/Exec/hydro_tests/double_bubble/problem_initialize.H b/Exec/hydro_tests/double_bubble/problem_initialize.H
@@ -5,6 +5,7 @@
 #include <eos.H>
 #include <model_parser.H>
 #include <initial_model.H>
+#include <global.H>
 
 AMREX_INLINE
 void problem_initialize ()
@@ -37,8 +38,14 @@ void problem_initialize ()
 
     // we'll generate the initial model at the needed resolution
 
+    // we use the fine grid dx for the model resolution
+    auto fine_geom = global::the_amr_ptr->Geom(global::the_amr_ptr->maxLevel());
+
+    auto dx = fine_geom.CellSizeArray();
+    auto dx_model = dx[AMREX_SPACEDIM-1];
+
     int nx = (2.0_rt * problem::center[AMREX_SPACEDIM-1] + 1.e-8_rt) /
-        problem::dx_model;
+        dx_model;
 
     model_t model_params;
     model_params.dens_base = problem::dens_base;

diff --git a/Exec/hydro_tests/gamma_law_bubble/_prob_params b/Exec/hydro_tests/gamma_law_bubble/_prob_params
@@ -9,5 +9,3 @@ y_pert_center     real           0.7_rt      y
 pert_width        real           0.025_rt    y
 
 do_isentropic     integer        0           y
-
-dx_model          real           0.00390625  y
diff --git a/Exec/hydro_tests/gamma_law_bubble/problem_initialize.H b/Exec/hydro_tests/gamma_law_bubble/problem_initialize.H
@@ -5,6 +5,7 @@
 #include <eos.H>
 #include <model_parser.H>
 #include <initial_model.H>
+#include <global.H>
 
 AMREX_INLINE
 void problem_initialize ()
@@ -28,8 +29,14 @@ void problem_initialize ()
     // first make a 1D initial model for the entire domain
     // we'll create it ourselves, but hook it into the model_parser
 
+    // we use the fine grid dx for the model resolution
+    auto fine_geom = global::the_amr_ptr->Geom(global::the_amr_ptr->maxLevel());
+
+    auto dx = fine_geom.CellSizeArray();
+    auto dx_model = dx[AMREX_SPACEDIM-1];
+
     int nx = (2.0_rt * problem::center[AMREX_SPACEDIM-1] + 1.e-8_rt) /
-        problem::dx_model;
+        dx_model;
 
     model_t model_params;
     model_params.p_base = problem::pres_base;

diff --git a/Exec/reacting_tests/bubble_convergence/converge_test.sh b/Exec/reacting_tests/bubble_convergence/converge_test.sh
@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# echo the commands
+set -x
+
 EXEC=./Castro2d.gnu.MPI.TRUESDC.ex
 
 CONV_TOOL=RichardsonConvergenceTest2d.gnu.ex 

diff --git a/Exec/reacting_tests/reacting_convergence/ci-benchmarks/react_converge_128_true_sdc.out b/Exec/reacting_tests/reacting_convergence/ci-benchmarks/react_converge_128_true_sdc.out
@@ -21,16 +21,16 @@
  rho_e_sdc_source            -6.1984065959e+22       6.3082607681e+21
  Temp_sdc_source                             0                      0
  rho_He4_sdc_source              -543231.29383           78510.973919
- rho_C12_sdc_source              -1694.4890575           10.073635147
+ rho_C12_sdc_source              -1694.4890575           10.073635142
  rho_O16_sdc_source           -0.0098093776861       7.1506350196e-06
  rho_Fe56_sdc_source         -5.4492579275e-05       7.8513247889e-06
  pressure                     1.4155320805e+22       4.2608130858e+22
- kineng                       3.2091279198e-13       1.6647276226e+18
+ kineng                        5.228354476e-13       1.6647276226e+18
  soundspeed                       212069503.63           257404342.21
  Gamma_1                          1.5601126452           1.5885713572
- MachNumber                   5.3425093423e-18        0.0086982771596
+ MachNumber                   6.8192135042e-18        0.0086982771596
  entropy                           348439780.9           349209883.57
- magvort                                     0       0.00018541051835
+ magvort                      6.3108872418e-30       0.00018541051835
  divu                            -0.1163387912          0.55816306007
  eint_E                       4.5262357143e+16       6.9937847678e+16
  eint_e                       4.5262357143e+16       6.9937843728e+16
@@ -49,11 +49,11 @@
  z_velocity                                  0                      0
  t_sound_t_enuc               0.00023710316663           0.0195732693
  enuc                         2.9131490847e+15       4.5102586513e+17
- magvel                       1.1329833044e-09           2067446.1363
- radvel                      -0.00067837531107           2067446.1363
- circvel                                     0           11.820124028
- magmom                       0.00056649165215       1.6422547233e+12
+ magvel                        1.446147223e-09           2067446.1363
+ radvel                      -0.00067837194793           2067446.1363
+ circvel                                     0           11.820144682
+ magmom                       0.00072307361144       1.6422547233e+12
  angular_momentum_x                         -0                     -0
  angular_momentum_y                          0                      0
- angular_momentum_z          -1.2410862711e+14       1.2410862711e+14
+ angular_momentum_z          -1.2410862734e+14       1.2410862747e+14
 
diff --git a/Exec/science/flame_wave/problem_initialize.H b/Exec/science/flame_wave/problem_initialize.H
@@ -3,6 +3,7 @@
 
 #include <prob_parameters.H>
 #include <eos.H>
+#include <global.H>
 #include <network.H>
 #include <model_parser.H>
 #include <initial_model.H>

diff --git a/Source/driver/Castro.H b/Source/driver/Castro.H
@@ -997,16 +997,6 @@ public:
 ///
     void expand_state(amrex::MultiFab& S, amrex::Real time, int ng);
 
-#ifdef AUX_UPDATE
-
-///
-/// Calculate auxiliary variables at new timestep
-///
-/// @param time     current time
-/// @param dt       timestep
-///
-    void advance_aux(amrex::Real time, amrex::Real dt);
-#endif
 
 
 // Hydrodynamics

diff --git a/Source/driver/Castro.cpp b/Source/driver/Castro.cpp
@@ -2534,36 +2534,6 @@ Castro::okToContinue ()
     return test;
 }
 
-#ifdef AUX_UPDATE
-void
-Castro::advance_aux(Real time, Real dt)
-{
-    BL_PROFILE("Castro::advance_aux()");
-
-    if (verbose && ParallelDescriptor::IOProcessor()) {
-      std::cout << "... special update for auxiliary variables \n";
-    }
-
-    MultiFab&  S_old = get_old_data(State_Type);
-    MultiFab&  S_new = get_new_data(State_Type);
-
-#ifdef AMREX_USE_OMP
-#pragma omp parallel
-#endif
-    for (MFIter mfi(S_old, TilingIfNotGPU()); mfi.isValid(); ++mfi)
-    {
-        const Box& box = mfi.tilebox();
-        FArrayBox& old_fab = S_old[mfi];
-        FArrayBox& new_fab = S_new[mfi];
-        void ca_auxupdate(BL_TO_FORTRAN(old_fab),
-                          BL_TO_FORTRAN(new_fab),
-                          box.loVect(), box.hiVect(),
-                          &dt);
-    }
-}
-#endif
-
-
 void
 Castro::FluxRegCrseInit() {
     BL_PROFILE("Castro::FluxRegCrseInit");

diff --git a/Source/driver/Castro_F.H b/Source/driver/Castro_F.H
@@ -3,26 +3,6 @@
 
 #include <AMReX_BLFort.H>
 
-#ifdef __cplusplus
-#include <AMReX.H>
-extern "C"
-{
-#endif
-
-
-#ifdef AUX_UPDATE
-  void ca_auxupdate
-    (BL_FORT_FAB_ARG(state_old),
-     BL_FORT_FAB_ARG(state_new),
-     const int* lo, const int* hi,
-     const amrex::Real * dt);
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-
 #ifdef MAESTRO_INIT
 BL_FORT_PROC_DECL(CA_INITDATA_MAESTRO,ca_initdata_maestro)
   (const int* lo, const int* hi, const int& MAESTRO_init_type,

diff --git a/Source/driver/Castro_advance.cpp b/Source/driver/Castro_advance.cpp
@@ -91,10 +91,6 @@ Castro::advance (Real time,
     }
 
     for (int lev = level; lev <= max_level_to_advance; ++lev) {
-#ifdef AUX_UPDATE
-        getLevel(lev).advance_aux(time, dt);
-#endif
-
 #ifdef GRAVITY
         // Update the point mass.
         if (use_point_mass == 1) {

diff --git a/Source/driver/Make.package b/Source/driver/Make.package
@@ -30,8 +30,9 @@ CEXE_sources += Derive.cpp
 CEXE_headers += Castro_generic_fill.H
 CEXE_sources += Castro_generic_fill.cpp
 
-CEXE_headers      += Castro_util.H
-CEXE_headers      += math.H
+CEXE_headers += Castro_util.H
+CEXE_headers += global.H
+CEXE_headers += math.H
 
 NEED_MGUTIL =
Original file line number	Diff line number	Diff line change
Expand Up		@@ -9,5 +9,3 @@ y_pert_center real 0.7_rt y
		pert_width real 0.025_rt y

		do_isentropic integer 0 y

		dx_model real 0.00390625 y