[WIP] add Catalyst support

.devcontainer/Dockerfile

@@ -8,10 +8,40 @@ RUN apt-get --yes -qq update \
                       sphinx-doc python3-sphinx-rtd-theme python3-sphinxcontrib.bibtex python3-sphinx-copybutton \
                       libopenmpi-dev \
                       libhdf5-mpi-dev \
+                      libglew-dev \
  && apt-get --yes -qq clean \
  && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /home/ubuntu
+ENV CMAKE_PREFIX_PATH="/opt/conduit:/opt/catalyst"
+ENV catalyst_DIR="/opt/catalyst/lib/catalyst"
+
+# build Conduit
+RUN git clone --recursive https://github.com/llnl/conduit.git \
+ && cd conduit \
+ && mkdir -p /opt/conduit \
+ && cmake -S src -B build -DCMAKE_BUILD_TYPE=Debug -DENABLE_MPI=ON -DCMAKE_INSTALL_PREFIX=/opt/conduit \
+ && cmake --build build --parallel 8 \
+ && cmake --install build \
+ && cmake --build build --target clean
+
+# build Catalyst
+RUN git clone https://gitlab.kitware.com/paraview/catalyst.git \
+ && cd catalyst \
+ && mkdir -p /opt/catalyst \
+ && cmake -B build -S . -DCATALYST_WITH_EXTERNAL_CONDUIT=ON -DCMAKE_INSTALL_PREFIX=/opt/catalyst \
+ && cmake --build build --parallel 8 \
+ && cmake --install build \
+ && cmake --build build --target clean
+
+# build Paraview
+RUN git clone --recursive https://gitlab.kitware.com/paraview/paraview.git \
+ && cd paraview \
+ && cmake -B build -S . -GNinja -DPARAVIEW_BUILD_EDITION=CATALYST_RENDERING -DPARAVIEW_ENABLE_CATALYST=ON -DPARAVIEW_USE_FORTRAN=OFF -DPARAVIEW_USE_PYTHON=ON -DPARAVIEW_USE_MPI=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/paraview \
+ && cmake --build build --parallel 8 \
+ && cmake --install build \
+ && cmake --build build --target clean
+
 USER ubuntu
 
 # install esbonio for Sphinx VSCode support (no Ubuntu package for this)

src/CMakeLists.txt

@@ -129,14 +129,15 @@ link_libraries(yaml-cpp::yaml-cpp)
 include(CTest)
 
 set (QuokkaSourcesNoEOS "${CMAKE_CURRENT_SOURCE_DIR}/main.cpp"
-                        "${CMAKE_CURRENT_SOURCE_DIR}/io/DiagBase.cpp" 
-                        "${CMAKE_CURRENT_SOURCE_DIR}/io/DiagFilter.cpp" 
-                        "${CMAKE_CURRENT_SOURCE_DIR}/io/DiagFramePlane.cpp" 
-                        "${CMAKE_CURRENT_SOURCE_DIR}/io/DiagPDF.cpp" 
                         "${CMAKE_CURRENT_SOURCE_DIR}/cooling/GrackleLikeCooling.cpp" 
                         "${CMAKE_CURRENT_SOURCE_DIR}/cooling/GrackleDataReader.cpp"
                         "${CMAKE_CURRENT_SOURCE_DIR}/cooling/TabulatedCooling.cpp" 
                         "${CMAKE_CURRENT_SOURCE_DIR}/cooling/CloudyDataReader.cpp" 
+                        "${CMAKE_CURRENT_SOURCE_DIR}/io/DiagBase.cpp" 
+                        "${CMAKE_CURRENT_SOURCE_DIR}/io/DiagFilter.cpp" 
+                        "${CMAKE_CURRENT_SOURCE_DIR}/io/DiagFramePlane.cpp" 
+                        "${CMAKE_CURRENT_SOURCE_DIR}/io/DiagPDF.cpp" 
+                        "${CMAKE_CURRENT_SOURCE_DIR}/io/catalyst.cpp"
                         "${openPMDSources}")
 
 set (QuokkaObjSources "${QuokkaSourcesNoEOS}" "${gamma_law_sources}")

src/io/Catalyst.H

@@ -0,0 +1,32 @@
+#ifndef CATALYST_H_
+#define CATALYST_H_
+
+#include <string>
+
+#include "AMReX_BaseFwd.H"
+#include "AMReX_Geometry.H"
+#include "AMReX_Vector.H"
+
+/**
+ * \brief This class aims at dumping performing in-situ analysis and visualization
+ * with Catalyst. Catalyst initialize and finalize are called in the constructor
+ * and destructor respectivelyThe function WriteToFile takes in the particles,
+ * meshes, and fields, and formats them to be used by given pipeline scripts. All
+ * exports are defined and executed externally through the given catalyst pipeline
+ * and implementation.
+ */
+class FlushFormatCatalyst
+{
+      public:
+	// Initailize
+	FlushFormatCatalyst();
+
+	/** Send particle, mesh, and field information through the catalyst pipeline */
+	void WriteToFile(amrex::Vector<std::string> const &varnames, amrex::Vector<amrex::MultiFab> const &mf, amrex::Vector<amrex::Geometry> const &geom,
+			 amrex::Vector<amrex::IntVect> const &ref_ratio, amrex::Vector<int> const &iteration, const double time, int nlev) const;
+
+	// Finalize
+	~FlushFormatCatalyst();
+};
+
+#endif

src/io/catalyst.cpp

@@ -0,0 +1,108 @@
+#include <string>
+
+#include "AMReX.H"
+#include "AMReX_ParmParse.H"
+
+#ifdef AMREX_USE_CATALYST
+#include "AMReX_Conduit_Blueprint.H"
+#include "catalyst_api.h"
+#include "conduit_cpp_to_c.hpp"
+#endif
+
+#include "Catalyst.H"
+
+using namespace amrex;
+
+FlushFormatCatalyst::FlushFormatCatalyst()
+{
+	ParmParse pp_catalyst("catalyst");
+	std::string scriptPaths{""};
+	std::string implementation{"paraview"};
+	std::string searchPaths{""};
+	pp_catalyst.query("script_paths", scriptPaths);
+	pp_catalyst.query("implementation", implementation);
+	pp_catalyst.query("implementation_search_paths", searchPaths);
+
+#ifdef AMREX_USE_CATALYST
+	conduit::Node node;
+
+	// Loop over all given paths and load all the scripts. Delimiters are ';' and ':'
+	size_t scriptNumber = 0;
+	size_t pos = 0;
+	std::string subpath;
+	while (scriptPaths.find(':') != std::string::npos || scriptPaths.find(';') != std::string::npos) {
+		pos = std::min(scriptPaths.find(':'), scriptPaths.find(';'));
+		subpath = scriptPaths.substr(0, pos);
+
+		node["catalyst/scripts/script" + std::to_string(scriptNumber)].set_string(subpath);
+
+		scriptNumber++;
+		scriptPaths.erase(0, pos + 1);
+	}
+	// Prevent empty end paths
+	if (scriptPaths.length() != 0) {
+		node["catalyst/scripts/script" + std::to_string(scriptNumber)].set_string(scriptPaths);
+	}
+
+	node["catalyst_load/implementation"].set_string(implementation);
+	node["catalyst_load/search_paths/" + implementation].set_string(searchPaths);
+
+	catalyst_status err = catalyst_initialize(conduit::c_node(&node));
+	if (err != catalyst_status_ok) {
+		std::string message = " Error: Failed to initialize Catalyst!\n";
+		std::cerr << message << err << std::endl;
+		amrex::Print() << message;
+		amrex::Abort(message);
+	}
+#endif // AMREX_USE_CATALYST
+}
+
+void FlushFormatCatalyst::WriteToFile(amrex::Vector<std::string> const &varnames, amrex::Vector<amrex::MultiFab> const &mf,
+				      amrex::Vector<amrex::Geometry> const &geom, amrex::Vector<amrex::IntVect> const &ref_ratio,
+				      amrex::Vector<int> const &iteration, const double time, int nlev) const
+{
+#ifdef AMREX_USE_CATALYST
+	amrex::Print() << "Running Catalyst pipeline scripts...";
+	const BL_PROFILE("FlushFormatCatalyst::WriteToFile()");
+
+	// Mesh data
+	conduit::Node node;
+	auto &state = node["catalyst/state"];
+	state["timestep"].set(iteration[0]);
+	state["time"].set(time);
+
+	auto &meshChannel = node["catalyst/channels/mesh"];
+	// meshChannel["type"].set_string("amrmesh");
+	meshChannel["type"].set_string("multimesh");
+	auto &meshData = meshChannel["data"];
+
+	amrex::MultiLevelToBlueprint(nlev, amrex::GetVecOfConstPtrs(mf), varnames, geom, time, iteration, ref_ratio, meshData);
+
+	// Execution
+	catalyst_status err = catalyst_execute(conduit::c_node(&node));
+	if (err != catalyst_status_ok) {
+		std::string message = " Error: Failed to execute Catalyst!\n";
+		std::cerr << message << err << std::endl;
+		amrex::Print() << message;
+	}
+#else
+	amrex::ignore_unused(varnames, mf, geom, iteration, time, nlev);
+#endif // AMREX_USE_CATALYST
+}
+
+FlushFormatCatalyst::~FlushFormatCatalyst()
+{
+#ifdef AMREX_USE_CATALYST
+	conduit::Node node;
+	catalyst_status err = catalyst_finalize(conduit::c_node(&node));
+	if (err != catalyst_status_ok) {
+		std::string message = " Error: Failed to finalize Catalyst!\n";
+		std::cerr << message << err << std::endl;
+		amrex::Print() << message;
+		amrex::Abort(message);
+	} else {
+		// Temporary, remove for final
+		std::cout << "Successfully finalized Catalyst" << std::endl;
+	}
+#endif // AMREX_USE_CATALYST
+}

src/simulation.hpp

@@ -76,14 +76,10 @@ namespace filesystem = experimental::filesystem;
 #include "AMReX_OpenBC.H"
 #endif
 
-#ifdef AMREX_USE_ASCENT
-#include <AMReX_Conduit_Blueprint.H>
-#include <ascent.hpp>
-#endif
-
 // internal headers
 #include "fundamental_constants.H"
 #include "grid.hpp"
+#include "io/Catalyst.H"
 #include "io/DiagBase.H"
 #include "physics_info.hpp"
 
@@ -93,13 +89,6 @@ namespace filesystem = experimental::filesystem;
 
 #define USE_YAFLUXREGISTER
 
-#ifdef AMREX_USE_ASCENT
-using namespace conduit;
-using namespace ascent;
-#endif
-
-enum class ParticleStep { BeforePoissonSolve, AfterPoissonSolve };
-
 using variant_t = std::variant<amrex::Real, std::string>;
 
 namespace YAML
@@ -335,10 +324,8 @@ template <typename problem_t> class AMRSimulation : public amrex::AmrCore
 	void kickParticlesAllLevels(amrex::Real dt);
 	void driftParticlesAllLevels(amrex::Real dt);
 
-#ifdef AMREX_USE_ASCENT
-	void AscentCustomActions(conduit::Node const &blueprintMesh);
 	void RenderAscent();
-#endif
+
       protected:
 	amrex::Vector<amrex::BCRec> BCs_cc_; // on level 0
 	amrex::Vector<amrex::BCRec> BCs_fc_; // on level 0
@@ -402,9 +389,7 @@ template <typename problem_t> class AMRSimulation : public amrex::AmrCore
 #endif
 
 	// external objects
-#ifdef AMREX_USE_ASCENT
-	Ascent ascent_;
-#endif
+	std::unique_ptr<FlushFormatCatalyst> catalyst_;
 };
 
 template <typename problem_t> void AMRSimulation<problem_t>::setChkFile(std::string const &chkfile_number) { restart_chkfile = chkfile_number; }
@@ -475,12 +460,10 @@ template <typename problem_t> void AMRSimulation<problem_t>::initialize()
 		}
 	}
 
-#ifdef AMREX_USE_ASCENT
-	// initialize Ascent
-	conduit::Node ascent_options;
-	ascent_options["mpi_comm"] = MPI_Comm_c2f(amrex::ParallelContext::CommunicatorSub());
-	ascent_.open(ascent_options);
-#endif
+	// initialize Catalyst
+	if (ascentInterval_ > 0) {
+		catalyst_ = std::make_unique<FlushFormatCatalyst>();
+	}
 }
 
 template <typename problem_t> void AMRSimulation<problem_t>::PerformanceHints()
@@ -490,9 +473,8 @@ template <typename problem_t> void AMRSimulation<problem_t>::PerformanceHints()
 		const amrex::Long nboxes = boxArray(ilev).size();
 		if (amrex::ParallelDescriptor::NProcs() > nboxes) {
 			amrex::Print() << "\n[Warning] [Performance] Too many resources / too little work!\n"
-				       << "  It looks like you requested more compute resources than "
-				       << "  the number of boxes of cells available on level " << ilev << " (" << nboxes << "). "
-				       << "You started with (" << amrex::ParallelDescriptor::NProcs() << ") MPI ranks, so ("
+				       << "  It looks like you requested more compute resources than " << "  the number of boxes of cells available on level "
+				       << ilev << " (" << nboxes << "). " << "You started with (" << amrex::ParallelDescriptor::NProcs() << ") MPI ranks, so ("
 				       << amrex::ParallelDescriptor::NProcs() - nboxes << ") rank(s) will have no work on this level.\n"
 #ifdef AMREX_USE_GPU
 				       << "  On GPUs, consider using 1-8 boxes per GPU per level that "
@@ -673,11 +655,9 @@ template <typename problem_t> void AMRSimulation<problem_t>::setInitialCondition
 		amrex::Abort();
 	}
 
-#ifdef AMREX_USE_ASCENT
 	if (ascentInterval_ > 0) {
 		RenderAscent();
 	}
-#endif
 
 	if (plotfileInterval_ > 0) {
 		WritePlotFile();
@@ -831,9 +811,7 @@ template <typename problem_t> void AMRSimulation<problem_t>::evolve()
 	AMREX_ALWAYS_ASSERT(areInitialConditionsDefined_);
 
 	amrex::Real cur_time = tNew_[0];
-#ifdef AMREX_USE_ASCENT
 	int last_ascent_step = 0;
-#endif
 	int last_projection_step = 0;
 	int last_statistics_step = 0;
 	int last_plot_file_step = 0;
@@ -895,12 +873,10 @@ template <typename problem_t> void AMRSimulation<problem_t>::evolve()
 			tNew_[lev] = cur_time;
 		}
 
-#ifdef AMREX_USE_ASCENT
 		if (ascentInterval_ > 0 && (step + 1) % ascentInterval_ == 0) {
 			last_ascent_step = step + 1;
 			RenderAscent();
 		}
-#endif
 
 		if (statisticsInterval_ > 0 && (step + 1) % statisticsInterval_ == 0) {
 			last_statistics_step = step + 1;
@@ -1001,11 +977,6 @@ template <typename problem_t> void AMRSimulation<problem_t>::evolve()
 	if (checkpointInterval_ > 0 && istep[0] > last_chk_file_step) {
 		WriteCheckpointFile();
 	}
-
-#ifdef AMREX_USE_ASCENT
-	// close Ascent
-	ascent_.close();
-#endif
 }
 
 template <typename problem_t> void AMRSimulation<problem_t>::calculateGpotAllLevels()
@@ -2204,45 +2175,13 @@ template <typename problem_t> void AMRSimulation<problem_t>::doDiagnostics()
 	}
 }
 
-// do in-situ rendering with Ascent
-#ifdef AMREX_USE_ASCENT
-template <typename problem_t> void AMRSimulation<problem_t>::AscentCustomActions(conduit::Node const &blueprintMesh)
-{
-	BL_PROFILE("AMRSimulation::AscentCustomActions()");
-
-	// add a scene with a pseudocolor plot
-	Node scenes;
-	scenes["s1/plots/p1/type"] = "pseudocolor";
-	scenes["s1/plots/p1/field"] = "gasDensity";
-
-	// set the output file name (ascent will add ".png")
-	scenes["s1/renders/r1/image_prefix"] = "render_density%05d";
-
-	// set camera position
-	amrex::Array<double, 3> position = {-0.6, -0.6, -0.8};
-	scenes["s1/renders/r1/camera/position"].set_float64_ptr(position.data(), 3);
-
-	// setup actions
-	Node actions;
-	Node &add_plots = actions.append();
-	add_plots["action"] = "add_scenes";
-	add_plots["scenes"] = scenes;
-	actions.append()["action"] = "execute";
-	actions.append()["action"] = "reset";
-
-	// send AMR mesh to ascent, do render
-	ascent_.publish(blueprintMesh);
-	ascent_.execute(actions); // will be replaced by ascent_actions.yml if present
-}
-
 // do Ascent render
 template <typename problem_t> void AMRSimulation<problem_t>::RenderAscent()
 {
 	BL_PROFILE("AMRSimulation::RenderAscent()");
 
 	// combine multifabs
 	amrex::Vector<amrex::MultiFab> mf = PlotFileMF(nghost_cc_);
-	amrex::Vector<const amrex::MultiFab *> mf_ptr = amrex::GetVecOfConstPtrs(mf);
 	amrex::Vector<std::string> varnames;
 	varnames.insert(varnames.end(), componentNames_cc_.begin(), componentNames_cc_.end());
 	varnames.insert(varnames.end(), derivedNames_.begin(), derivedNames_.end());
@@ -2264,18 +2203,8 @@ template <typename problem_t> void AMRSimulation<problem_t>::RenderAscent()
 		rescaledGeom[i].ProbDomain(rescaledRealBox);
 	}
 
-	// wrap MultiFabs into a Blueprint mesh
-	conduit::Node blueprintMesh;
-	amrex::MultiLevelToBlueprint(finest_level + 1, mf_ptr, varnames, rescaledGeom, tNew_[0], istep, refRatio(), blueprintMesh);
-
-	// copy to host mem (needed for DataBinning)
-	conduit::Node bpMeshHost;
-	bpMeshHost.set(blueprintMesh);
-
-	// pass Blueprint mesh to Ascent, run actions
-	AscentCustomActions(bpMeshHost);
+	catalyst_->WriteToFile(varnames, mf, rescaledGeom, refRatio(), istep, tNew_[0], finestLevel());
 }
-#endif // AMREX_USE_ASCENT
 
 template <typename problem_t> auto AMRSimulation<problem_t>::GetPlotfileVarNames() const -> amrex::Vector<std::string>
 {