Dev

examples/cavity/cavity_impedance.json

@@ -3,11 +3,11 @@
   {
     "Type": "Eigenmode",
     "Verbose": 2,
-    "Output": "postpro/impedance"
+    "Output": "D:/WelSimLLC/executable28/_palace_examples/cavity/postpro/impedance"
   },
   "Model":
   {
-    "Mesh": "mesh/cavity.msh",
+    "Mesh": "D:/WelSimLLC/executable28/_palace_examples/cavity/mesh/cavity.msh",
     "L0": 1.0e-2,  // cm
     "Refinement":
     {

examples/cavity/cavity_pec.json

@@ -3,11 +3,11 @@
   {
     "Type": "Eigenmode",
     "Verbose": 2,
-    "Output": "postpro/pec"
+    "Output": "D:/WelSimLLC/executable28/_palace_examples/cavity/postpro/pec"
   },
   "Model":
   {
-    "Mesh": "mesh/cavity.msh",
+    "Mesh": "D:/WelSimLLC/executable28/_palace_examples/cavity/mesh/cavity.msh",
     "L0": 1.0e-2,  // cm
     "Refinement":
     {

examples/coaxial/coaxial_matched.json

@@ -3,11 +3,11 @@
   {
     "Type": "Transient",
     "Verbose": 2,
-    "Output": "postpro/matched"
+    "Output": "D:/WelSimLLC/executable28/_palace_examples/coaxial/postpro/matched"
   },
   "Model":
   {
-    "Mesh": "mesh/coaxial.msh",
+    "Mesh": "D:/WelSimLLC/executable28/_palace_examples/coaxial/mesh/coaxial.msh",
     "L0": 1.0e-3  // mm
   },
   "Domains":

examples/coaxial/coaxial_open.json

@@ -3,11 +3,11 @@
   {
     "Type": "Transient",
     "Verbose": 2,
-    "Output": "postpro/open"
+    "Output": "D:/WelSimLLC/executable28/_palace_examples/coaxial/postpro/open"
   },
   "Model":
   {
-    "Mesh": "mesh/coaxial.msh",
+    "Mesh": "D:/WelSimLLC/executable28/_palace_examples/coaxial/mesh/coaxial.msh",
     "L0": 1.0e-3  // mm
   },
   "Domains":

examples/coaxial/coaxial_short.json

@@ -3,11 +3,11 @@
   {
     "Type": "Transient",
     "Verbose": 2,
-    "Output": "postpro/short"
+    "Output": "D:/WelSimLLC/executable28/_palace_examples/coaxial/postpro/short"
   },
   "Model":
   {
-    "Mesh": "mesh/coaxial.msh",
+    "Mesh": "D:/WelSimLLC/executable28/_palace_examples/coaxial/mesh/coaxial.msh",
     "L0": 1.0e-3  // mm
   },
   "Domains":

examples/cpw/cpw_lumped_adaptive.json

@@ -3,11 +3,11 @@
   {
     "Type": "Driven",
     "Verbose": 2,
-    "Output": "postpro/lumped_adaptive"
+    "Output": "D:/WelSimLLC/executable28/_palace_examples/cpw/postpro/lumped_adaptive"
   },
   "Model":
   {
-    "Mesh": "mesh/cpw_lumped.msh",
+    "Mesh": "D:/WelSimLLC/executable28/_palace_examples/cpw/mesh/cpw_lumped.msh",
     "L0": 1.0e-6,  // μm
     "Refinement":
     {

examples/cpw/cpw_lumped_uniform.json

@@ -3,11 +3,11 @@
   {
     "Type": "Driven",
     "Verbose": 2,
-    "Output": "postpro/lumped_uniform"
+    "Output": "D:/WelSimLLC/executable28/_palace_examples/cpw/postpro/lumped_uniform"
   },
   "Model":
   {
-    "Mesh": "mesh/cpw_lumped.msh",
+    "Mesh": "D:/WelSimLLC/executable28/_palace_examples/cpw/mesh/cpw_lumped.msh",
     "L0": 1.0e-6,  // μm
     "Refinement":
     {

examples/cpw/cpw_wave_adaptive.json

@@ -3,11 +3,11 @@
   {
     "Type": "Driven",
     "Verbose": 2,
-    "Output": "postpro/wave_adaptive"
+    "Output": "D:/WelSimLLC/executable28/_palace_examples/cpw/postpro/wave_adaptive"
   },
   "Model":
   {
-    "Mesh": "mesh/cpw_wave.msh",
+    "Mesh": "D:/WelSimLLC/executable28/_palace_examples/cpw/mesh/cpw_wave.msh",
     "L0": 1.0e-6,  // μm
     "Refinement":
     {

examples/cpw/cpw_wave_uniform.json

@@ -3,11 +3,11 @@
   {
     "Type": "Driven",
     "Verbose": 2,
-    "Output": "postpro/wave_uniform"
+    "Output": "D:/WelSimLLC/executable28/_palace_examples/cpw/postpro/wave_uniform"
   },
   "Model":
   {
-    "Mesh": "mesh/cpw_wave.msh",
+    "Mesh": "D:/WelSimLLC/executable28/_palace_examples/cpw/mesh/cpw_wave.msh",
     "L0": 1.0e-6,  // μm
     "Refinement":
     {

examples/rings/rings.json

@@ -3,11 +3,11 @@
   {
     "Type": "Magnetostatic",
     "Verbose": 2,
-    "Output": "postpro"
+    "Output": "D:/WelSimLLC/executable28/_palace_examples/rings/postpro"
   },
   "Model":
   {
-    "Mesh": "mesh/rings.msh",
+    "Mesh": "D:/WelSimLLC/executable28/_palace_examples/rings/mesh/rings.msh",
     "L0": 1.0e-6  // um
   },
   "Domains":

examples/spheres/spheres.json

@@ -3,11 +3,11 @@
   {
     "Type": "Electrostatic",
     "Verbose": 2,
-    "Output": "postpro"
+    "Output": "D:/WelSimLLC/executable28/_palace_examples/spheres/postpro"
   },
   "Model":
   {
-    "Mesh": "mesh/spheres.msh",
+    "Mesh": "D:/WelSimLLC/executable28/_palace_examples/spheres/mesh/spheres.msh",
     "L0": 1.0e-2  // cm
   },
   "Domains":

myRunningScripts.txt

@@ -0,0 +1,51 @@
+palace_d _palace_examples/spheres/spheres.json
+
+Electrostatic:
+palace _palace_examples/spheres/spheres.json, 20s for OpenMP
+
+Magnetostatic:
+palace _palace_examples/rings/rings.json, 36s for OpenMP
+
+Eigenmode:
+palace _palace_examples/cavity/cavity_impedance.json, 14s for OpenMP
+palace _palace_examples/cavity/cavity_pec.json, 10s for OpenMP
+
+Driven:
+palace _palace_examples/cpw/cpw_lumped_adaptive.json, 216s for OpenMP
+palace _palace_examples/cpw/cpw_lumped_uniform.json, 170s for OpenMP
+palace _palace_examples/cpw/cpw_wave_adaptive.json
+palace _palace_examples/cpw/cpw_wave_uniform.json
+
+Transient:
+palace _palace_examples/coaxial/coaxial_matched.json, 30s for OpenMP
+palace _palace_examples/coaxial/coaxial_open.json, 29s for OpenMP
+palace _palace_examples/coaxial/coaxial_short.json, 29s for OpenMP
+
+
+=== CUDA building ===
+#ifndef __CUDACC__  
+#define __CUDACC__
+#endif
+#include <cuda.h>
+#include <cuda_runtime.h>
+#include <device_launch_parameters.h>
+//#include <device_functions.h>
+
+
+
+=== CUDA libs ===
+MAGMA\magma.lib
+MAGMA\magma_sparse.lib
+$(CUDA_PATH)\lib\x64\cublas.lib
+$(CUDA_PATH)\lib\x64\cudart.lib
+$(CUDA_PATH)\lib\x64\cuda.lib
+$(CUDA_PATH)\lib\x64\cusparse.lib
+$(CUDA_PATH)\lib\x64\nvrtc.lib
+
+
+
+
+=== Currently CUDA Windows building issues ===
+1. In MFEM, On Windows, the enclosing parent function ("AddWithMarkers_") for an extended __host__ __device__ lambda cannot have internal or no linkage.
+2. 
+

palace/env_var.bat

@@ -1,14 +1,14 @@
-set WELSIM_LIBPACK=D:\WelSimLLC\CodeDV\libPack
-set WELSIM_EXEC=D:\WelSimLLC\executable28
-
-set BOOST_VC=143
-set BOOST_VERSION=1_80
-set VTK_VER=8.2
-set ITK_VER=5.2
-set QTDIR=C:\Qt\5.15.2\msvc2019_64
-set QTBIN=%QTDIR%\bin
-set INTEL_MKL=C:\Program Files (x86)\Intel\oneAPI\mkl\2022.2.0
-
-
-set PATH="C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\";%QTBIN%;%PATH%
-set MSVS_DEVENV="C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\devenv.exe" 
+set WELSIM_LIBPACK=D:\WelSimLLC\CodeDV\libPack
+set WELSIM_EXEC=D:\WelSimLLC\executable28
+
+set BOOST_VC=143
+set BOOST_VERSION=1_80
+set VTK_VER=8.2
+set ITK_VER=5.2
+set QTDIR=C:\Qt\5.15.2\msvc2019_64
+set QTBIN=%QTDIR%\bin
+set INTEL_MKL=C:\Program Files (x86)\Intel\oneAPI\mkl\2022.2.0
+
+
+set PATH="C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\";%QTBIN%;%PATH%
+set MSVS_DEVENV="C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\devenv.exe" 

palace/libpalace.props

@@ -1,19 +1,19 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <ImportGroup Label="PropertySheets" />
-  <PropertyGroup Label="UserMacros" />
-  <PropertyGroup>
-    <ExternalIncludePath>$(MSMPI_INC);.\;$(MSMPI_INC)\x64;$(WELSIM_LIBPACK)\include\palace\mfem;$(WELSIM_LIBPACK)\include\zlib;$(WELSIM_LIBPACK)\include\hypre;$(WELSIM_LIBPACK)\include\petsc;$(WELSIM_LIBPACK)\include\SLEPc;$(WELSIM_LIBPACK)\include\fmt;$(WELSIM_LIBPACK)\include\palace;$(WELSIM_LIBPACK)\include\palace\libCEED;$(WELSIM_LIBPACK)\include;$(INTEL_MKL)\include;$(WELSIM_LIBPACK)\include\eigen3;$(WELSIM_LIBPACK)\include\mumps;$(WELSIM_LIBPACK)\include\palace\arpack;$(ExternalIncludePath)</ExternalIncludePath>
-    <OutDir>$(Platform)\$(Configuration)\</OutDir>
-  </PropertyGroup>
-  <ItemDefinitionGroup>
-    <PostBuildEvent>
-      <Command>postbuild_libpalace</Command>
-    </PostBuildEvent>
-    <ClCompile>
-      <MultiProcessorCompilation>true</MultiProcessorCompilation>
-      <PreprocessorDefinitions>_UNICODE;UNICODE;CEED_SKIP_VISIBILITY;PALACE_WITH_ARPACK;PALACE_WITH_MUMPS;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-    </ClCompile>
-  </ItemDefinitionGroup>
-  <ItemGroup />
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ImportGroup Label="PropertySheets" />
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <ExternalIncludePath>$(MSMPI_INC);.\;$(MSMPI_INC)\x64;$(WELSIM_LIBPACK)\include\palace\mfem;$(WELSIM_LIBPACK)\include\palace\arpack;$(WELSIM_LIBPACK)\include\palace\libCEED;$(WELSIM_LIBPACK)\include\palace;$(WELSIM_LIBPACK)\include\zlib;$(WELSIM_LIBPACK)\include\hypre;$(WELSIM_LIBPACK)\include\petsc;$(WELSIM_LIBPACK)\include\SLEPc;$(WELSIM_LIBPACK)\include\fmt;$(WELSIM_LIBPACK)\include;$(INTEL_MKL)\include;$(WELSIM_LIBPACK)\include\eigen3;$(WELSIM_LIBPACK)\include\mumps;$(CUDA_PATH)\include;$(ExternalIncludePath)</ExternalIncludePath>
+    <OutDir>$(Platform)\$(Configuration)\</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <PostBuildEvent>
+      <Command>postbuild_libpalace</Command>
+    </PostBuildEvent>
+    <ClCompile>
+      <MultiProcessorCompilation>true</MultiProcessorCompilation>
+      <PreprocessorDefinitions>_UNICODE;UNICODE;CEED_SKIP_VISIBILITY;PALACE_WITH_ARPACK;PALACE_WITH_MUMPS;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup />
 </Project>

palace/libpalace_d.props

@@ -1,19 +1,19 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <ImportGroup Label="PropertySheets" />
-  <PropertyGroup Label="UserMacros" />
-  <PropertyGroup>
-    <ExternalIncludePath>$(MSMPI_INC);.\;$(MSMPI_INC)\x64;$(WELSIM_LIBPACK)\include\palace\mfem;$(WELSIM_LIBPACK)\include\zlib;$(WELSIM_LIBPACK)\include\hypre;$(WELSIM_LIBPACK)\include\petsc;$(WELSIM_LIBPACK)\include\SLEPc;$(WELSIM_LIBPACK)\include\fmt;$(WELSIM_LIBPACK)\include\palace;$(WELSIM_LIBPACK)\include\palace\libCEED;$(WELSIM_LIBPACK)\include;$(INTEL_MKL)\include;$(WELSIM_LIBPACK)\include\eigen3;$(WELSIM_LIBPACK)\include\mumps;$(WELSIM_LIBPACK)\include\palace\arpack;$(ExternalIncludePath)</ExternalIncludePath>
-    <OutDir>$(Platform)\$(Configuration)\</OutDir>
-  </PropertyGroup>
-  <ItemDefinitionGroup>
-    <PostBuildEvent>
-      <Command>postbuild_libpalace_d</Command>
-    </PostBuildEvent>
-    <ClCompile>
-      <MultiProcessorCompilation>true</MultiProcessorCompilation>
-      <PreprocessorDefinitions>_UNICODE;UNICODE;CEED_SKIP_VISIBILITY;PALACE_WITH_ARPACK;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-    </ClCompile>
-  </ItemDefinitionGroup>
-  <ItemGroup />
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ImportGroup Label="PropertySheets" />
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <ExternalIncludePath>$(MSMPI_INC);.\;$(MSMPI_INC)\x64;$(WELSIM_LIBPACK)\include\palace\mfem;$(WELSIM_LIBPACK)\include\zlib;$(WELSIM_LIBPACK)\include\hypre;$(WELSIM_LIBPACK)\include\petsc;$(WELSIM_LIBPACK)\include\SLEPc;$(WELSIM_LIBPACK)\include\fmt;$(WELSIM_LIBPACK)\include\palace;$(WELSIM_LIBPACK)\include\palace\libCEED;$(WELSIM_LIBPACK)\include;$(INTEL_MKL)\include;$(WELSIM_LIBPACK)\include\eigen3;$(WELSIM_LIBPACK)\include\mumps;$(WELSIM_LIBPACK)\include\palace\arpack;$(CUDA_PATH)\include;$(ExternalIncludePath)</ExternalIncludePath>
+    <OutDir>$(Platform)\$(Configuration)\</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <PostBuildEvent>
+      <Command>postbuild_libpalace_d</Command>
+    </PostBuildEvent>
+    <ClCompile>
+      <MultiProcessorCompilation>true</MultiProcessorCompilation>
+      <PreprocessorDefinitions>_UNICODE;UNICODE;CEED_SKIP_VISIBILITY;PALACE_WITH_ARPACK;_CRT_SECURE_NO_WARNINGS;_SILENCE_STDEXT_ARR_ITERS_DEPRECATION_WARNING;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup />
 </Project>

palace/main.cpp

@@ -74,10 +74,10 @@ static int GetDeviceId(MPI_Comm comm)
 #if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
   MPI_Comm node_comm;
   MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, Mpi::Rank(comm), MPI_INFO_NULL,
-                      node_comm);
+                      &node_comm);
   int node_size = Mpi::Rank(node_comm);
   MPI_Comm_free(&node_comm);
-  return node_size % mfem::Device::GetNgpu();
+  return node_size % mfem::Device::GetNumGPU();
 #else
   return 0;
 #endif
@@ -153,11 +153,11 @@ static void PrintPalaceInfo(MPI_Comm comm, int np, int nt, mfem::Device &device)
     Mpi::Print(comm, ", {:d} OpenMP thread{}", nt, (nt > 1) ? "s" : "");
   }
 #if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
-  int ngpu = mfem::Device::GetNgpu();
+  int ngpu = mfem::Device::GetNumGPU();
 #if defined(MFEM_USE_CUDA)
-  const char *device_name = "CUDA"
+  const char *device_name = "CUDA";
 #else
-  const char *device_name = "HIP"
+  const char *device_name = "HIP";
 #endif
       Mpi::Print(comm, "\n{:d} detected {} device{}{}", ngpu, device_name,
                  (ngpu > 1) ? "s" : "",

palace/models/laplaceoperator.cpp

@@ -111,6 +111,16 @@ std::map<int, mfem::Array<int>> ConstructSources(const IoData &iodata)
   return source_attr_lists;
 }
 
+std::map<int, double> ConstructBCValues(const IoData &iodata)
+{
+  std::map<int, double> values;
+  for (const auto &[idx, data] : iodata.boundaries.lumpedport)
+  {
+    values[idx] = data.voltage;
+  }
+  return values;
+}
+
 }  // namespace
 
 LaplaceOperator::LaplaceOperator(const IoData &iodata,
@@ -125,7 +135,7 @@ LaplaceOperator::LaplaceOperator(const IoData &iodata,
     h1_fespaces(fem::ConstructFiniteElementSpaceHierarchy<mfem::H1_FECollection>(
         iodata.solver.linear.mg_max_levels, mesh, h1_fecs, &dbc_marker, &dbc_tdof_lists)),
     nd_fespace(h1_fespaces.GetFinestFESpace(), mesh.back().get(), nd_fec.get()),
-    mat_op(iodata, *mesh.back()), source_attr_lists(ConstructSources(iodata))
+    mat_op(iodata, *mesh.back()), source_attr_lists(ConstructSources(iodata)), dbc_values(ConstructBCValues(iodata))
 {
   // Print essential BC information.
   if (dbc_marker.Size() && dbc_marker.Max() > 0)
@@ -194,8 +204,12 @@ void LaplaceOperator::GetExcitationVector(int idx, const Operator &K, Vector &X,
   mfem::Array<int> source_marker;
   const mfem::Array<int> &source_list = source_attr_lists[idx];
   mesh::AttrToMarker(dbc_marker.Size(), source_list, source_marker);
-  mfem::ConstantCoefficient one(1.0);
-  x.ProjectBdrCoefficient(one, source_marker);  // Values are only correct on master
+  mfem::ConstantCoefficient dbc_val(1.0);
+  std::map<int, double>::iterator it = dbc_values.find(idx);
+  if (it != dbc_values.end()) {
+    dbc_val = mfem::ConstantCoefficient(it->second);
+  }
+  x.ProjectBdrCoefficient(dbc_val, source_marker);  // Values are only correct on master
 
   // Eliminate the essential BC to get the RHS vector.
   X.SetSize(GetH1Space().GetTrueVSize());

palace/models/laplaceoperator.hpp

@@ -46,6 +46,8 @@ class LaplaceOperator
 
   // Boundary attributes for each terminal index.
   std::map<int, mfem::Array<int>> source_attr_lists;
+  std::map<int, double> dbc_values;
+
 
 public:
   LaplaceOperator(const IoData &iodata,
@@ -56,6 +58,7 @@ class LaplaceOperator
 
   // Access source attribute lists.
   const auto &GetSources() const { return source_attr_lists; }
+  const auto &GetBCValues() const { return dbc_values; }
 
   // Return the parallel finite element space objects.
   auto &GetH1Spaces() { return h1_fespaces; }