Skip to content

Commit

Permalink
Merge pull request #21 from PetrKryslUCSD/development
Browse files Browse the repository at this point in the history
Development
  • Loading branch information
PetrKryslUCSD authored Aug 19, 2024
2 parents 12d05c2 + 07ab73a commit a1e535f
Show file tree
Hide file tree
Showing 10 changed files with 460 additions and 45 deletions.
2 changes: 1 addition & 1 deletion Project.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name = "FinEtoolsDDMethods"
uuid = "093fdad3-4fce-4574-aa49-c2a24de4f00d"
authors = ["Petr Krysl <[email protected]>"]
version = "0.4.1"
version = "0.4.2"

[deps]
CoNCMOR = "3a989f29-cff0-4f1c-87be-5dcdd41bd240"
Expand Down
10 changes: 5 additions & 5 deletions examples/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -110,11 +110,11 @@ to see the available options.

## How to run a CoNC (MPI-parallel) example

At the moment only the heat conduction examples have been cast in this form. Try
At the moment only the heat conduction and shell analysis examples have been cast in this form. Try
```
mpiexec -n 5 julia --project=. .\conc\heat\Poisson2D_mpi_driver.jl
```

## To do

- MPI parallelization: When making the stiffness matrices of the partitions, for the shell it will be necessary to associate the geometry (normals!) with the entire geometry.
or
```
mpiexec -n 5 julia --project=. .\conc\shells\barrel_mpi_driver.jl
```
2 changes: 1 addition & 1 deletion examples/conc/heat/Poisson2D_mpi_driver.jl
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ using Statistics
using MPI

function peeksolution(iter, x, resnorm)
@info("Iteration: $(iter), Residual norm: $(resnorm)")
@info("Iteration $(iter): residual norm $(resnorm)")
end

function _execute(N, mesher, volrule, nelperpart, nbf1max, overlap, itmax, relrestol, visualize)
Expand Down
4 changes: 2 additions & 2 deletions examples/conc/pace.jl
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ include(raw"lindef/fibers_examples.jl")
fibers_examples.test()
include(raw"shells/barrel_examples.jl")
barrel_examples.test()
include(raw"shells/cos_2t_press_hyperboloid_free_examples.jl")
cos_2t_press_hyperboloid_free_examples.test()
include(raw"shells/cos_2t_p_hyp_free_examples.jl")
cos_2t_p_hyp_free_examples.test()
include(raw"shells/LE5_Z_cantilever_examples.jl")
LE5_Z_cantilever_examples.test()
229 changes: 229 additions & 0 deletions examples/conc/shells/LE5_Z_cantilever_mpi_driver.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,229 @@
"""
MODEL DESCRIPTION
Z-section cantilever under torsional loading.
Linear elastic analysis, Young's modulus = 210 GPa, Poisson's ratio = 0.3.
All displacements are fixed at X=0.
Torque of 1.2 MN-m applied at X=10. The torque is applied by two
uniformly distributed shear loads of 0.6 MN at each flange surface.
Objective of the analysis is to compute the axial stress at X = 2.5 from fixed end.
NAFEMS REFERENCE SOLUTION
Axial stress at X = 2.5 from fixed end (point A) at the midsurface is -108 MPa.
"""
module LE5_Z_cantilever_mpi_driver
using FinEtools
using FinEtools.MeshExportModule: VTK
using FinEtoolsDeforLinear
using FinEtoolsFlexStructures.FESetShellT3Module: FESetShellT3
using FinEtoolsFlexStructures.FESetShellQ4Module: FESetShellQ4
using FinEtoolsFlexStructures.FEMMShellT3FFModule
using FinEtoolsFlexStructures.RotUtilModule: initial_Rfield, update_rotation_field!
using MPI
using FinEtoolsDDMethods
using FinEtoolsDDMethods.CGModule: pcg_mpi_2level_Schwarz
using FinEtoolsDDMethods.PartitionCoNCDDMPIModule
using FinEtoolsDDMethods.CoNCUtilitiesModule: patch_coordinates
using SymRCM
using Metis
using Test
using LinearAlgebra
using SparseArrays
using PlotlyLight
using Krylov
using LinearOperators
import Base: size, eltype
import LinearAlgebra: mul!
import CoNCMOR: CoNCData, transfmatrix, LegendreBasis
using Targe2
using DataDrop
using Statistics
using ShellStructureTopo: create_partitions

# using MatrixSpy

function zcant!(csmatout, XYZ, tangents, feid, qpid)
r = vec(XYZ);
cross3!(r, view(tangents, :, 1), view(tangents, :, 2))
csmatout[:, 3] .= vec(r)/norm(vec(r))
csmatout[:, 1] .= (1.0, 0.0, 0.0)
cross3!(view(csmatout, :, 2), view(csmatout, :, 3), view(csmatout, :, 1))
return csmatout
end

# Parameters:
E = 210e9
nu = 0.3
L = 10.0
input = "nle5xf3c.inp"

function computetrac!(forceout, XYZ, tangents, feid, qpid)
r = vec(XYZ); r[2] = 0.0
r .= vec(r)/norm(vec(r))
theta = atan(r[3], r[1])
n = cross(tangents[:, 1], tangents[:, 2])
n = n/norm(n)
forceout[1:3] = n*pressure*cos(2*theta)
forceout[4:6] .= 0.0
# @show dot(n, forceout[1:3])
return forceout
end

function _execute(ncoarse, aspect, nelperpart, nbf1max, overlap, ref, itmax, relrestol, visualize)
CTE = 0.0
distortion = 0.0
n = ncoarse * ref # number of elements along the edge of the block
thickness = 0.1

MPI.Init()
comm = MPI.COMM_WORLD
rank = MPI.Comm_rank(comm)
nprocs = MPI.Comm_size(comm)

nfpartitions = nprocs - 1

rank == 0 && (@info "Number of processes: $nprocs")
rank == 0 && (@info "Number of partitions: $nfpartitions")

tolerance = thickness/1000
output = import_ABAQUS(joinpath(dirname(@__FILE__()), input))
fens = output["fens"]
fes = output["fesets"][1]

connected = findunconnnodes(fens, fes);
fens, new_numbering = compactnodes(fens, connected);
fes = renumberconn!(fes, new_numbering);

for r in 1:ref
fens, fes = T3refine(fens, fes)
end

MR = DeforModelRed3D
mater = MatDeforElastIso(MR, 0.0, E, nu, CTE)
ocsys = CSys(3, 3, zcant!)

sfes = FESetShellT3()
accepttodelegate(fes, sfes)
femm = FEMMShellT3FFModule.make(IntegDomain(fes, TriRule(1), thickness), mater)
stiffness = FEMMShellT3FFModule.stiffness
associategeometry! = FEMMShellT3FFModule.associategeometry!

# Construct the requisite fields, geometry and displacement
# Initialize configuration variables
geom0 = NodalField(fens.xyz)
u0 = NodalField(zeros(size(fens.xyz,1), 3))
Rfield0 = initial_Rfield(fens)
dchi = NodalField(zeros(size(fens.xyz,1), 6))

# Apply EBC's
# plane of symmetry perpendicular to Z
l1 = selectnode(fens; box = Float64[0 0 -Inf Inf -Inf Inf], inflate = tolerance)
for i in [1,2,3,]
setebc!(dchi, l1, true, i)
end
applyebc!(dchi)
numberdofs!(dchi);

fr = dofrange(dchi, DOF_KIND_FREE)
dr = dofrange(dchi, DOF_KIND_DATA)

rank == 0 && (@info("Refinement factor: $(ref)"))
rank == 0 && (@info("Number of elements per partition: $(nelperpart)"))
rank == 0 && (@info("Number of 1D basis functions: $(nbf1max)"))
rank == 0 && (@info("Number of fine grid partitions: $(nfpartitions)"))
rank == 0 && (@info("Overlap: $(overlap)"))
rank == 0 && (@info("Number of elements: $(count(fes))"))
rank == 0 && (@info("Number of free dofs = $(nfreedofs(dchi))"))

nl = selectnode(fens; box = Float64[10.0 10.0 1.0 1.0 0 0], tolerance = tolerance)
loadbdry1 = FESetP1(reshape(nl, 1, 1))
lfemm1 = FEMMBase(IntegDomain(loadbdry1, PointRule()))
fi1 = ForceIntensity(Float64[0, 0, +0.6e6, 0, 0, 0]);
nl = selectnode(fens; box = Float64[10.0 10.0 -1.0 -1.0 0 0], tolerance = tolerance)
loadbdry2 = FESetP1(reshape(nl, 1, 1))
lfemm2 = FEMMBase(IntegDomain(loadbdry2, PointRule()))
fi2 = ForceIntensity(Float64[0, 0, -0.6e6, 0, 0, 0]);
F = distribloads(lfemm1, geom0, dchi, fi1, 3) + distribloads(lfemm2, geom0, dchi, fi2, 3);
F_f = F[fr]

associategeometry!(femm, geom0)

cpartitioning, ncpartitions = shell_cluster_partitioning(fens, fes, nelperpart)
rank == 0 && (@info("Number of clusters (coarse grid partitions): $(ncpartitions)"))

mor = CoNCData(list -> patch_coordinates(fens.xyz, list), cpartitioning)
Phi = transfmatrix(mor, LegendreBasis, nbf1max, dchi)
Phi = Phi[fr, :]
rank == 0 && (@info("Size of the reduced problem: $(size(Phi, 2))"))


function make_matrix(fes)
femm.integdomain.fes = fes
return stiffness(femm, geom0, u0, Rfield0, dchi);
end

t1 = time()
partition = nothing
if rank > 0
cpi = CoNCPartitioningInfo(fens, fes, nfpartitions, overlap, dchi)
partition = CoNCPartitionData(cpi, rank, fes, Phi, make_matrix, nothing)
end
MPI.Barrier(comm)
rank == 0 && (@info "Create partitions time: $(time() - t1)")

function peeksolution(iter, x, resnorm)
@info("Iteration $(iter): residual norm $(resnorm)")
end

t1 = time()
Kr_ff = spzeros(size(Phi, 2), size(Phi, 2))
if rank > 0
Kr_ff = partition.reduced_K
end
ks = MPI.gather(Kr_ff, comm; root=0)
if rank == 0
for k in ks
Kr_ff += k
end
Krfactor = lu(Kr_ff)
end
rank == 0 && (@info "Create global factor: $(time() - t1)")

t1 = time()
norm_F_f = norm(F_f)
(u_f, stats) = pcg_mpi_2level_Schwarz(
comm,
rank,
(q, p) -> partition_multiply!(q, partition, p),
F_f,
zeros(size(F_f)),
(q, p) -> precondition_global_solve!(q, Krfactor, Phi, p),
(q, p) -> precondition_local_solve!(q, partition, p);
itmax=itmax, atol=relrestol * norm_F_f, rtol=0,
peeksolution=peeksolution)

rank == 0 && @info("Number of iterations: $(stats.niter)")
stats = (niter=stats.niter, resnorm=stats.resnorm ./ norm_F_f)
scattersysvec!(dchi, u_f, DOF_KIND_FREE)
rank == 0 && @info("Solution ($(round(time() - t1, digits=3)) [s])")

MPI.Finalize()

true
end

function test(;aspect = 100, nelperpart = 200, nbf1max = 2, overlap = 1, ref = 1, itmax = 2000, relrestol = 1e-6, visualize = false)
_execute(32, aspect, nelperpart, nbf1max, overlap, ref, itmax, relrestol, visualize)
end

test()

nothing
end # module


33 changes: 7 additions & 26 deletions examples/conc/shells/barrel_mpi_driver.jl
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ using FinEtoolsFlexStructures.FESetShellT3Module: FESetShellT3
using FinEtoolsFlexStructures.FESetShellQ4Module: FESetShellQ4
using FinEtoolsFlexStructures.FEMMShellT3FFModule
using FinEtoolsFlexStructures.RotUtilModule: initial_Rfield, update_rotation_field!
using MPI
using FinEtoolsDDMethods
using FinEtoolsDDMethods.CGModule: pcg_mpi_2level_Schwarz
using FinEtoolsDDMethods.PartitionCoNCDDMPIModule
Expand All @@ -35,7 +36,7 @@ using DataDrop
using ILUZero
using Statistics
using ShellStructureTopo: make_topo_faces, create_partitions
using MPI


# using MatrixSpy

Expand Down Expand Up @@ -192,11 +193,10 @@ function _execute(ncoarse, nelperpart, nbf1max, nfpartitions, overlap, ref, itma
MPI.Barrier(comm)
rank == 0 && (@info "Create partitions time: $(time() - t1)")

peeksolution(iter, x, resnorm) = begin
rank == 0 && (@info("Iteration: $(iter)"))
rank == 0 && (@info("Residual Norm: $(resnorm)"))
function peeksolution(iter, x, resnorm)
@info("Iteration $(iter): residual norm $(resnorm)")
end

t1 = time()
Kr_ff = spzeros(size(Phi, 2), size(Phi, 2))
if rank > 0
Expand Down Expand Up @@ -229,27 +229,8 @@ function _execute(ncoarse, nelperpart, nbf1max, nfpartitions, overlap, ref, itma
scattersysvec!(dchi, u_f, DOF_KIND_FREE)
rank == 0 && @info("Solution ($(round(time() - t1, digits=3)) [s])")

if visualize
# f = "barrel" *
# "-rf=$(ref)" *
# "-ne=$(nelperpart)" *
# "-n1=$(nbf1max)" *
# "-nf=$(nfpartitions)" *
# "-cg-sol"
# VTK.vtkexportmesh(f * ".vtk", fens, fes; vectors=[("u", deepcopy(u.values),)])
VTK.vtkexportmesh("barrel-sol.vtk", fens, fes;
vectors=[("u", deepcopy(dchi.values[:, 1:3]),)])

p = 1
for nodelist in nodelists
cel = connectedelems(fes, nodelist, count(fens))
vtkexportmesh("barrel-patch$(p).vtk", fens, subset(fes, cel))
sfes = FESetP1(reshape(nodelist, length(nodelist), 1))
vtkexportmesh("barrel-nodes$(p).vtk", fens, sfes)
p += 1
end
end

MPI.Finalize()

true
end

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
module cos_2t_press_hyperboloid_free_examples
module cos_2t_p_hyp_free_examples
using FinEtools
using FinEtools.MeshExportModule: VTK
using FinEtoolsDeforLinear
Expand Down Expand Up @@ -129,7 +129,7 @@ function _execute(ncoarse, aspect, nelperpart, nbf1max, nfpartitions, overlap, r

if visualize
partitionsfes = FESetP1(reshape(1:count(fens), count(fens), 1))
vtkexportmesh("cos_2t_press_hyperboloid_free-clusters.vtk", fens, partitionsfes; scalars=[("partition", cpartitioning)])
vtkexportmesh("cos_2t_p_hyp_free-clusters.vtk", fens, partitionsfes; scalars=[("partition", cpartitioning)])
end

mor = CoNCData(list -> patch_coordinates(fens.xyz, list), cpartitioning)
Expand Down Expand Up @@ -197,7 +197,7 @@ function _execute(ncoarse, aspect, nelperpart, nbf1max, nfpartitions, overlap, r
"stats" => stats,
"time" => t1 - t0,
)
f = "cos_2t_press_hyperboloid_free" *
f = "cos_2t_p_hyp_free" *
"-as=$(aspect)" *
"-rf=$(ref)" *
"-ne=$(nelperpart)" *
Expand All @@ -208,15 +208,15 @@ function _execute(ncoarse, aspect, nelperpart, nbf1max, nfpartitions, overlap, r
scattersysvec!(dchi, u_f)

if visualize
VTK.vtkexportmesh("cos_2t_press_hyperboloid_free-sol.vtk", fens, fes;
VTK.vtkexportmesh("cos_2t_p_hyp_free-sol.vtk", fens, fes;
vectors=[("u", deepcopy(dchi.values[:, 1:3]),)])

p = 1
for nodelist in nodelists
cel = connectedelems(fes, nodelist, count(fens))
vtkexportmesh("cos_2t_press_hyperboloid_free-subdomain-patch$(p).vtk", fens, subset(fes, cel))
vtkexportmesh("cos_2t_p_hyp_free-subdomain-patch$(p).vtk", fens, subset(fes, cel))
sfes = FESetP1(reshape(nodelist, length(nodelist), 1))
vtkexportmesh("cos_2t_press_hyperboloid_free-subdomain-nodes$(p).vtk", fens, sfes)
vtkexportmesh("cos_2t_p_hyp_free-subdomain-nodes$(p).vtk", fens, sfes)
p += 1
end
end
Expand Down
Loading

0 comments on commit a1e535f

Please sign in to comment.