-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
14 changed files
with
228 additions
and
149 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,16 +1,15 @@ | ||
module AMDGPUExt | ||
|
||
using AMDGPU | ||
using KernelAbstractions | ||
|
||
using FastIce.Architecture | ||
import FastIce.Architecture: heuristic_groupsize, set_device! | ||
|
||
set_device!(dev::HIPDevice) = AMDGPU.device!(dev) | ||
|
||
set_device!(::ROCBackend, id::Integer) = AMDGPU.device_id!(id) | ||
set_device!(::HIPDevice, id::Integer) = AMDGPU.device_id!(id) | ||
|
||
heuristic_groupsize(::ROCBackend, ::Val{1}) = (256, ) | ||
heuristic_groupsize(::ROCBackend, ::Val{2}) = (128, 2, ) | ||
heuristic_groupsize(::ROCBackend, ::Val{3}) = (128, 2, 1, ) | ||
heuristic_groupsize(::HIPDevice, ::Val{1}) = (256, ) | ||
heuristic_groupsize(::HIPDevice, ::Val{2}) = (128, 2, ) | ||
heuristic_groupsize(::HIPDevice, ::Val{3}) = (128, 2, 1, ) | ||
|
||
end | ||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,16 +1,15 @@ | ||
module CUDAExt | ||
|
||
using CUDA | ||
using KernelAbstractions | ||
|
||
using FastIce.Architecture | ||
import FastIce.Architecture: heuristic_groupsize, set_device! | ||
|
||
set_device!(dev::CuDevice) = CUDA.device!(dev) | ||
|
||
set_device!(::CUDABackend, id::Integer) = CUDA.device!(id-1) | ||
set_device!(::CuDevice, id::Integer) = CUDA.device!(id-1) | ||
|
||
heuristic_groupsize(::CUDABackend, ::Val{1}) = (256, ) | ||
heuristic_groupsize(::CUDABackend, ::Val{2}) = (32, 8, ) | ||
heuristic_groupsize(::CUDABackend, ::Val{3}) = (32, 8, 1, ) | ||
heuristic_groupsize(::CuDevice, ::Val{1}) = (256, ) | ||
heuristic_groupsize(::CuDevice, ::Val{2}) = (32, 8, ) | ||
heuristic_groupsize(::CuDevice, ::Val{3}) = (32, 8, 1, ) | ||
|
||
end | ||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
using FastIce.Architectures | ||
using FastIce.Distributed | ||
using FastIce.Fields | ||
using FastIce.Grids | ||
using FastIce.BoundaryConditions | ||
using FastIce.KernelLaunch | ||
|
||
using KernelAbstractions | ||
using MPI | ||
|
||
@kernel function fill_field!(f, val, offset=nothing) | ||
I = @index(Global, Cartesian) | ||
if !isnothing(offset) | ||
I += offset | ||
end | ||
f[I] = val | ||
end | ||
|
||
MPI.Init() | ||
|
||
arch = Architecture(CPU(), (0, 0)) | ||
grid = CartesianGrid(; origin=(0.0, 0.0), extent=(1.0, 1.0), size=(10, 10)) | ||
field = Field(backend(arch), grid, (Center(), Center()); halo=1) | ||
|
||
me = global_rank(details(arch)) | ||
|
||
bc = FieldBoundaryConditions((field,), (DirichletBC{HalfCell}(me),)) | ||
|
||
boundary_conditions = ((bc, bc), | ||
(bc, bc)) | ||
|
||
boundary_conditions = ntuple(Val(length(boundary_conditions))) do D | ||
ntuple(Val(2)) do S | ||
if neighbor(details(arch), D, S) != MPI.PROC_NULL | ||
DistributedBoundaryConditions(Val(S), Val(D), (field, )) | ||
else | ||
boundary_conditions[D][S] | ||
end | ||
end | ||
end | ||
|
||
hide_boundaries = HideBoundaries{2}(arch) | ||
|
||
outer_width = (4, 4) | ||
|
||
launch!(arch, grid, fill_field! => (field, me); location=Center(), hide_boundaries, boundary_conditions, outer_width) | ||
|
||
sleep(me) | ||
@show coordinates(details(arch)) | ||
display(parent(field)) | ||
|
||
MPI.Finalize() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,69 +1,46 @@ | ||
module Architectures | ||
|
||
export AbstractArchitecture | ||
|
||
export SingleDeviceArchitecture | ||
export Architecture | ||
|
||
export launch!, set_device!, set_device_and_priority!, heuristic_groupsize | ||
export synchronize | ||
export synchronize, backend, device, details | ||
|
||
using FastIce.Grids | ||
|
||
using KernelAbstractions | ||
import KernelAbstractions.Kernel | ||
|
||
abstract type AbstractArchitecture end | ||
|
||
device(::AbstractArchitecture) = error("device function must be defined for architecture") | ||
backend(::AbstractArchitecture) = error("backend function must be defined for architecture") | ||
|
||
set_device!(arch::AbstractArchitecture) = set_device!(device(arch)) | ||
|
||
synchronize(arch::AbstractArchitecture) = KernelAbstractions.synchronize(backend(arch)) | ||
|
||
function set_device_and_priority!(arch::AbstractArchitecture, prio::Symbol) | ||
set_device!(arch) | ||
KernelAbstractions.priority!(backend(arch), prio) | ||
return | ||
end | ||
|
||
heuristic_groupsize(arch::AbstractArchitecture) = heuristic_groupsize(device(arch)) | ||
|
||
struct SingleDeviceArchitecture{B,D} <: AbstractArchitecture | ||
struct Architecture{Kind,B,D,Details} | ||
backend::B | ||
device::D | ||
details::Details | ||
end | ||
|
||
function SingleDeviceArchitecture(backend::Backend) | ||
device = set_device!(backend, 1) | ||
return SingleDeviceArchitecture(backend, device) | ||
end | ||
|
||
set_device!(::SingleDeviceArchitecture{CPU}) = nothing | ||
set_device!(::CPU, id::Integer) = nothing | ||
|
||
heuristic_groupsize(::SingleDeviceArchitecture{CPU}) = 256 | ||
|
||
device(arch::SingleDeviceArchitecture) = arch.device | ||
struct SingleDevice end | ||
|
||
backend(arch::SingleDeviceArchitecture) = arch.backend | ||
|
||
function launch!(arch::SingleDeviceArchitecture, grid::CartesianGrid, kernel; kwargs...) | ||
worksize = size(grid, Vertex()) | ||
launch!(arch, worksize, kernel; kwargs...) | ||
function Architecture(backend::Backend, device_id::Integer=1) | ||
device = set_device!(backend, device_id) | ||
return Architecture{SingleDevice,typeof(backend),typeof(device),Nothing}(backend, device, nothing) | ||
end | ||
|
||
function launch!(arch::SingleDeviceArchitecture, worksize::NTuple{N,Int}, kernel::Pair{Kernel,Args}; | ||
boundary_conditions=nothing, async=true) where {N,Args} | ||
fun, args = kernel | ||
device(arch::Architecture) = arch.device | ||
backend(arch::Architecture) = arch.backend | ||
details(arch::Architecture) = arch.details | ||
|
||
groupsize = heuristic_groupsize(arch) | ||
synchronize(arch::Architecture) = KernelAbstractions.synchronize(arch.backend) | ||
|
||
fun(arch.backend, groupsize, worksize)(args...) | ||
isnothing(boundary_conditions) || apply_boundary_conditions!(boundary_conditions) | ||
set_device!(arch::Architecture) = set_device!(arch.device) | ||
|
||
async || synchronize(arch.backend) | ||
function set_device_and_priority!(arch::Architecture, prio::Symbol) | ||
set_device!(arch) | ||
KernelAbstractions.priority!(arch.backend, prio) | ||
return | ||
end | ||
|
||
set_device!(::Architecture{Kind,CPU}) where {Kind} = nothing | ||
set_device!(::CPU, id::Integer) = nothing | ||
|
||
heuristic_groupsize(arch::Architecture, ::Val{N}) where {N} = heuristic_groupsize(arch.device, Val(N)) | ||
heuristic_groupsize(::Architecture{Kind,CPU}, N) where {Kind} = 256 | ||
|
||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
struct HideBoundaries{N} | ||
pipelines::NTuple{N,Tuple{Pipeline,Pipeline}} | ||
function HideBoundaries{N}(arch::Architecture) where {N} | ||
pre() = set_device_and_priority!(arch, :high) | ||
pipelines = ntuple(Val(N)) do _ | ||
return ntuple(_ -> Pipeline(; pre), Val(2)) | ||
end | ||
return new{N}(pipelines) | ||
end | ||
end | ||
|
||
function hide(fun::F, hb::HideBoundaries{N}, arch::Architecture, grid::CartesianGrid{N}, boundary_conditions, worksize; | ||
outer_width=nothing) where {F,N} | ||
inner_range, outer_ranges = split_ndrange(worksize, outer_width) | ||
fun(inner_range) | ||
for dim in N:-1:1 | ||
ntuple(Val(2)) do side | ||
pipe = hb.pipelines[dim][side] | ||
range = outer_ranges[dim][side] | ||
bc = boundary_conditions[dim][side] | ||
# execute outer range and boundary conditions asynchronously | ||
put!(pipe) do | ||
fun(range) | ||
apply_boundary_conditions!(Val(side), Val(dim), arch, grid, bc) | ||
Architectures.synchronize(arch) | ||
end | ||
end | ||
wait.(hb.pipelines[dim]) # synchronize spatial dimension | ||
end | ||
return | ||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.