From 5b85d1089c8d185fe2ae10cb7d9cb665a538d859 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 15 Jul 2024 18:03:39 -0700 Subject: [PATCH] fix: remove inlining of sigmoid_fast pre 1.11- --- src/activations.jl | 5 ++++- test/ext_cuda/activations.jl | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/activations.jl b/src/activations.jl index 43e1c7075..2df90cd2e 100644 --- a/src/activations.jl +++ b/src/activations.jl @@ -823,7 +823,10 @@ julia> hardσ(0.2f0) 0.53333336f0 ``` """ -@inline function sigmoid_fast(x::Real) +function sigmoid_fast(x::Real) + @static if VERSION ≥ v"1.11-" + @inline + end t = @fastmath exp(-abs(x)) y = ifelse(x ≥ 0, inv(1 + t), t / (1 + t)) ifelse(x > 40, one(y), ifelse(x < -80, zero(y), y)) diff --git a/test/ext_cuda/activations.jl b/test/ext_cuda/activations.jl index fb9d2ebfc..9d15b1fc5 100644 --- a/test/ext_cuda/activations.jl +++ b/test/ext_cuda/activations.jl @@ -41,3 +41,13 @@ end @test Array(y) == [tanh(1f0)] @test Array(x) == [tanh(tanh(1f0))] end + +@testset "fused act addition broadcast" begin + x = CUDA.rand(Float32, 10, 10) + b = CUDA.rand(Float32, 10) + + for act in getfield.((NNlib,), NNlib.ACTIVATIONS) + fused_act_add = act ∘ + + @test fused_act_add.(x, b) ≈ act.(x .+ b) + end +end