Added GELU activation. (#52)

lib/torch.rb

@@ -124,6 +124,7 @@
 
 # nn activations
 require_relative "torch/nn/elu"
+require_relative "torch/nn/gelu"
 require_relative "torch/nn/hardshrink"
 require_relative "torch/nn/leaky_relu"
 require_relative "torch/nn/log_sigmoid"

lib/torch/nn/functional.rb

@@ -182,6 +182,10 @@ def elu(input, alpha: 1, inplace: false)
           end
         end
 
+        def gelu(input, approximate: 'none')
+          NN.gelu(input, approximate: approximate)
+        end
+
         def hardshrink(input, lambd = 0.5)
           Torch.hardshrink(input, lambd)
         end

lib/torch/nn/gelu.rb

@@ -0,0 +1,18 @@
+module Torch
+  module NN
+    class GELU < Module
+      def initialize(approximate: 'none')
+        super()
+        @approximate = approximate
+      end
+
+      def forward(input)
+        F.gelu(input, approximate: @approximate)
+      end
+
+      def extra_inspect
+        "approximate: '%{@approximate}'"
+      end
+    end
+  end
+end

test/nn/activations_test.rb

@@ -72,4 +72,10 @@ def test_tanhshrink
     input = Torch.randn(2)
     _output = m.call(input)
   end
+
+  def test_gelu
+    m = Torch::NN::GELU.new
+    input = Torch.randn(2)
+    _output = m.call(input)
+  end
 end