diff --git a/nada_numpy/array.py b/nada_numpy/array.py index 436e4c0..623c6ba 100644 --- a/nada_numpy/array.py +++ b/nada_numpy/array.py @@ -16,8 +16,8 @@ from nada_numpy.nada_typing import (NadaBoolean, NadaCleartextType, NadaInteger, NadaRational, NadaUnsignedInteger) -from nada_numpy.types import (Rational, SecretRational, get_log_scale, - public_rational, rational, secret_rational) +from nada_numpy.types import (Rational, SecretRational, fxp_abs, get_log_scale, + public_rational, rational, secret_rational, sign) from nada_numpy.utils import copy_metadata @@ -1078,6 +1078,666 @@ def T(self): # pylint:disable=invalid-name return NadaArray(result) return result + # Non-linear functions + + def sign(self) -> "NadaArray": + """Computes the sign value (0 is considered positive)""" + if self.is_rational: + return self.apply(sign) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Sign is not compatible with {dtype}, only with Rational and SecretRational types." + ) + + def abs(self) -> "NadaArray": + """Computes the absolute value""" + if self.is_rational: + return self.apply(fxp_abs) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Abs is not compatible with {dtype}, only with Rational and SecretRational types." + ) + + def exp(self, iterations: int = 8) -> "NadaArray": + """ + Approximates the exponential function using a limit approximation. + + The exponential function is approximated using the following limit: + + exp(x) = lim_{n -> ∞} (1 + x / n) ^ n + + The exponential function is computed by choosing n = 2 ** d, where d is set to `iterations`. + The calculation is performed by computing (1 + x / n) once and then squaring it `d` times. + + Approximation accuracy range (with 16 bit precision): + + ---------------------------------- + + | Input range x | Relative error | + + ---------------------------------- + + | [-2, 2] | <1% | + | [-7, 7] | <10% | + | [-8, 15] | <35% | + + ---------------------------------- + + + Args: + iterations (int, optional): The number of iterations for the limit approximation. + Defaults to 8. + + Returns: + NadaArray: The approximated value of the exponential function. + """ + if self.is_rational: + + def exp(x): + return x.exp(iterations=iterations) + + return self.apply(exp) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Exp is not compatible with {dtype}, only with Rational and SecretRational types." + ) + + def polynomial(self, coefficients: list) -> "NadaArray": + """ + Computes a polynomial function on a value with given coefficients. + + The coefficients can be provided as a list of values. + They should be ordered from the linear term (order 1) first, + ending with the highest order term. + **Note: The constant term is not included.** + + Args: + coefficients (list): The coefficients of the polynomial, ordered by increasing degree. + + Returns: + NadaArray: The result of the polynomial function applied to the input x. + """ + if self.is_rational: + + def polynomial(x): + return x.polynomial(coefficients=coefficients) + + return self.apply(polynomial) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Polynomial is not compatible with {dtype},\ + only with Rational and SecretRational types." + ) + + def log( + self, + input_in_01: bool = False, + iterations: int = 2, + exp_iterations: int = 8, + order: int = 8, + ) -> "NadaArray": + """ + Approximates the natural logarithm using 8th order modified Householder iterations. + This approximation is accurate within 2% relative error on the interval [0.0001, 250]. + + The iterations are computed as follows: + + h = 1 - x * exp(-y_n) + y_{n+1} = y_n - sum(h^k / k for k in range(1, order + 1)) + + Approximation accuracy range (with 16 bit precision): + + ------------------------------------- + + | Input range x | Relative error | + + ------------------------------------- + + | [0.001, 200] | <1% | + | [0.00003, 253] | <10% | + | [0.0000001, 253] | <40% | + | [253, +∞[ | Unstable | + + ------------------------------------- + + + Args: + input_in_01 (bool, optional): Indicates if the input is within the domain [0, 1]. + This setting optimizes the function for this domain, useful for computing + log-probabilities in entropy functions. + + To shift the domain of convergence, a constant 'a' is used with the identity: + + ln(u) = ln(au) - ln(a) + + Given the convergence domain for log() function is approximately [1e-4, 1e2], + we set a = 100. + Defaults to False. + iterations (int, optional): Number of Householder iterations for the approximation. + Defaults to 2. + exp_iterations (int, optional): Number of iterations for the limit + approximation of exp. Defaults to 8. + order (int, optional): Number of polynomial terms used (order of + Householder approximation). Defaults to 8. + + Returns: + NadaArray: The approximate value of the natural logarithm. + """ + if self.is_rational: + + def log(x): + return x.log( + input_in_01=input_in_01, + iterations=iterations, + exp_iterations=exp_iterations, + order=order, + ) + + return self.apply(log) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Log is not compatible with {dtype}, only with Rational and SecretRational types." + ) + + def reciprocal( # pylint: disable=too-many-arguments + self, + all_pos: bool = False, + initial: Optional["Rational"] = None, + input_in_01: bool = False, + iterations: int = 10, + log_iters: int = 1, + exp_iters: int = 8, + method: str = "NR", + ) -> "NadaArray": + r""" + Approximates the reciprocal of a number through two possible methods: Newton-Raphson + and log. + + Methods: + 'NR' : `Newton-Raphson`_ method computes the reciprocal using iterations + of :math:`x_{i+1} = (2x_i - x * x_i^2)` and uses + :math:`3*exp(1 - 2x) + 0.003` as an initial guess by default. + + Approximation accuracy range (with 16 bit precision): + + ------------------------------------ + + | Input range |x| | Relative error | + + ------------------------------------ + + | [0.1, 64] | <0% | + | [0.0003, 253] | <15% | + | [0.00001, 253] | <90% | + | [253, +∞[ | Unstable | + + ------------------------------------ + + + 'log' : Computes the reciprocal of the input from the observation that: + :math:`x^{-1} = exp(-log(x))` + + Approximation accuracy range (with 16 bit precision): + + ------------------------------------ + + | Input range |x| | Relative error | + + ------------------------------------ + + | [0.0003, 253] | <15% | + | [0.00001, 253] | <90% | + | [253, +∞[ | Unstable | + + ------------------------------------ + + + Args: + all_pos (bool, optional): determines whether all elements of the + input are known to be positive, which optimizes the step of + computing the sign of the input. Defaults to False. + initial (Rational, optional): sets the initial value for the + Newton-Raphson method. By default, this will be set to :math: + `3*exp(-(x-.5)) + 0.003` as this allows the method to converge over + a fairly large domain. + input_in_01 (bool, optional) : Allows a user to indicate that the input is + in the range [0, 1], causing the function optimize for this range. + This is useful for improving the accuracy of functions on + probabilities (e.g. entropy functions). + iterations (int, optional): determines the number of Newton-Raphson iterations to run + for the `NR` method. Defaults to 10. + log_iters (int, optional): determines the number of Householder + iterations to run when computing logarithms for the `log` method. Defaults to 1. + exp_iters (int, optional): determines the number of exp + iterations to run when computing exp. Defaults to 8. + method (str, optional): method used to compute reciprocal. Defaults to "NR". + + Returns: + NadaArray: The approximate value of the reciprocal + + .. _Newton-Raphson: + https://en.wikipedia.org/wiki/Newton%27s_method + """ + if self.is_rational: + # pylint:disable=duplicate-code + def reciprocal(x): + return x.reciprocal( + all_pos=all_pos, + initial=initial, + input_in_01=input_in_01, + iterations=iterations, + log_iters=log_iters, + exp_iters=exp_iters, + method=method, + ) + + return self.apply(reciprocal) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Reciprocal is not compatible with {dtype},\ + only with Rational and SecretRational types." + ) + + def inv_sqrt( + self, + initial: Optional["SecretRational"] = None, + iterations: int = 5, + method: str = "NR", + ) -> "NadaArray": + r""" + Computes the inverse square root of the input using the Newton-Raphson method. + + Approximation accuracy range (with 16 bit precision): + + ---------------------------------- + + | Input range x | Relative error | + + ---------------------------------- + + | [0.1, 170] | <0% | + | [0.001, 200] | <50% | + | [200, +∞[ | Unstable | + + ---------------------------------- + + + Args: + initial (Union[SecretRational, None], optional): sets the initial value for the + Newton-Raphson iterations. By default, this will be set to allow the + method to converge over a fairly large domain. + iterations (int, optional): determines the number of Newton-Raphson iterations to run. + method (str, optional): method used to compute inv_sqrt. Defaults to "NR". + + Returns: + NadaArray: The approximate value of the inv_sqrt. + + .. _Newton-Raphson: + https://en.wikipedia.org/wiki/Fast_inverse_square_root#Newton's_method + """ + if self.is_rational: + + def inv_sqrt(x): + return x.inv_sqrt(initial=initial, iterations=iterations, method=method) + + return self.apply(inv_sqrt) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Inverse square-root is not compatible with {dtype},\ + only with Rational and SecretRational types." + ) + + def sqrt( + self, + initial: Optional["SecretRational"] = None, + iterations: int = 5, + method: str = "NR", + ) -> "NadaArray": + r""" + Computes the square root of the input by computing its inverse square root using + the Newton-Raphson method and multiplying by the input. + + Approximation accuracy range (with 16 bit precision): + + ---------------------------------- + + | Input range x | Relative error | + + ---------------------------------- + + | [0.1, 170] | <0% | + | [0.001, 200] | <50% | + | [200, +∞[ | Unstable | + + ---------------------------------- + + + Args: + initial (Union[SecretRational, None], optional): sets the initial value for the inverse + square root Newton-Raphson iterations. By default, this will be set to allow + convergence over a fairly large domain. Defaults to None. + iterations (int, optional): determines the number of Newton-Raphson iterations to run. + Defaults to 5. + method (str, optional): method used to compute sqrt. Defaults to "NR". + + Returns: + NadaArray: The approximate value of the sqrt. + + .. _Newton-Raphson: + https://en.wikipedia.org/wiki/Fast_inverse_square_root#Newton's_method + """ + if self.is_rational: + + def sqrt(x): + return x.sqrt(initial=initial, iterations=iterations, method=method) + + return self.apply(sqrt) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Square-root is not compatible with {dtype},\ + only with Rational and SecretRational types." + ) + + # Trigonometry + + def cossin(self, iterations: int = 10) -> "NadaArray": + r"""Computes cosine and sine through e^(i * input) where i is the imaginary unit through the + formula: + + .. math:: + Re\{e^{i * input}\}, Im\{e^{i * input}\} = \cos(input), \sin(input) + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + Tuple[NadaArray, NadaArray]: + A tuple where the first element is cos and the second element is the sin. + """ + if self.is_rational: + + def cossin(x): + return x.cossin(iterations=iterations) + + return self.apply(cossin) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Cosine and Sine are not compatible with {dtype},\ + only with Rational and SecretRational types." + ) + + def cos(self, iterations: int = 10) -> "NadaArray": + r"""Computes the cosine of the input using cos(x) = Re{exp(i * x)}. + + Note: unstable outside [-30, 30] + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + NadaArray: The approximate value of the cosine. + """ + if self.is_rational: + + def cos(x): + return x.cos(iterations=iterations) + + return self.apply(cos) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Cosine is not compatible with {dtype},\ + only with Rational and SecretRational types." + ) + + def sin(self, iterations: int = 10) -> "NadaArray": + r"""Computes the sine of the input using sin(x) = Im{exp(i * x)}. + + Note: unstable outside [-30, 30] + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + NadaArray: The approximate value of the sine. + """ + if self.is_rational: + + def sin(x): + return x.sin(iterations=iterations) + + return self.apply(sin) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Sine is not compatible with {dtype}, only with Rational and SecretRational types." + ) + + def tan(self, iterations: int = 10) -> "NadaArray": + r"""Computes the tan of the input using tan(x) = sin(x) / cos(x). + + Note: unstable outside [-30, 30] + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + NadaArray: The approximate value of the tan. + """ + if self.is_rational: + + def tan(x): + return x.tan(iterations=iterations) + + return self.apply(tan) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Tangent is not compatible with {dtype},\ + only with Rational and SecretRational types." + ) + + # Activation functions + + def tanh( + self, chebyshev_terms: int = 32, method: str = "reciprocal" + ) -> "NadaArray": + r"""Computes the hyperbolic tangent function using the identity + + .. math:: + tanh(x) = 2\sigma(2x) - 1 + + Methods: + If a valid method is given, this function will compute tanh using that method: + + "reciprocal" - computes tanh using the identity + + .. math:: + tanh(x) = 2\sigma(2x) - 1 + + Note: stable for x in [-250, 250]. Unstable otherwise. + + "chebyshev" - computes tanh via Chebyshev approximation with truncation. + + .. math:: + tanh(x) = \sum_{j=1}^chebyshev_terms c_{2j - 1} P_{2j - 1} (x / maxval) + + where c_i is the ith Chebyshev series coefficient and P_i is ith polynomial. + + Note: stable for all input range as the approximation is truncated + to +/-1 outside [-1, 1]. + + "motzkin" - computes tanh via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.3 based on the Motzkin’s polynomial preprocessing technique. + + Note: stable for all input range as the approximation is truncated + to +/-1 outside [-1, 1]. + + Args: + chebyshev_terms (int, optional): highest degree of Chebyshev polynomials. + Must be even and at least 6. Defaults to 32. + method (str, optional): method used to compute tanh function. Defaults to "reciprocal". + + Returns: + NadaArray: The tanh evaluation. + + Raises: + ValueError: Raised if method type is not supported. + """ + if self.is_rational: + + def tanh(x): + return x.tanh(chebyshev_terms=chebyshev_terms, method=method) + + return self.apply(tanh) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Hyperbolic tangent is not compatible with {dtype},\ + only with Rational and SecretRational types." + ) + + def sigmoid( + self, chebyshev_terms: int = 32, method: str = "reciprocal" + ) -> "NadaArray": + r"""Computes the sigmoid function using the following definition + + .. math:: + \sigma(x) = (1 + e^{-x})^{-1} + + Methods: + If a valid method is given, this function will compute sigmoid + using that method: + + "chebyshev" - computes tanh via Chebyshev approximation with + truncation and uses the identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "motzkin" - computes tanh via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.3 based on the Motzkin’s polynomial preprocessing technique. It uses + the identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "reciprocal" - computes sigmoid using :math:`1 + e^{-x}` and computing + the reciprocal + + Note: stable for x in [-500, 500]. Unstable otherwise. + + Args: + chebyshev_terms (int, optional): highest degree of Chebyshev polynomials. + Must be even and at least 6. Defaults to 32. + method (str, optional): method used to compute sigmoid function. + Defaults to "reciprocal". + + Returns: + NadaArray: The sigmoid evaluation. + + Raises: + ValueError: Raised if method type is not supported. + """ + if self.is_rational: + + def sigmoid(x): + return x.sigmoid(chebyshev_terms=chebyshev_terms, method=method) + + return self.apply(sigmoid) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Sigmoid is not compatible with {dtype},\ + only with Rational and SecretRational types." + ) + + def gelu( + self, method: str = "tanh", tanh_method: str = "reciprocal" + ) -> "NadaArray": + r"""Computes the gelu function using the following definition + + .. math:: + gelu(x) = x/2 * (1 + tanh(\sqrt{2/\pi} * (x + 0.04471 * x^3))) + + Methods: + If a valid method is given, this function will compute gelu + using that method: + + "tanh" - computes gelu using the common approximation function + + Note: stable for x in [-18, 18]. Unstable otherwise. + + "motzkin" - computes gelu via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.2 based on the Motzkin’s polynomial preprocessing technique. + + Note: stable for all input range as the approximation is truncated + to relu function outside [-2.7, 2.7]. + + Args: + method (str, optional): method used to compute gelu function. Defaults to "tanh". + tanh_method (str, optional): method used for tanh function. Defaults to "reciprocal". + + Returns: + NadaArray: The gelu evaluation. + + Raises: + ValueError: Raised if method type is not supported. + """ + if self.is_rational: + + def gelu(x): + return x.gelu(method=method, tanh_method=tanh_method) + + return self.apply(gelu) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Gelu is not compatible with {dtype}, only with Rational and SecretRational types." + ) + + def silu( + self, + method_sigmoid: str = "reciprocal", + ) -> "NadaArray": + r"""Computes the gelu function using the following definition + + .. math:: + silu(x) = x * sigmoid(x) + + Sigmoid methods: + If a valid method is given, this function will compute sigmoid + using that method: + + "chebyshev" - computes tanh via Chebyshev approximation with + truncation and uses the identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "motzkin" - computes tanh via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.3 based on the Motzkin’s polynomial preprocessing technique. + It uses the identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "reciprocal" - computes sigmoid using :math:`1 + e^{-x}` and computing + the reciprocal + + Note: stable for x in [-500, 500]. Unstable otherwise. + + Args: + method_sigmoid (str, optional): method used to compute sigmoid function. + Defaults to "reciprocal". + + Returns: + NadaArray: The sigmoid evaluation. + + Raises: + ValueError: Raised if sigmoid method type is not supported. + """ + if self.is_rational: + + def silu(x): + return x.silu(method_sigmoid=method_sigmoid) + + return self.apply(silu) + + dtype = get_dtype(self.inner) + raise TypeError( + f"Silu is not compatible with {dtype}, only with Rational and SecretRational types." + ) + def _check_type_compatibility( value: Any, diff --git a/nada_numpy/funcs.py b/nada_numpy/funcs.py index 21d3a7e..73fe042 100644 --- a/nada_numpy/funcs.py +++ b/nada_numpy/funcs.py @@ -3,6 +3,8 @@ and manipulation of arrays and party objects. """ +# pylint:disable=too-many-lines + from typing import Any, Callable, List, Optional, Sequence, Tuple, Union import numpy as np @@ -57,6 +59,22 @@ "take", "trace", "transpose", + "sign", + "abs", + "exp", + "polynomial", + "log", + "reciprocal", + "inv_sqrt", + "sqrt", + "cossin", + "sin", + "cos", + "tan", + "tanh", + "sigmoid", + "gelu", + "silu", ] @@ -591,3 +609,528 @@ def trace(a: NadaArray, *args, **kwargs): @copy_metadata(np.transpose) def transpose(a: NadaArray, *args, **kwargs): return a.transpose(*args, **kwargs) + + +# Non-linear functions + + +def sign(arr: NadaArray) -> "NadaArray": + """Computes the sign value (0 is considered positive)""" + return arr.sign() + + +def abs(arr: NadaArray) -> "NadaArray": + """Computes the absolute value""" + return arr.abs() + + +def exp(arr: NadaArray, iterations: int = 8) -> "NadaArray": + """ + Approximates the exponential function using a limit approximation. + + The exponential function is approximated using the following limit: + + exp(x) = lim_{n -> ∞} (1 + x / n) ^ n + + The exponential function is computed by choosing n = 2 ** d, where d is set to `iterations`. + The calculation is performed by computing (1 + x / n) once and then squaring it `d` times. + + Approximation accuracy range (with 16 bit precision): + + ---------------------------------- + + | Input range x | Relative error | + + ---------------------------------- + + | [-2, 2] | <1% | + | [-7, 7] | <10% | + | [-8, 15] | <35% | + + ---------------------------------- + + + Args: + iterations (int, optional): The number of iterations for the limit approximation. + Defaults to 8. + + Returns: + NadaArray: The approximated value of the exponential function. + """ + return arr.exp(iterations=iterations) + + +def polynomial(arr: NadaArray, coefficients: list) -> "NadaArray": + """ + Computes a polynomial function on a value with given coefficients. + + The coefficients can be provided as a list of values. + They should be ordered from the linear term (order 1) first, ending with the highest order term. + **Note: The constant term is not included.** + + Args: + coefficients (list): The coefficients of the polynomial, ordered by increasing degree. + + Returns: + NadaArray: The result of the polynomial function applied to the input x. + """ + return arr.polynomial(coefficients=coefficients) + + +def log( + arr: NadaArray, + input_in_01: bool = False, + iterations: int = 2, + exp_iterations: int = 8, + order: int = 8, +) -> "NadaArray": + """ + Approximates the natural logarithm using 8th order modified Householder iterations. + This approximation is accurate within 2% relative error on the interval [0.0001, 250]. + + The iterations are computed as follows: + + h = 1 - x * exp(-y_n) + y_{n+1} = y_n - sum(h^k / k for k in range(1, order + 1)) + + Approximation accuracy range (with 16 bit precision): + + ------------------------------------- + + | Input range x | Relative error | + + ------------------------------------- + + | [0.001, 200] | <1% | + | [0.00003, 253] | <10% | + | [0.0000001, 253] | <40% | + | [253, +∞[ | Unstable | + + ------------------------------------- + + + Args: + input_in_01 (bool, optional): Indicates if the input is within the domain [0, 1]. + This setting optimizes the function for this domain, useful for computing + log-probabilities in entropy functions. + + To shift the domain of convergence, a constant 'a' is used with the identity: + + ln(u) = ln(au) - ln(a) + + Given the convergence domain for log() function is approximately [1e-4, 1e2], + we set a = 100. + Defaults to False. + iterations (int, optional): Number of Householder iterations for the approximation. + Defaults to 2. + exp_iterations (int, optional): Number of iterations for the limit approximation of exp. + Defaults to 8. + order (int, optional): Number of polynomial terms used (order of Householder approximation). + Defaults to 8. + + Returns: + NadaArray: The approximate value of the natural logarithm. + """ + return arr.log( + input_in_01=input_in_01, + iterations=iterations, + exp_iterations=exp_iterations, + order=order, + ) + + +def reciprocal( # pylint: disable=too-many-arguments + arr: NadaArray, + all_pos: bool = False, + initial: Optional["Rational"] = None, + input_in_01: bool = False, + iterations: int = 10, + log_iters: int = 1, + exp_iters: int = 8, + method: str = "NR", +) -> "NadaArray": + r""" + Approximates the reciprocal of a number through two possible methods: Newton-Raphson + and log. + + Methods: + 'NR' : `Newton-Raphson`_ method computes the reciprocal using iterations + of :math:`x_{i+1} = (2x_i - x * x_i^2)` and uses + :math:`3*exp(1 - 2x) + 0.003` as an initial guess by default. + + Approximation accuracy range (with 16 bit precision): + + ------------------------------------ + + | Input range |x| | Relative error | + + ------------------------------------ + + | [0.1, 64] | <0% | + | [0.0003, 253] | <15% | + | [0.00001, 253] | <90% | + | [253, +∞[ | Unstable | + + ------------------------------------ + + + 'log' : Computes the reciprocal of the input from the observation that: + :math:`x^{-1} = exp(-log(x))` + + Approximation accuracy range (with 16 bit precision): + + ------------------------------------ + + | Input range |x| | Relative error | + + ------------------------------------ + + | [0.0003, 253] | <15% | + | [0.00001, 253] | <90% | + | [253, +∞[ | Unstable | + + ------------------------------------ + + + Args: + all_pos (bool, optional): determines whether all elements of the + input are known to be positive, which optimizes the step of + computing the sign of the input. Defaults to False. + initial (Rational, optional): sets the initial value for the + Newton-Raphson method. By default, this will be set to :math: + `3*exp(-(x-.5)) + 0.003` as this allows the method to converge over + a fairly large domain. + input_in_01 (bool, optional) : Allows a user to indicate that the input is + in the range [0, 1], causing the function optimize for this range. + This is useful for improving the accuracy of functions on + probabilities (e.g. entropy functions). + iterations (int, optional): determines the number of Newton-Raphson iterations to run + for the `NR` method. Defaults to 10. + log_iters (int, optional): determines the number of Householder + iterations to run when computing logarithms for the `log` method. Defaults to 1. + exp_iters (int, optional): determines the number of exp + iterations to run when computing exp. Defaults to 8. + method (str, optional): method used to compute reciprocal. Defaults to "NR". + + Returns: + NadaArray: The approximate value of the reciprocal + + .. _Newton-Raphson: + https://en.wikipedia.org/wiki/Newton%27s_method + """ + # pylint:disable=duplicate-code + return arr.reciprocal( + all_pos=all_pos, + initial=initial, + input_in_01=input_in_01, + iterations=iterations, + log_iters=log_iters, + exp_iters=exp_iters, + method=method, + ) + + +def inv_sqrt( + arr: NadaArray, + initial: Optional["SecretRational"] = None, + iterations: int = 5, + method: str = "NR", +) -> "NadaArray": + r""" + Computes the inverse square root of the input using the Newton-Raphson method. + + Approximation accuracy range (with 16 bit precision): + + ---------------------------------- + + | Input range x | Relative error | + + ---------------------------------- + + | [0.1, 170] | <0% | + | [0.001, 200] | <50% | + | [200, +∞[ | Unstable | + + ---------------------------------- + + + Args: + initial (Union[SecretRational, None], optional): sets the initial value for the + Newton-Raphson iterations. By default, this will be set to allow the + method to converge over a fairly large domain. + iterations (int, optional): determines the number of Newton-Raphson iterations to run. + method (str, optional): method used to compute inv_sqrt. Defaults to "NR". + + Returns: + NadaArray: The approximate value of the inv_sqrt. + + .. _Newton-Raphson: + https://en.wikipedia.org/wiki/Fast_inverse_square_root#Newton's_method + """ + return arr.inv_sqrt(initial=initial, iterations=iterations, method=method) + + +def sqrt( + arr: NadaArray, + initial: Optional["SecretRational"] = None, + iterations: int = 5, + method: str = "NR", +) -> "NadaArray": + r""" + Computes the square root of the input by computing its inverse square root using + the Newton-Raphson method and multiplying by the input. + + Approximation accuracy range (with 16 bit precision): + + ---------------------------------- + + | Input range x | Relative error | + + ---------------------------------- + + | [0.1, 170] | <0% | + | [0.001, 200] | <50% | + | [200, +∞[ | Unstable | + + ---------------------------------- + + + Args: + initial (Union[SecretRational, None], optional): sets the initial value for the inverse + square root Newton-Raphson iterations. By default, this will be set to allow + convergence over a fairly large domain. Defaults to None. + iterations (int, optional): determines the number of Newton-Raphson iterations to run. + Defaults to 5. + method (str, optional): method used to compute sqrt. Defaults to "NR". + + Returns: + NadaArray: The approximate value of the sqrt. + + .. _Newton-Raphson: + https://en.wikipedia.org/wiki/Fast_inverse_square_root#Newton's_method + """ + return arr.sqrt(initial=initial, iterations=iterations, method=method) + + +# Trigonometry + + +def cossin(arr: NadaArray, iterations: int = 10) -> "NadaArray": + r"""Computes cosine and sine through e^(i * input) where i is the imaginary unit through the + formula: + + .. math:: + Re\{e^{i * input}\}, Im\{e^{i * input}\} = \cos(input), \sin(input) + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + Tuple[NadaArray, NadaArray]: + A tuple where the first element is cos and the second element is the sin. + """ + return arr.cossin(iterations=iterations) + + +def cos(arr: NadaArray, iterations: int = 10) -> "NadaArray": + r"""Computes the cosine of the input using cos(x) = Re{exp(i * x)}. + + Note: unstable outside [-30, 30] + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + NadaArray: The approximate value of the cosine. + """ + return arr.cos(iterations=iterations) + + +def sin(arr: NadaArray, iterations: int = 10) -> "NadaArray": + r"""Computes the sine of the input using sin(x) = Im{exp(i * x)}. + + Note: unstable outside [-30, 30] + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + NadaArray: The approximate value of the sine. + """ + return arr.sin(iterations=iterations) + + +def tan(arr: NadaArray, iterations: int = 10) -> "NadaArray": + r"""Computes the tan of the input using tan(x) = sin(x) / cos(x). + + Note: unstable outside [-30, 30] + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + NadaArray: The approximate value of the tan. + """ + return arr.tan(iterations=iterations) + + +# Activation functions + + +def tanh( + arr: NadaArray, + chebyshev_terms: int = 32, + method: str = "reciprocal", +) -> "NadaArray": + r"""Computes the hyperbolic tangent function using the identity + + .. math:: + tanh(x) = 2\sigma(2x) - 1 + + Methods: + If a valid method is given, this function will compute tanh using that method: + + "reciprocal" - computes tanh using the identity + + .. math:: + tanh(x) = 2\sigma(2x) - 1 + + Note: stable for x in [-250, 250]. Unstable otherwise. + + "chebyshev" - computes tanh via Chebyshev approximation with truncation. + + .. math:: + tanh(x) = \sum_{j=1}^chebyshev_terms c_{2j - 1} P_{2j - 1} (x / maxval) + + where c_i is the ith Chebyshev series coefficient and P_i is ith polynomial. + + Note: stable for all input range as the approximation is truncated + to +/-1 outside [-1, 1]. + + "motzkin" - computes tanh via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.3 based on the Motzkin’s polynomial preprocessing technique. + + Note: stable for all input range as the approximation is truncated + to +/-1 outside [-1, 1]. + + Args: + chebyshev_terms (int, optional): highest degree of Chebyshev polynomials. + Must be even and at least 6. Defaults to 32. + method (str, optional): method used to compute tanh function. Defaults to "reciprocal". + + Returns: + NadaArray: The tanh evaluation. + + Raises: + ValueError: Raised if method type is not supported. + """ + return arr.tanh(chebyshev_terms=chebyshev_terms, method=method) + + +def sigmoid( + arr: NadaArray, + chebyshev_terms: int = 32, + method: str = "reciprocal", +) -> "NadaArray": + r"""Computes the sigmoid function using the following definition + + .. math:: + \sigma(x) = (1 + e^{-x})^{-1} + + Methods: + If a valid method is given, this function will compute sigmoid + using that method: + + "chebyshev" - computes tanh via Chebyshev approximation with + truncation and uses the identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "motzkin" - computes tanh via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.3 based on the Motzkin’s polynomial preprocessing technique. It uses + the identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "reciprocal" - computes sigmoid using :math:`1 + e^{-x}` and computing + the reciprocal + + Note: stable for x in [-500, 500]. Unstable otherwise. + + Args: + chebyshev_terms (int, optional): highest degree of Chebyshev polynomials. + Must be even and at least 6. Defaults to 32. + method (str, optional): method used to compute sigmoid function. Defaults to "reciprocal". + + Returns: + NadaArray: The sigmoid evaluation. + + Raises: + ValueError: Raised if method type is not supported. + """ + + return arr.sigmoid(chebyshev_terms=chebyshev_terms, method=method) + + +def gelu( + arr: NadaArray, + method: str = "tanh", + tanh_method: str = "reciprocal", +) -> "NadaArray": + r"""Computes the gelu function using the following definition + + .. math:: + gelu(x) = x/2 * (1 + tanh(\sqrt{2/\pi} * (x + 0.04471 * x^3))) + + Methods: + If a valid method is given, this function will compute gelu + using that method: + + "tanh" - computes gelu using the common approximation function + + Note: stable for x in [-18, 18]. Unstable otherwise. + + "motzkin" - computes gelu via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.2 based on the Motzkin’s polynomial preprocessing technique. + + Note: stable for all input range as the approximation is truncated + to relu function outside [-2.7, 2.7]. + + Args: + method (str, optional): method used to compute gelu function. Defaults to "tanh". + tanh_method (str, optional): method used for tanh function. Defaults to "reciprocal". + + Returns: + NadaArray: The gelu evaluation. + + Raises: + ValueError: Raised if method type is not supported. + """ + + return arr.gelu(method=method, tanh_method=tanh_method) + + +def silu( + arr: NadaArray, + method_sigmoid: str = "reciprocal", +) -> "NadaArray": + r"""Computes the gelu function using the following definition + + .. math:: + silu(x) = x * sigmoid(x) + + Sigmoid methods: + If a valid method is given, this function will compute sigmoid + using that method: + + "chebyshev" - computes tanh via Chebyshev approximation with + truncation and uses the identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "motzkin" - computes tanh via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.3 based on the Motzkin’s polynomial preprocessing technique. It uses the + identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "reciprocal" - computes sigmoid using :math:`1 + e^{-x}` and computing + the reciprocal + + Note: stable for x in [-500, 500]. Unstable otherwise. + + Args: + method_sigmoid (str, optional): method used to compute sigmoid function. + Defaults to "reciprocal". + + Returns: + NadaArray: The sigmoid evaluation. + + Raises: + ValueError: Raised if sigmoid method type is not supported. + """ + return arr.silu(method_sigmoid=method_sigmoid) diff --git a/nada_numpy/types.py b/nada_numpy/types.py index d7c9a02..1382ee3 100644 --- a/nada_numpy/types.py +++ b/nada_numpy/types.py @@ -2,8 +2,9 @@ # pylint:disable=too-many-lines +import functools import warnings -from typing import Optional, Union +from typing import List, Optional, Tuple, Union import nada_dsl as dsl import numpy as np @@ -143,7 +144,7 @@ def if_else( return result -class Rational: +class Rational: # pylint:disable=too-many-public-methods """Wrapper class to store scaled Integer values representing a fixed-point number.""" def __init__( @@ -521,6 +522,30 @@ def __neg__(self) -> "Rational": """ return Rational(self.value * Integer(-1), self.log_scale, is_scaled=True) + def __lshift__(self, other: UnsignedInteger) -> "Rational": + """ + Left shift the Rational value. + + Args: + other (UnsignedInteger): The value to left shift by. + + Returns: + Rational: Left shifted Rational value. + """ + return Rational(self.value << other, self.log_scale) + + def __rshift__(self, other: UnsignedInteger) -> "Rational": + """ + Right shift the Rational value. + + Args: + other (UnsignedInteger): The value to right shift by. + + Returns: + Rational: Right shifted Rational value. + """ + return Rational(self.value >> other, self.log_scale) + def __lt__(self, other: _NadaRational) -> Union[PublicBoolean, SecretBoolean]: """ Check if this Rational is less than another. @@ -681,8 +706,572 @@ def rescale_down(self, log_scale: Optional[int] = None) -> "Rational": is_scaled=True, ) + # Non-linear functions + + def sign(self) -> "Rational": + """Computes the sign value (0 is considered positive)""" + + result = sign(self) + if not isinstance(result, Rational): + raise TypeError("sign input should be of type Rational.") + return result + + def abs(self) -> "Rational": + """Computes the absolute value""" + + result = fxp_abs(self) + if not isinstance(result, Rational): + raise TypeError("abs input should be of type Rational.") + return result + + def exp(self, iterations: int = 8) -> "Rational": + """ + Approximates the exponential function using a limit approximation. + + The exponential function is approximated using the following limit: + + exp(x) = lim_{n -> ∞} (1 + x / n) ^ n + + The function is computed by choosing n = 2 ** d, where d is set to `iterations`. + The calculation is performed by computing (1 + x / n) once and then squaring it `d` times. + + Approximation accuracy range (with 16 bit precision): + + ---------------------------------- + + | Input range x | Relative error | + + ---------------------------------- + + | [-2, 2] | <1% | + | [-7, 7] | <10% | + | [-8, 15] | <35% | + + ---------------------------------- + + + Args: + iterations (int, optional): The number of iterations for the limit approximation. + Defaults to 8. + + Returns: + Rational: The approximated value of the exponential function. + """ + + result = exp(self, iterations=iterations) + if not isinstance(result, Rational): + raise TypeError("exp input should be of type Rational.") + return result + + def polynomial(self, coefficients: list) -> "Rational": + """ + Computes a polynomial function on a value with given coefficients. + + The coefficients can be provided as a list of values. + They should be ordered from the linear term (order 1) first, ending with the + highest order term. + **Note: The constant term is not included.** + + Args: + coefficients (list): The coefficients of the polynomial, ordered by increasing degree. + + Returns: + Rational: The result of the polynomial function applied to the input x. + """ + + result = polynomial(self, coefficients=coefficients) + if not isinstance(result, Rational): + raise TypeError("polynomial input should be of type Rational.") + return result + + def log( + self, + input_in_01: bool = False, + iterations: int = 2, + exp_iterations: int = 8, + order: int = 8, + ) -> "Rational": + """ + Approximates the natural logarithm using 8th order modified Householder iterations. + This approximation is accurate within 2% relative error on the interval [0.0001, 250]. + + The iterations are computed as follows: + + h = 1 - x * exp(-y_n) + y_{n+1} = y_n - sum(h^k / k for k in range(1, order + 1)) + + Approximation accuracy range (with 16 bit precision): + + ------------------------------------- + + | Input range x | Relative error | + + ------------------------------------- + + | [0.001, 200] | <1% | + | [0.00003, 253] | <10% | + | [0.0000001, 253] | <40% | + | [253, +∞[ | Unstable | + + ------------------------------------- + + + Args: + input_in_01 (bool, optional): Indicates if the input is within the domain [0, 1]. + This setting optimizes the function for this domain, useful for computing + log-probabilities in entropy functions. + + To shift the domain of convergence, a constant 'a' is used with the identity: + + ln(u) = ln(au) - ln(a) + + Given the convergence domain for log() function is approximately [1e-4, 1e2], + we set a = 100. + Defaults to False. + iterations (int, optional): Number of Householder iterations for the approximation. + Defaults to 2. + exp_iterations (int, optional): Number of iterations for the limit approximation + of exp. Defaults to 8. + order (int, optional): Number of polynomial terms used (order of Householder + approximation). Defaults to 8. + + Returns: + Rational: The approximate value of the natural logarithm. + """ + + result = log( + self, + input_in_01=input_in_01, + iterations=iterations, + exp_iterations=exp_iterations, + order=order, + ) + if not isinstance(result, Rational): + raise TypeError("log input should be of type Rational.") + return result + + def reciprocal( # pylint: disable=too-many-arguments + self, + all_pos: bool = False, + initial: Optional["Rational"] = None, + input_in_01: bool = False, + iterations: int = 10, + log_iters: int = 1, + exp_iters: int = 8, + method: str = "NR", + ) -> "Rational": + r""" + Approximates the reciprocal of a number through two possible methods: Newton-Raphson + and log. + + Methods: + 'NR' : `Newton-Raphson`_ method computes the reciprocal using iterations + of :math:`x_{i+1} = (2x_i - x * x_i^2)` and uses + :math:`3*exp(1 - 2x) + 0.003` as an initial guess by default. + + Approximation accuracy range (with 16 bit precision): + + ------------------------------------ + + | Input range |x| | Relative error | + + ------------------------------------ + + | [0.1, 64] | <0% | + | [0.0003, 253] | <15% | + | [0.00001, 253] | <90% | + | [253, +∞[ | Unstable | + + ------------------------------------ + + + 'log' : Computes the reciprocal of the input from the observation that: + :math:`x^{-1} = exp(-log(x))` + + Approximation accuracy range (with 16 bit precision): + + ------------------------------------ + + | Input range |x| | Relative error | + + ------------------------------------ + + | [0.0003, 253] | <15% | + | [0.00001, 253] | <90% | + | [253, +∞[ | Unstable | + + ------------------------------------ + + + Args: + all_pos (bool, optional): determines whether all elements of the + input are known to be positive, which optimizes the step of + computing the sign of the input. Defaults to False. + initial (Rational, optional): sets the initial value for the + Newton-Raphson method. By default, this will be set to :math: + `3*exp(-(x-.5)) + 0.003` as this allows the method to converge over + a fairly large domain. + input_in_01 (bool, optional) : Allows a user to indicate that the input is + in the range [0, 1], causing the function optimize for this range. + This is useful for improving the accuracy of functions on + probabilities (e.g. entropy functions). + iterations (int, optional): determines the number of Newton-Raphson iterations to run + for the `NR` method. Defaults to 10. + log_iters (int, optional): determines the number of Householder + iterations to run when computing logarithms for the `log` method. Defaults to 1. + exp_iters (int, optional): determines the number of exp + iterations to run when computing exp. Defaults to 8. + method (str, optional): method used to compute reciprocal. Defaults to "NR". + + Returns: + Rational: The approximate value of the reciprocal + + .. _Newton-Raphson: + https://en.wikipedia.org/wiki/Newton%27s_method + """ + + result = reciprocal( + self, + all_pos=all_pos, + initial=initial, + input_in_01=input_in_01, + iterations=iterations, + log_iters=log_iters, + exp_iters=exp_iters, + method=method, + ) + if not isinstance(result, Rational): + raise TypeError("reciprocal input should be of type Rational.") + return result + + def inv_sqrt( + self, + initial: Optional["Rational"] = None, + iterations: int = 5, + method: str = "NR", + ) -> "Rational": + r""" + Computes the inverse square root of the input using the Newton-Raphson method. + + Approximation accuracy range (with 16 bit precision): + + ---------------------------------- + + | Input range x | Relative error | + + ---------------------------------- + + | [0.1, 170] | <0% | + | [0.001, 200] | <50% | + | [200, +∞[ | Unstable | + + ---------------------------------- + + + Args: + initial (Union[Rational, None], optional): sets the initial value for the + Newton-Raphson iterations. By default, this will be set to allow the + method to converge over a fairly large domain. + iterations (int, optional): determines the number of Newton-Raphson iterations to run. + method (str, optional): method used to compute inv_sqrt. Defaults to "NR". + + Returns: + Rational: The approximate value of the inv_sqrt. + + .. _Newton-Raphson: + https://en.wikipedia.org/wiki/Fast_inverse_square_root#Newton's_method + """ + + result = inv_sqrt(self, initial=initial, iterations=iterations, method=method) + if not isinstance(result, Rational): + raise TypeError("inv_sqrt input should be of type Rational.") + return result + + def sqrt( + self, + initial: Optional["Rational"] = None, + iterations: int = 5, + method: str = "NR", + ) -> "Rational": + r""" + Computes the square root of the input by computing its inverse square root using + the Newton-Raphson method and multiplying by the input. + + Approximation accuracy range (with 16 bit precision): + + ---------------------------------- + + | Input range x | Relative error | + + ---------------------------------- + + | [0.1, 170] | <0% | + | [0.001, 200] | <50% | + | [200, +∞[ | Unstable | + + ---------------------------------- + + + Args: + initial (Union[Rational, None], optional): sets the initial value for the inverse + square root Newton-Raphson iterations. By default, this will be set to allow + convergence over a fairly large domain. Defaults to None. + iterations (int, optional): determines the number of Newton-Raphson iterations to run. + Defaults to 5. + method (str, optional): method used to compute sqrt. Defaults to "NR". + + Returns: + Rational: The approximate value of the sqrt. + + .. _Newton-Raphson: + https://en.wikipedia.org/wiki/Fast_inverse_square_root#Newton's_method + """ + + result = sqrt(self, initial=initial, iterations=iterations, method=method) + if not isinstance(result, Rational): + raise TypeError("sqrt input should be of type Rational.") + return result + + # Trigonometry + + def cossin(self, iterations: int = 10) -> Tuple["Rational", "Rational"]: + r"""Computes cosine and sine through e^(i * input) where i is the imaginary unit through + the formula: + + .. math:: + Re\{e^{i * input}\}, Im\{e^{i * input}\} = \cos(input), \sin(input) + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + Tuple[Rational, Rational]: + A tuple where the first element is cos and the second element is the sin. + """ + + result = cossin(self, iterations=iterations) + if not isinstance(result, Rational): + raise TypeError("cossin input should be of type Rational.") + return result + + def cos(self, iterations: int = 10) -> "Rational": + r"""Computes the cosine of the input using cos(x) = Re{exp(i * x)}. + + Note: unstable outside [-30, 30] + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + Rational: The approximate value of the cosine. + """ + + result = cos(self, iterations=iterations) + if not isinstance(result, Rational): + raise TypeError("cos input should be of type Rational.") + return result + + def sin(self, iterations: int = 10) -> "Rational": + r"""Computes the sine of the input using sin(x) = Im{exp(i * x)}. + + Note: unstable outside [-30, 30] + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + Rational: The approximate value of the sine. + """ + + result = sin(self, iterations=iterations) + if not isinstance(result, Rational): + raise TypeError("sin input should be of type Rational.") + return result + + def tan(self, iterations: int = 10) -> "Rational": + r"""Computes the tan of the input using tan(x) = sin(x) / cos(x). + + Note: unstable outside [-30, 30] + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + Rational: The approximate value of the tan. + """ + + result = tan(self, iterations=iterations) + if not isinstance(result, Rational): + raise TypeError("tan input should be of type Rational.") + return result + + # Activation functions + + def tanh(self, chebyshev_terms: int = 32, method: str = "reciprocal") -> "Rational": + r"""Computes the hyperbolic tangent function using the identity -class SecretRational: + .. math:: + tanh(x) = 2\sigma(2x) - 1 + + Methods: + If a valid method is given, this function will compute tanh using that method: + + "reciprocal" - computes tanh using the identity + + .. math:: + tanh(x) = 2\sigma(2x) - 1 + + Note: stable for x in [-250, 250]. Unstable otherwise. + + "chebyshev" - computes tanh via Chebyshev approximation with truncation. + + .. math:: + tanh(x) = \sum_{j=1}^chebyshev_terms c_{2j - 1} P_{2j - 1} (x / maxval) + + where c_i is the ith Chebyshev series coefficient and P_i is ith polynomial. + + Note: stable for all input range as the approximation is truncated + to +/-1 outside [-1, 1]. + + "motzkin" - computes tanh via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.3 based on the Motzkin’s polynomial preprocessing technique. + + Note: stable for all input range as the approximation is truncated + to +/-1 outside [-1, 1]. + + Args: + chebyshev_terms (int, optional): highest degree of Chebyshev polynomials. + Must be even and at least 6. Defaults to 32. + method (str, optional): method used to compute tanh function. Defaults to "reciprocal". + + Returns: + Rational: The tanh evaluation. + + Raises: + ValueError: Raised if method type is not supported. + """ + + result = tanh(self, chebyshev_terms=chebyshev_terms, method=method) + if not isinstance(result, Rational): + raise TypeError("tanh input should be of type Rational.") + return result + + def sigmoid( + self, chebyshev_terms: int = 32, method: str = "reciprocal" + ) -> "Rational": + r"""Computes the sigmoid function using the following definition + + .. math:: + \sigma(x) = (1 + e^{-x})^{-1} + + Methods: + If a valid method is given, this function will compute sigmoid + using that method: + + "chebyshev" - computes tanh via Chebyshev approximation with + truncation and uses the identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "motzkin" - computes tanh via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.3 based on the Motzkin’s polynomial preprocessing technique. It uses + the identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "reciprocal" - computes sigmoid using :math:`1 + e^{-x}` and computing + the reciprocal + + Note: stable for x in [-500, 500]. Unstable otherwise. + + Args: + chebyshev_terms (int, optional): highest degree of Chebyshev polynomials. + Must be even and at least 6. Defaults to 32. + method (str, optional): method used to compute sigmoid function. + Defaults to "reciprocal". + + Returns: + Rational: The sigmoid evaluation. + + Raises: + ValueError: Raised if method type is not supported. + """ + + result = sigmoid(self, chebyshev_terms=chebyshev_terms, method=method) + if not isinstance(result, Rational): + raise TypeError("sigmoid input should be of type Rational.") + return result + + def gelu(self, method: str = "tanh", tanh_method: str = "reciprocal") -> "Rational": + r"""Computes the gelu function using the following definition + + .. math:: + gelu(x) = x/2 * (1 + tanh(\sqrt{2/\pi} * (x + 0.04471 * x^3))) + + Methods: + If a valid method is given, this function will compute gelu + using that method: + + "tanh" - computes gelu using the common approximation function + + Note: stable for x in [-18, 18]. Unstable otherwise. + + "motzkin" - computes gelu via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.2 based on the Motzkin’s polynomial preprocessing technique. + + Note: stable for all input range as the approximation is truncated + to relu function outside [-2.7, 2.7]. + + Args: + method (str, optional): method used to compute gelu function. Defaults to "tanh". + tanh_method (str, optional): method used for tanh function. Defaults to "reciprocal". + + Returns: + Rational: The gelu evaluation. + + Raises: + ValueError: Raised if method type is not supported. + """ + + result = gelu(self, method=method, tanh_method=tanh_method) + if not isinstance(result, Rational): + raise TypeError("gelu input should be of type Rational.") + return result + + def silu( + self, + method_sigmoid: str = "reciprocal", + ) -> "Rational": + r"""Computes the gelu function using the following definition + + .. math:: + silu(x) = x * sigmoid(x) + + Sigmoid methods: + If a valid method is given, this function will compute sigmoid + using that method: + + "chebyshev" - computes tanh via Chebyshev approximation with + truncation and uses the identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "motzkin" - computes tanh via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.3 based on the Motzkin’s polynomial preprocessing technique. + It uses the identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "reciprocal" - computes sigmoid using :math:`1 + e^{-x}` and computing + the reciprocal + + Note: stable for x in [-500, 500]. Unstable otherwise. + + Args: + method_sigmoid (str, optional): method used to compute sigmoid function. + Defaults to "reciprocal". + + Returns: + Rational: The sigmoid evaluation. + + Raises: + ValueError: Raised if sigmoid method type is not supported. + """ + + if method_sigmoid is None: + method_sigmoid = "reciprocal" + + result = silu(self, method_sigmoid=method_sigmoid) + if not isinstance(result, Rational): + raise TypeError("silu input should be of type Rational.") + return result + + +class SecretRational: # pylint:disable=too-many-public-methods """Wrapper class to store scaled SecretInteger values representing a fixed-point number.""" def __init__( @@ -1236,7 +1825,7 @@ def rescale_down(self, log_scale: Optional[int] = None) -> "SecretRational": Rescale the SecretRational value downwards by a scaling factor. Args: - log_scale (int, optional): The scaling factor. Defaults to RationalConfig.log_scale. + log_scale (int, optional): The scaling factor. Defaults to RationalConfig. Returns: SecretRational: Rescaled SecretRational value. @@ -1250,60 +1839,626 @@ def rescale_down(self, log_scale: Optional[int] = None) -> "SecretRational": is_scaled=True, ) + # Non-linear functions -def secret_rational( - name: str, party: Party, log_scale: Optional[int] = None, is_scaled: bool = True -) -> SecretRational: - """ - Creates a SecretRational from a variable in the Nillion network. + def sign(self) -> "SecretRational": + """Computes the sign value (0 is considered positive)""" - Args: - name (str): Name of variable in Nillion network. - party (Party): Name of party that provided variable. - log_scale (int, optional): Quantization scaling factor. Defaults to None. - is_scaled (bool, optional): Flag that indicates whether provided value has already been - scaled by log_scale factor. Defaults to True. + result = sign(self) + if not isinstance(result, SecretRational): + raise TypeError("sign input should be of type SecretRational.") + return result - Returns: - SecretRational: Instantiated SecretRational object. - """ - value = SecretInteger(Input(name=name, party=party)) - return SecretRational(value, log_scale, is_scaled) + def abs(self) -> "SecretRational": + """Computes the absolute value""" + result = fxp_abs(self) + if not isinstance(result, SecretRational): + raise TypeError("abs input should be of type SecretRational.") + return result -def public_rational( - name: str, party: Party, log_scale: Optional[int] = None, is_scaled: bool = True -) -> Rational: - """ - Creates a Rational from a variable in the Nillion network. + def exp(self, iterations: int = 8) -> "SecretRational": + """ + Approximates the exponential function using a limit approximation. - Args: - name (str): Name of variable in Nillion network. - party (Party): Name of party that provided variable. - log_scale (int, optional): Quantization scaling factor. Defaults to None. - is_scaled (bool, optional): Flag that indicates whether provided value has already been - scaled by log_scale factor. Defaults to True. + The exponential function is approximated using the following limit: - Returns: - Rational: Instantiated Rational object. - """ - value = PublicInteger(Input(name=name, party=party)) - return Rational(value, log_scale, is_scaled) + exp(x) = lim_{n -> ∞} (1 + x / n) ^ n + The exponential function is computed by choosing n = 2 ** d, where d is + set to `iterations`. The calculation is performed by computing (1 + x / n) + once and then squaring it `d` times. -def rational( - value: Union[int, float, np.floating], - log_scale: Optional[int] = None, - is_scaled: bool = False, -) -> Rational: - """ - Creates a Rational from a number variable. + Approximation accuracy range (with 16 bit precision): + + ---------------------------------- + + | Input range x | Relative error | + + ---------------------------------- + + | [-2, 2] | <1% | + | [-7, 7] | <10% | + | [-8, 15] | <35% | + + ---------------------------------- + - Args: - value (Union[int, float, np.floating]): Provided input value. - log_scale (int, optional): Quantization scaling factor. Defaults to default log_scale. - is_scaled (bool, optional): Flag that indicates whether provided value has already been - scaled by log_scale factor. Defaults to True. + Args: + iterations (int, optional): The number of iterations for the limit approximation. + Defaults to 8. + + Returns: + SecretRational: The approximated value of the exponential function. + """ + + result = exp(self, iterations=iterations) + if not isinstance(result, SecretRational): + raise TypeError("exp input should be of type SecretRational.") + return result + + def polynomial(self, coefficients: list) -> "SecretRational": + """ + Computes a polynomial function on a value with given coefficients. + + The coefficients can be provided as a list of values. + They should be ordered from the linear term (order 1) first, ending with the + highest order term. + **Note: The constant term is not included.** + + Args: + coefficients (list): The coefficients of the polynomial, ordered by increasing degree. + + Returns: + SecretRational: The result of the polynomial function applied to the input x. + """ + + result = polynomial(self, coefficients=coefficients) + if not isinstance(result, SecretRational): + raise TypeError("polynomial input should be of type SecretRational.") + return result + + def log( + self, + input_in_01: bool = False, + iterations: int = 2, + exp_iterations: int = 8, + order: int = 8, + ) -> "SecretRational": + """ + Approximates the natural logarithm using 8th order modified Householder iterations. + This approximation is accurate within 2% relative error on the interval [0.0001, 250]. + + The iterations are computed as follows: + + h = 1 - x * exp(-y_n) + y_{n+1} = y_n - sum(h^k / k for k in range(1, order + 1)) + + Approximation accuracy range (with 16 bit precision): + + ------------------------------------- + + | Input range x | Relative error | + + ------------------------------------- + + | [0.001, 200] | <1% | + | [0.00003, 253] | <10% | + | [0.0000001, 253] | <40% | + | [253, +∞[ | Unstable | + + ------------------------------------- + + + Args: + input_in_01 (bool, optional): Indicates if the input is within the domain [0, 1]. + This setting optimizes the function for this domain, useful for computing + log-probabilities in entropy functions. + + To shift the domain of convergence, a constant 'a' is used with the identity: + + ln(u) = ln(au) - ln(a) + + Given the convergence domain for log() function is approximately [1e-4, 1e2], + we set a = 100. + Defaults to False. + iterations (int, optional): Number of Householder iterations for the approximation. + Defaults to 2. + exp_iterations (int, optional): Number of iterations for the limit approximation of + exp. Defaults to 8. + order (int, optional): Number of polynomial terms used (order of + Householder approximation). Defaults to 8. + + Returns: + SecretRational: The approximate value of the natural logarithm. + """ + + result = log( + self, + input_in_01=input_in_01, + iterations=iterations, + exp_iterations=exp_iterations, + order=order, + ) + if not isinstance(result, SecretRational): + raise TypeError("log input should be of type SecretRational.") + return result + + def reciprocal( # pylint: disable=too-many-arguments + self, + all_pos: bool = False, + initial: Optional["Rational"] = None, + input_in_01: bool = False, + iterations: int = 10, + log_iters: int = 1, + exp_iters: int = 8, + method: str = "NR", + ) -> "SecretRational": + r""" + Approximates the reciprocal of a number through two possible methods: Newton-Raphson + and log. + + Methods: + 'NR' : `Newton-Raphson`_ method computes the reciprocal using iterations + of :math:`x_{i+1} = (2x_i - x * x_i^2)` and uses + :math:`3*exp(1 - 2x) + 0.003` as an initial guess by default. + + Approximation accuracy range (with 16 bit precision): + + ------------------------------------ + + | Input range |x| | Relative error | + + ------------------------------------ + + | [0.1, 64] | <0% | + | [0.0003, 253] | <15% | + | [0.00001, 253] | <90% | + | [253, +∞[ | Unstable | + + ------------------------------------ + + + 'log' : Computes the reciprocal of the input from the observation that: + :math:`x^{-1} = exp(-log(x))` + + Approximation accuracy range (with 16 bit precision): + + ------------------------------------ + + | Input range |x| | Relative error | + + ------------------------------------ + + | [0.0003, 253] | <15% | + | [0.00001, 253] | <90% | + | [253, +∞[ | Unstable | + + ------------------------------------ + + + Args: + all_pos (bool, optional): determines whether all elements of the + input are known to be positive, which optimizes the step of + computing the sign of the input. Defaults to False. + initial (Rational, optional): sets the initial value for the + Newton-Raphson method. By default, this will be set to :math: + `3*exp(-(x-.5)) + 0.003` as this allows the method to converge over + a fairly large domain. + input_in_01 (bool, optional) : Allows a user to indicate that the input is + in the range [0, 1], causing the function optimize for this range. + This is useful for improving the accuracy of functions on + probabilities (e.g. entropy functions). + iterations (int, optional): determines the number of Newton-Raphson iterations to run + for the `NR` method. Defaults to 10. + log_iters (int, optional): determines the number of Householder + iterations to run when computing logarithms for the `log` method. Defaults to 1. + exp_iters (int, optional): determines the number of exp + iterations to run when computing exp. Defaults to 8. + method (str, optional): method used to compute reciprocal. Defaults to "NR". + + Returns: + SecretRational: The approximate value of the reciprocal + + .. _Newton-Raphson: + https://en.wikipedia.org/wiki/Newton%27s_method + """ + + result = reciprocal( + self, + all_pos=all_pos, + initial=initial, + input_in_01=input_in_01, + iterations=iterations, + log_iters=log_iters, + exp_iters=exp_iters, + method=method, + ) + if not isinstance(result, SecretRational): + raise TypeError("reciprocal input should be of type SecretRational.") + return result + + def inv_sqrt( + self, + initial: Optional["SecretRational"] = None, + iterations: int = 5, + method: str = "NR", + ) -> "SecretRational": + r""" + Computes the inverse square root of the input using the Newton-Raphson method. + + Approximation accuracy range (with 16 bit precision): + + ---------------------------------- + + | Input range x | Relative error | + + ---------------------------------- + + | [0.1, 170] | <0% | + | [0.001, 200] | <50% | + | [200, +∞[ | Unstable | + + ---------------------------------- + + + Args: + initial (Union[SecretRational, None], optional): sets the initial value for the + Newton-Raphson iterations. By default, this will be set to allow the + method to converge over a fairly large domain. + iterations (int, optional): determines the number of Newton-Raphson iterations to run. + method (str, optional): method used to compute inv_sqrt. Defaults to "NR". + + Returns: + SecretRational: The approximate value of the inv_sqrt. + + .. _Newton-Raphson: + https://en.wikipedia.org/wiki/Fast_inverse_square_root#Newton's_method + """ + + result = inv_sqrt(self, initial=initial, iterations=iterations, method=method) + if not isinstance(result, SecretRational): + raise TypeError("inv_sqrt input should be of type SecretRational.") + return result + + def sqrt( + self, + initial: Optional["SecretRational"] = None, + iterations: int = 5, + method: str = "NR", + ) -> "SecretRational": + r""" + Computes the square root of the input by computing its inverse square root using + the Newton-Raphson method and multiplying by the input. + + Approximation accuracy range (with 16 bit precision): + + ---------------------------------- + + | Input range x | Relative error | + + ---------------------------------- + + | [0.1, 170] | <0% | + | [0.001, 200] | <50% | + | [200, +∞[ | Unstable | + + ---------------------------------- + + + Args: + initial (Union[SecretRational, None], optional): sets the initial value for the + inverse square root Newton-Raphson iterations. By default, this will be set + to allow convergence over a fairly large domain. Defaults to None. + iterations (int, optional): determines the number of Newton-Raphson iterations to run. + Defaults to 5. + method (str, optional): method used to compute sqrt. Defaults to "NR". + + Returns: + SecretRational: The approximate value of the sqrt. + + .. _Newton-Raphson: + https://en.wikipedia.org/wiki/Fast_inverse_square_root#Newton's_method + """ + + result = sqrt(self, initial=initial, iterations=iterations, method=method) + if not isinstance(result, SecretRational): + raise TypeError("sqrt input should be of type SecretRational.") + return result + + # Trigonometry + + def cossin(self, iterations: int = 10) -> Tuple["SecretRational", "SecretRational"]: + r"""Computes cosine and sine through e^(i * input) where i is the imaginary unit + through the formula: + + .. math:: + Re\{e^{i * input}\}, Im\{e^{i * input}\} = \cos(input), \sin(input) + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + Tuple[SecretRational, SecretRational]: + A tuple where the first element is cos and the second element is the sin. + """ + + result = cossin(self, iterations=iterations) + if not isinstance(result, SecretRational): + raise TypeError("cossin input should be of type SecretRational.") + return result + + def cos(self, iterations: int = 10) -> "SecretRational": + r"""Computes the cosine of the input using cos(x) = Re{exp(i * x)}. + + Note: unstable outside [-30, 30] + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + SecretRational: The approximate value of the cosine. + """ + + result = cos(self, iterations=iterations) + if not isinstance(result, SecretRational): + raise TypeError("cos input should be of type SecretRational.") + return result + + def sin(self, iterations: int = 10) -> "SecretRational": + r"""Computes the sine of the input using sin(x) = Im{exp(i * x)}. + + Note: unstable outside [-30, 30] + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + SecretRational: The approximate value of the sine. + """ + + result = sin(self, iterations=iterations) + if not isinstance(result, SecretRational): + raise TypeError("sin input should be of type SecretRational.") + return result + + def tan(self, iterations: int = 10) -> "SecretRational": + r"""Computes the tan of the input using tan(x) = sin(x) / cos(x). + + Note: unstable outside [-30, 30] + + Args: + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + SecretRational: The approximate value of the tan. + """ + + result = tan(self, iterations=iterations) + if not isinstance(result, SecretRational): + raise TypeError("tan input should be of type SecretRational.") + return result + + # Activation functions + + def tanh( + self, chebyshev_terms: int = 32, method: str = "reciprocal" + ) -> "SecretRational": + r"""Computes the hyperbolic tangent function using the identity + + .. math:: + tanh(x) = 2\sigma(2x) - 1 + + Methods: + If a valid method is given, this function will compute tanh using that method: + + "reciprocal" - computes tanh using the identity + + .. math:: + tanh(x) = 2\sigma(2x) - 1 + + Note: stable for x in [-250, 250]. Unstable otherwise. + + "chebyshev" - computes tanh via Chebyshev approximation with truncation. + + .. math:: + tanh(x) = \sum_{j=1}^chebyshev_terms c_{2j - 1} P_{2j - 1} (x / maxval) + + where c_i is the ith Chebyshev series coefficient and P_i is ith polynomial. + + Note: stable for all input range as the approximation is truncated + to +/-1 outside [-1, 1]. + + "motzkin" - computes tanh via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.3 based on the Motzkin’s polynomial preprocessing technique. + + Note: stable for all input range as the approximation is truncated + to +/-1 outside [-1, 1]. + + Args: + chebyshev_terms (int, optional): highest degree of Chebyshev polynomials. + Must be even and at least 6. Defaults to 32. + method (str, optional): method used to compute tanh function. Defaults to "reciprocal". + + Returns: + SecretRational: The tanh evaluation. + + Raises: + ValueError: Raised if method type is not supported. + """ + + result = tanh(self, chebyshev_terms=chebyshev_terms, method=method) + if not isinstance(result, SecretRational): + raise TypeError("tanh input should be of type SecretRational.") + return result + + def sigmoid( + self, chebyshev_terms: int = 32, method: str = "reciprocal" + ) -> "SecretRational": + r"""Computes the sigmoid function using the following definition + + .. math:: + \sigma(x) = (1 + e^{-x})^{-1} + + Methods: + If a valid method is given, this function will compute sigmoid + using that method: + + "chebyshev" - computes tanh via Chebyshev approximation with + truncation and uses the identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "motzkin" - computes tanh via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.3 based on the Motzkin’s polynomial preprocessing technique. + It uses the identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "reciprocal" - computes sigmoid using :math:`1 + e^{-x}` and computing + the reciprocal + + Note: stable for x in [-500, 500]. Unstable otherwise. + + Args: + chebyshev_terms (int, optional): highest degree of Chebyshev polynomials. + Must be even and at least 6. Defaults to 32. + method (str, optional): method used to compute sigmoid function. + Defaults to "reciprocal". + + Returns: + SecretRational: The sigmoid evaluation. + + Raises: + ValueError: Raised if method type is not supported. + """ + + result = sigmoid(self, chebyshev_terms=chebyshev_terms, method=method) + if not isinstance(result, SecretRational): + raise TypeError("sigmoid input should be of type SecretRational.") + return result + + def gelu( + self, method: str = "tanh", tanh_method: str = "reciprocal" + ) -> "SecretRational": + r"""Computes the gelu function using the following definition + + .. math:: + gelu(x) = x/2 * (1 + tanh(\sqrt{2/\pi} * (x + 0.04471 * x^3))) + + Methods: + If a valid method is given, this function will compute gelu + using that method: + + "tanh" - computes gelu using the common approximation function + + Note: stable for x in [-18, 18]. Unstable otherwise. + + "motzkin" - computes gelu via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.2 based on the Motzkin’s polynomial preprocessing technique. + + Note: stable for all input range as the approximation is truncated + to relu function outside [-2.7, 2.7]. + + Args: + method (str, optional): method used to compute gelu function. Defaults to "tanh". + tanh_method (str, optional): method used for tanh function. Defaults to "reciprocal". + + Returns: + SecretRational: The gelu evaluation. + + Raises: + ValueError: Raised if method type is not supported. + """ + + result = gelu(self, method=method, tanh_method=tanh_method) + if not isinstance(result, SecretRational): + raise TypeError("gelu input should be of type SecretRational.") + return result + + def silu( + self, + method_sigmoid: str = "reciprocal", + ) -> "SecretRational": + r"""Computes the gelu function using the following definition + + .. math:: + silu(x) = x * sigmoid(x) + + Sigmoid methods: + If a valid method is given, this function will compute sigmoid + using that method: + + "chebyshev" - computes tanh via Chebyshev approximation with + truncation and uses the identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "motzkin" - computes tanh via approximation from the paper + "BOLT: Privacy-Preserving, Accurate and Efficient Inference for Transformers" + on section 5.3 based on the Motzkin’s polynomial preprocessing technique. + It uses the identity: + + .. math:: + \sigma(x) = \frac{1}{2}tanh(\frac{x}{2}) + \frac{1}{2} + + Note: stable for all input range as the approximation is truncated + to 0/1 outside [-1, 1]. + + "reciprocal" - computes sigmoid using :math:`1 + e^{-x}` and computing + the reciprocal + + Note: stable for x in [-500, 500]. Unstable otherwise. + + Args: + method_sigmoid (str, optional): method used to compute sigmoid function. + Defaults to "reciprocal". + + Returns: + SecretRational: The sigmoid evaluation. + + Raises: + ValueError: Raised if sigmoid method type is not supported. + """ + + result = silu(self, method_sigmoid=method_sigmoid) + if not isinstance(result, SecretRational): + raise TypeError("silu input should be of type SecretRational.") + return result + + +def secret_rational( + name: str, party: Party, log_scale: Optional[int] = None, is_scaled: bool = True +) -> SecretRational: + """ + Creates a SecretRational from a variable in the Nillion network. + + Args: + name (str): Name of variable in Nillion network. + party (Party): Name of party that provided variable. + log_scale (int, optional): Quantization scaling factor. Defaults to None. + is_scaled (bool, optional): Flag that indicates whether provided value has already been + scaled by log_scale factor. Defaults to True. + + Returns: + SecretRational: Instantiated SecretRational object. + """ + value = SecretInteger(Input(name=name, party=party)) + return SecretRational(value, log_scale, is_scaled) + + +def public_rational( + name: str, party: Party, log_scale: Optional[int] = None, is_scaled: bool = True +) -> Rational: + """ + Creates a Rational from a variable in the Nillion network. + + Args: + name (str): Name of variable in Nillion network. + party (Party): Name of party that provided variable. + log_scale (int, optional): Quantization scaling factor. Defaults to None. + is_scaled (bool, optional): Flag that indicates whether provided value has already been + scaled by log_scale factor. Defaults to True. + + Returns: + Rational: Instantiated Rational object. + """ + value = PublicInteger(Input(name=name, party=party)) + return Rational(value, log_scale, is_scaled) + + +def rational( + value: Union[int, float, np.floating], + log_scale: Optional[int] = None, + is_scaled: bool = False, +) -> Rational: + """ + Creates a Rational from a number variable. + + Args: + value (Union[int, float, np.floating]): Provided input value. + log_scale (int, optional): Quantization scaling factor. Defaults to default log_scale. + is_scaled (bool, optional): Flag that indicates whether provided value has already been + scaled by log_scale factor. Defaults to True. Returns: Rational: Instantiated Rational object. @@ -1389,7 +2544,8 @@ class _RationalConfig(metaclass=_MetaRationalConfig): def set_log_scale(new_log_scale: int) -> None: """ Sets the default Rational log scaling factor to a new value. - Note that this value is the LOG scale and will be used as a base-2 exponent during quantization. + Note that this value is the LOG scale and will be used as a base-2 exponent + during quantization. Args: new_log_scale (int): New log scaling factor. @@ -1415,3 +2571,503 @@ def get_log_scale() -> int: def reset_log_scale() -> None: """Resets the Rational log scaling factor to the original default value""" _RationalConfig.log_scale = _RationalConfig.default_log_scale + + +# Fixed-point math operations + +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +# +# Part of the code is from the CrypTen Facebook Project: +# https://github.com/facebookresearch/CrypTen/blob/main/crypten/common/functions/logic.py +# https://github.com/facebookresearch/CrypTen/blob/main/crypten/common/functions/approximations.py +# +# Modifications: +# July, 2024 +# - Nada datatypes. +# - Relative accuracy documentation. +# - Some performance improvements. +# - Fixed Tanh Chebyshev method by changing '_hardtanh' implementation. +# - Tan. +# - Motzkin's prolynomial preprocessing approach. +# - GeLU and SiLU functions. + + +def sign(x: _NadaRational) -> _NadaRational: + """Computes the sign value (0 is considered positive)""" + + ltz_cond = x < rational(0) + ltz = ltz_cond.if_else(rational(1), rational(0)) + + return rational(1) - ltz - ltz + + +def fxp_abs(x: _NadaRational) -> _NadaRational: + """Computes the absolute value of a rational""" + return x * sign(x) + + +def exp(x: _NadaRational, iterations: int = 8) -> _NadaRational: + """ + Approximates the exponential function using a limit approximation. + """ + + iters_na = UnsignedInteger(iterations) + + result = rational(1) + (x >> iters_na) + for _ in range(iterations): + result = result**2 + return result + + +def polynomial(x: _NadaRational, coefficients: List[Rational]) -> _NadaRational: + """ + Computes a polynomial function on a value with given coefficients. + + The coefficients can be provided as a list of values. + They should be ordered from the linear term (order 1) first, ending with the highest order term. + **Note: The constant term is not included.** + """ + result = coefficients[0] * x + + for power, coeff in enumerate(coefficients[1:], start=2): + result += coeff * (x**power) + + return result + + +def log( + x: _NadaRational, + input_in_01: bool = False, + iterations: int = 2, + exp_iterations: int = 8, + order: int = 8, +) -> _NadaRational: + """ + Approximates the natural logarithm using 8th order modified Householder iterations. + """ + + if input_in_01: + return log( + x * rational(100), + iterations=iterations, + exp_iterations=exp_iterations, + order=order, + ) - rational(4.605170) + + # Initialization to a decent estimate (found by qualitative inspection): + # ln(x) = x/120 - 20exp(-2x - 1.0) + 3.0 + term1 = x * rational(1 / 120.0) + term2 = exp(-x - x - rational(1), iterations=exp_iterations) * rational(20) + y = term1 - term2 + rational(3.0) + + # 8th order Householder iterations + for _ in range(iterations): + h = rational(1) - x * exp(-y, iterations=exp_iterations) + y -= polynomial(h, [rational(1 / (i + 1)) for i in range(order)]) + return y + + +def reciprocal( # pylint: disable=too-many-arguments + x: _NadaRational, + all_pos: bool = False, + initial: Optional[Rational] = None, + input_in_01: bool = False, + iterations: int = 10, + log_iters: int = 1, + exp_iters: int = 8, + method: str = "NR", +) -> _NadaRational: + """ + Approximates the reciprocal of a number through two possible methods: Newton-Raphson + and log. + """ + if input_in_01: + rec = reciprocal( + x * rational(64), + method=method, + all_pos=True, + initial=initial, + iterations=iterations, + ) * rational(64) + return rec + + if not all_pos: + sgn = sign(x) + pos = sgn * x + return sgn * reciprocal( + pos, method=method, all_pos=True, initial=initial, iterations=iterations + ) + + if method == "NR": + if initial is None: + # Initialization to a decent estimate (found by qualitative inspection): + # 1/x = 3exp(1 - 2x) + 0.003 + result = rational(3) * exp( + rational(1) - x - x, iterations=exp_iters + ) + rational(0.003) + else: + result = initial + for _ in range(iterations): + result = result + result - result * result * x + return result + if method == "log": + return exp(-log(x, iterations=log_iters), iterations=exp_iters) + raise ValueError(f"Invalid method {method} given for reciprocal function") + + +def inv_sqrt( + x: _NadaRational, + initial: Optional[Union[_NadaRational, None]] = None, + iterations: int = 5, + method: str = "NR", +) -> _NadaRational: + """ + Computes the inverse square root of the input using the Newton-Raphson method. + """ + + if method == "NR": + if initial is None: + # Initialization to a decent estimate (found by qualitative inspection): + # exp(- x/2 - 0.2) * 2.2 + 0.2 - x/1024 + y = exp(-(x >> UnsignedInteger(1)) - rational(0.2)) * rational( + 2.2 + ) + rational(0.2) + y -= x >> UnsignedInteger(10) # div by 1024 + else: + y = initial + + # Newton Raphson iterations for inverse square root + for _ in range(iterations): + y = (y * (rational(3) - x * y * y)) >> UnsignedInteger(1) + return y + raise ValueError(f"Invalid method {method} given for inv_sqrt function") + + +def sqrt( + x: _NadaRational, + initial: Union[_NadaRational, None] = None, + iterations: int = 5, + method: str = "NR", +) -> _NadaRational: + """ + Computes the square root of the input by computing its inverse square root using + the Newton-Raphson method and multiplying by the input. + """ + + if method == "NR": + return inv_sqrt(x, initial=initial, iterations=iterations, method=method) * x + + raise ValueError(f"Invalid method {method} given for sqrt function") + + +# Trigonometry + + +def _eix(x: _NadaRational, iterations: int = 10) -> Tuple[_NadaRational, _NadaRational]: + r"""Computes e^(i * x) where i is the imaginary unit through the formula: + + .. math:: + Re\{e^{i * x}\}, Im\{e^{i * x}\} = \cos(x), \sin(x) + + Args: + x (Union[Rational, SecretRational]): the input value. + iterations (int, optional): determines the number of iterations to run. Defaults to 10. + + Returns: + Tuple[Union[Rational, SecretRational], Union[Rational, SecretRational]]: + A tuple where the first element is cos and the second element is the sin. + """ + + one = rational(1) + im = x >> UnsignedInteger(iterations) + + # First iteration uses knowledge that `re` is public and = 1 + re = one - im * im + im *= rational(2) + + # Compute (a + bi)^2 -> (a^2 - b^2) + (2ab)i `iterations` times + for _ in range(iterations - 1): + a2 = re * re + b2 = im * im + im = im * re + im *= rational(2) + re = a2 - b2 + + return re, im + + +def cossin( + x: _NadaRational, iterations: int = 10 +) -> Tuple[_NadaRational, _NadaRational]: + r""" + Computes cosine and sine through e^(i * x) where i is the imaginary unit. + """ + return _eix(x, iterations=iterations) + + +def cos(x: _NadaRational, iterations: int = 10) -> _NadaRational: + r""" + Computes the cosine of x using cos(x) = Re{exp(i * x)}. + """ + return cossin(x, iterations=iterations)[0] + + +def sin(x: _NadaRational, iterations: int = 10) -> _NadaRational: + r""" + Computes the sine of x using sin(x) = Im{exp(i * x)}. + """ + return cossin(x, iterations=iterations)[1] + + +def tan(x: _NadaRational, iterations: int = 10) -> _NadaRational: + r""" + Computes the tan of x using tan(x) = sin(x) / cos(x). + """ + c, s = cossin(x, iterations=iterations) + return s * reciprocal(c) + + +# Activation functions + + +@functools.lru_cache(maxsize=10) +def chebyshev_series(func, width, terms): + """ + Computes Chebyshev coefficients. + """ + n_range = np.arange(start=0, stop=terms, dtype=float) + x = width * np.cos((n_range + 0.5) * np.pi / terms) + y = func(x) + cos_term = np.cos(np.outer(n_range, n_range + 0.5) * np.pi / terms) + coeffs = (2 / terms) * np.sum(y * cos_term, axis=1) + return coeffs + + +def tanh( + x: _NadaRational, chebyshev_terms: int = 32, method: str = "reciprocal" +) -> _NadaRational: + """ + Computes the hyperbolic tangent function. + """ + + if method == "reciprocal": + return sigmoid(x + x, method=method) * rational(2) - rational(1) + if method == "chebyshev": + coeffs = chebyshev_series(np.tanh, 1, chebyshev_terms)[1::2] + # transform np.array of float to na.array of rationals + coeffs = np.vectorize(rational)(coeffs) + out = _chebyshev_polynomials(x, chebyshev_terms).transpose() @ coeffs + # truncate outside [-maxval, maxval] + return _hardtanh(x, out) + if method == "motzkin": + # Using approximation from "BOLT: Privacy-Preserving, Accurate and Efficient + # Inference for Transformers" + # section 5.3 based on the Motzkin’s polynomial preprocessing technique. + + # ltz is used for absolute value of x and to compute sign (used to generate result). + # We don't use 'abs' and 'sign' functions to avoid computing ltz twice. + # sign = 1 - 2 * ltz, where ltz = (x < rational(0)).if_else(rational(1), rational(0)) + sgn = rational(1) - rational(2) * (x < rational(0)).if_else( + rational(1), rational(0) + ) + # absolute value + abs_x = x * sgn + + # Motzkin’s polynomial preprocessing + t0 = rational(-4.259314087994767) + t1 = rational(18.86353816972803) + t2 = rational(-36.42402897526823) + t3 = rational(-0.013232131886235352) + t4 = rational(-3.3289339650097993) + t5 = rational(-0.0024920889620412097) + tanh_p0 = (abs_x + t0) * abs_x + t1 + tanh_p1 = (tanh_p0 + abs_x + t2) * tanh_p0 * t3 * abs_x + t4 * abs_x + t5 + + return (abs_x > rational(2.855)).if_else(sgn, sgn * tanh_p1) + raise ValueError(f"Unrecognized method {method} for tanh") + + +### Auxiliary functions for tanh + + +def _chebyshev_polynomials(x: _NadaRational, terms: int) -> np.ndarray: + """Evaluates odd degree Chebyshev polynomials at x. + + Chebyshev Polynomials of the first kind are defined as: + + .. math:: + P_0(x) = 1, P_1(x) = x, P_n(x) = 2 P_{n - 1}(x) - P_{n-2}(x) + + Args: + x (Union["Rational", "SecretRational"]): input at which polynomials are evaluated + terms (int): highest degree of Chebyshev polynomials. + Must be even and at least 6. + Returns: + NadaArray of polynomials evaluated at x of shape `(terms, *x)`. + + Raises: + ValueError: Raised if 'terrms' is odd and < 6. + """ + if terms % 2 != 0 or terms < 6: + raise ValueError("Chebyshev terms must be even and >= 6") + + # Initiate base polynomials + # P_0 + # polynomials = np.array([x]) + # y = rational(4) * x * x - rational(2) + # z = y - rational(1) + # # P_1 + # polynomials = np.append(polynomials, z * x) + + # # Generate remaining Chebyshev polynomials using the recurrence relation + # for k in range(2, terms // 2): + # next_polynomial = y * polynomials[k - 1] - polynomials[k - 2] + # polynomials = np.append(polynomials, next_polynomial) + + # return polynomials + + + polynomials = [x] + y = rational(4) * x * x - rational(2) + z = y - rational(1) + # P_1 + polynomials.append(z * x) + + # Generate remaining Chebyshev polynomials using the recurrence relation + for k in range(2, terms // 2): + next_polynomial = y * polynomials[k - 1] - polynomials[k - 2] + polynomials.append(next_polynomial) + + return np.array(polynomials) + + +def _hardtanh( + x: _NadaRational, + output: _NadaRational, + abs_const: _NadaRational = rational(1), + abs_range: _NadaRational = rational(1), +) -> _NadaRational: + r"""Applies the HardTanh function element-wise. + + HardTanh is defined as: + + .. math:: + \text{HardTanh}(x) = \begin{cases} + 1 & \text{ if } x > 1 \\ + -1 & \text{ if } x < -1 \\ + Tanh(x) & \text{ otherwise } \\ + \end{cases} + + The range of the linear region :math:`[-1, 1]` can be adjusted using + :attr:`abs_range`. + + Args: + x (Union[Rational, SecretRational]): the input value of the Tanh. + output (Union[Rational, SecretRational]): the output value of the approximation of Tanh. + abs_const (Union[Rational, SecretRational]): constant value to which |Tanh(x)| converges. + Defaults to 1. + abs_range (Union[Rational, SecretRational]): absolute value of the range. Defaults to 1. + + Returns: + Union[Rational, SecretRational]: HardTanh output. + """ + # absolute value + sgn = sign(x) + abs_x = x * sgn + # chekc if inside [-abs_range, abs_range] interval + ineight_cond = abs_x < abs_range + result = ineight_cond.if_else(output, abs_const * sgn) + + return result + + +### End of auxiliary functions for tanh + + +def sigmoid( + x: _NadaRational, chebyshev_terms: int = 32, method: str = "reciprocal" +) -> _NadaRational: + """ + Computes the sigmoid function. + """ + if method == "chebyshev": + tanh_approx = tanh( + x >> UnsignedInteger(1), method=method, chebyshev_terms=chebyshev_terms + ) + return (tanh_approx >> UnsignedInteger(1)) + rational(0.5) + if method == "motzkin": + tanh_approx = tanh( + x >> UnsignedInteger(1), method=method, chebyshev_terms=chebyshev_terms + ) + return (tanh_approx >> UnsignedInteger(1)) + rational(0.5) + if method == "reciprocal": + # ltz is used for absolute value of x and to generate 'result'. + # We don't use 'abs' function to avoid computing ltz twice. + ltz_cond = x < rational(0) + ltz = ltz_cond.if_else(rational(1), rational(0)) + # compute absolute value of x + sgn = rational(1) - rational(2) * ltz + pos_x = x * sgn + + denominator = exp(-pos_x) + rational(1) + pos_output = reciprocal( + denominator, all_pos=True, initial=rational(0.75), iterations=3, exp_iters=9 + ) + + # result is equivalent to (1 - ltz).if_else(pos_output, 1 - pos_output) + result = pos_output + ltz - rational(2) * pos_output * ltz + return result + raise ValueError(f"Unrecognized method {method} for sigmoid") + + +def gelu( + x: _NadaRational, method: str = "tanh", tanh_method: str = "reciprocal" +) -> _NadaRational: + """ + Computes the gelu function. + """ + + if method == "tanh": + # Using common approximation: + # x/2 * (1 + tanh(0.797884560 * ( x + 0.04471 * x ** 3 ) ) ) + # where 0.797884560 ~= sqrt(2/pi). + val = rational(0.797884560) * (x + rational(0.044715) * x**3) + return (x * (rational(1) + tanh(val, method=tanh_method))) >> UnsignedInteger(1) + if method == "motzkin": + # Using approximation from "BOLT: Privacy-Preserving, Accurate and Efficient + # Inference for Transformers" + # section 5.2 based on the Motzkin’s polynomial preprocessing technique. + + # ltz is used for absolute value of x and to compute relu. + # We don't use 'abs' and '_relu' functions to avoid computing ltz twice. + ltz = (x < rational(0)).if_else(rational(1), rational(0)) + # absolute value + sgn = rational(1) - rational(2) * ltz + abs_x = x * sgn + # relu + relu = x * (rational(1) - ltz) + + # Motzkin’s polynomial preprocessing + g0 = rational(0.14439048359960427) + g1 = rational(-0.7077117131613893) + g2 = rational(4.5702822654246535) + g3 = rational(-8.15444702051307) + g4 = rational(16.382265425072532) + gelu_p0 = (g0 * abs_x + g1) * abs_x + g2 + gelu_p1 = (gelu_p0 + g0 * abs_x + g3) * gelu_p0 + g4 + (x >> UnsignedInteger(1)) + + return (abs_x > rational(2.7)).if_else(relu, gelu_p1) + raise ValueError(f"Unrecognized method {method} for gelu") + + +def silu( + x: _NadaRational, + method_sigmoid: str = "reciprocal", +) -> _NadaRational: + """ + Computes the gelu function + """ + return x * sigmoid(x, method=method_sigmoid) diff --git a/tests/nada-tests/nada-project.toml b/tests/nada-tests/nada-project.toml index 4b2b28b..ce878d3 100644 --- a/tests/nada-tests/nada-project.toml +++ b/tests/nada-tests/nada-project.toml @@ -180,4 +180,16 @@ prime_size = 128 [[programs]] path = "src/type_guardrails.py" +prime_size = 128 + +[[programs]] +path = "src/fxpmath_funcs.py" +prime_size = 128 + +[[programs]] +path = "src/fxpmath_methods.py" +prime_size = 128 + +[[programs]] +path = "src/fxpmath_arrays.py" prime_size = 128 \ No newline at end of file diff --git a/tests/nada-tests/src/fxpmath_arrays.py b/tests/nada-tests/src/fxpmath_arrays.py new file mode 100644 index 0000000..1516f19 --- /dev/null +++ b/tests/nada-tests/src/fxpmath_arrays.py @@ -0,0 +1,64 @@ +import numpy as np +from nada_dsl import * + +import nada_numpy as na + + +def nada_main(): + + parties = na.parties(2) + + # We use na.SecretRational to create a secret rational number for party 0 + a = na.secret_rational("my_input_0", parties[0]) + + c = na.NadaArray(np.array([a, na.rational(1.5)])) + + result_sign = c.sign() + result_abs = c.abs() + result_exp = c.exp() + result_log = c.log() + result_rec_NR = c.reciprocal(method="NR") + result_rec_log = c.reciprocal(method="log") + result_isqrt = c.inv_sqrt() + result_sqrt = c.sqrt() + result_sin = c.sin() + result_cos = c.cos() + result_tan = c.tan() + result_tanh = c.tanh() + result_tanh_che = c.tanh(method="chebyshev") + result_tanh_motz = c.tanh(method="motzkin") + result_sig = c.sigmoid() + result_sig_che = c.sigmoid(method="chebyshev") + result_sig_motz = c.sigmoid(method="motzkin") + result_gelu = c.gelu() + result_gelu_motz = c.gelu(method="motzkin") + result_silu = c.silu() + result_silu_che = c.silu(method_sigmoid="chebyshev") + result_silu_motz = c.silu(method_sigmoid="motzkin") + + final_result = ( + result_sign.output(parties[1], "result_sign") + + result_abs.output(parties[1], "result_abs") + + result_exp.output(parties[1], "result_exp") + + result_log.output(parties[1], "result_log") + + result_rec_NR.output(parties[1], "result_rec_NR") + + result_rec_log.output(parties[1], "result_rec_log") + + result_isqrt.output(parties[1], "result_isqrt") + + result_sqrt.output(parties[1], "result_sqrt") + + result_sin.output(parties[1], "result_sin") + + result_cos.output(parties[1], "result_cos") + + result_tan.output(parties[1], "result_tan") + + result_tanh.output(parties[1], "result_tanh") + + result_tanh_che.output(parties[1], "result_tanh_che") + + result_tanh_motz.output(parties[1], "result_tanh_motz") + + result_sig.output(parties[1], "result_sig") + + result_sig_che.output(parties[1], "result_sig_che") + + result_sig_motz.output(parties[1], "result_sig_motz") + + result_gelu.output(parties[1], "result_gelu") + + result_gelu_motz.output(parties[1], "result_gelu_motz") + + result_silu.output(parties[1], "result_silu") + + result_silu_che.output(parties[1], "result_silu_che") + + result_silu_motz.output(parties[1], "result_silu_motz") + ) + + return final_result diff --git a/tests/nada-tests/src/fxpmath_funcs.py b/tests/nada-tests/src/fxpmath_funcs.py new file mode 100644 index 0000000..62fe6ad --- /dev/null +++ b/tests/nada-tests/src/fxpmath_funcs.py @@ -0,0 +1,55 @@ +from nada_dsl import * + +import nada_numpy as na + + +def nada_main(): + + parties = na.parties(2) + + # We use na.SecretRational to create a secret rational number for party 0 + a = na.secret_rational("my_input_0", parties[0]) + + result_exp = na.exp(a * na.rational(2)) + result_log = na.log(a * na.rational(100)) + result_rec_NR = na.reciprocal(a * na.rational(2), method="NR") + result_rec_log = na.reciprocal(a * na.rational(4), method="log") + result_isqrt = na.inv_sqrt(a * na.rational(210)) + result_sqrt = na.sqrt(a * na.rational(16)) + result_sin = na.sin(a * na.rational(2.1)) + result_cos = na.cos(a * na.rational(2.1)) + result_tan = na.tan(a * na.rational(4.8)) + result_tanh = na.tanh(a * na.rational(1.3)) + result_tanh_che = na.tanh(a * na.rational(0.3), method="chebyshev") + result_tanh_motz = na.tanh(a * na.rational(0.4), method="motzkin") + result_sig = na.sigmoid(a * na.rational(0.1)) + result_sig_che = na.sigmoid(a * na.rational(-0.1), method="chebyshev") + result_sig_motz = na.sigmoid(a * na.rational(10), method="motzkin") + result_gelu = na.gelu(a * na.rational(-13)) + result_gelu_motz = na.gelu(a * na.rational(-13), method="motzkin") + result_silu = na.silu(a * na.rational(10)) + result_silu_che = na.silu(a * na.rational(-10), method_sigmoid="chebyshev") + result_silu_motz = na.silu(a * na.rational(0), method_sigmoid="motzkin") + + return [ + Output(result_exp.value, "result_exp", parties[1]), + Output(result_log.value, "result_log", parties[1]), + Output(result_rec_NR.value, "result_rec_NR", parties[1]), + Output(result_rec_log.value, "result_rec_log", parties[1]), + Output(result_isqrt.value, "result_isqrt", parties[1]), + Output(result_sqrt.value, "result_sqrt", parties[1]), + Output(result_sin.value, "result_sin", parties[1]), + Output(result_cos.value, "result_cos", parties[1]), + Output(result_tan.value, "result_tan", parties[1]), + Output(result_tanh.value, "result_tanh", parties[1]), + Output(result_tanh_che.value, "result_tanh_che", parties[1]), + Output(result_tanh_motz.value, "result_tanh_motz", parties[1]), + Output(result_sig.value, "result_sig", parties[1]), + Output(result_sig_che.value, "result_sig_che", parties[1]), + Output(result_sig_motz.value, "result_sig_motz", parties[1]), + Output(result_gelu.value, "result_gelu", parties[1]), + Output(result_gelu_motz.value, "result_gelu_motz", parties[1]), + Output(result_silu.value, "result_silu", parties[1]), + Output(result_silu_che.value, "result_silu_che", parties[1]), + Output(result_silu_motz.value, "result_silu_motz", parties[1]), + ] diff --git a/tests/nada-tests/src/fxpmath_methods.py b/tests/nada-tests/src/fxpmath_methods.py new file mode 100644 index 0000000..3f9a172 --- /dev/null +++ b/tests/nada-tests/src/fxpmath_methods.py @@ -0,0 +1,55 @@ +from nada_dsl import * + +import nada_numpy as na + + +def nada_main(): + + parties = na.parties(2) + + # We use na.SecretRational to create a secret rational number for party 0 + a = na.secret_rational("my_input_0", parties[0]) + + result_exp = (a * na.rational(2)).exp() + result_log = (a * na.rational(100)).log() + result_rec_NR = (a * na.rational(2)).reciprocal(method="NR") + result_rec_log = (a * na.rational(4)).reciprocal(method="log") + result_isqrt = (a * na.rational(210)).inv_sqrt() + result_sqrt = (a * na.rational(16)).sqrt() + result_sin = (a * na.rational(2.1)).sin() + result_cos = (a * na.rational(2.1)).cos() + result_tan = (a * na.rational(4.8)).tan() + result_tanh = (a * na.rational(1.3)).tanh() + result_tanh_che = (a * na.rational(0.3)).tanh(method="chebyshev") + result_tanh_motz = (a * na.rational(0.4)).tanh(method="motzkin") + result_sig = (a * na.rational(0.1)).sigmoid() + result_sig_che = (a * na.rational(-0.1)).sigmoid(method="chebyshev") + result_sig_motz = (a * na.rational(10)).sigmoid(method="motzkin") + result_gelu = (a * na.rational(-13)).gelu() + result_gelu_motz = (a * na.rational(-13)).gelu(method="motzkin") + result_silu = (a * na.rational(10)).silu() + result_silu_che = (a * na.rational(-10)).silu(method_sigmoid="chebyshev") + result_silu_motz = (a * na.rational(0)).silu(method_sigmoid="motzkin") + + return [ + Output(result_exp.value, "result_exp", parties[1]), + Output(result_log.value, "result_log", parties[1]), + Output(result_rec_NR.value, "result_rec_NR", parties[1]), + Output(result_rec_log.value, "result_rec_log", parties[1]), + Output(result_isqrt.value, "result_isqrt", parties[1]), + Output(result_sqrt.value, "result_sqrt", parties[1]), + Output(result_sin.value, "result_sin", parties[1]), + Output(result_cos.value, "result_cos", parties[1]), + Output(result_tan.value, "result_tan", parties[1]), + Output(result_tanh.value, "result_tanh", parties[1]), + Output(result_tanh_che.value, "result_tanh_che", parties[1]), + Output(result_tanh_motz.value, "result_tanh_motz", parties[1]), + Output(result_sig.value, "result_sig", parties[1]), + Output(result_sig_che.value, "result_sig_che", parties[1]), + Output(result_sig_motz.value, "result_sig_motz", parties[1]), + Output(result_gelu.value, "result_gelu", parties[1]), + Output(result_gelu_motz.value, "result_gelu_motz", parties[1]), + Output(result_silu.value, "result_silu", parties[1]), + Output(result_silu_che.value, "result_silu_che", parties[1]), + Output(result_silu_motz.value, "result_silu_motz", parties[1]), + ] diff --git a/tests/nada-tests/tests/fxpmath_arrays.yaml b/tests/nada-tests/tests/fxpmath_arrays.yaml new file mode 100644 index 0000000..a59f757 --- /dev/null +++ b/tests/nada-tests/tests/fxpmath_arrays.yaml @@ -0,0 +1,95 @@ +--- +program: fxpmath_arrays +inputs: + my_input_0: + SecretInteger: "3" +expected_outputs: + result_sig_0: + SecretInteger: "32711" + result_sqrt_1: + Integer: "80265" + result_sig_motz_1: + Integer: "53477" + result_silu_0: + SecretInteger: "1" + result_gelu_0: + SecretInteger: "1" + result_tan_1: + Integer: "888645" + result_tanh_1: + Integer: "59434" + result_tan_0: + SecretInteger: "0" + result_exp_0: + SecretInteger: "65536" + result_sig_1: + Integer: "53644" + result_silu_motz_1: + Integer: "80215" + result_gelu_motz_1: + Integer: "91629" + result_sig_che_1: + Integer: "53580" + result_log_0: + SecretInteger: "-614828" + result_sqrt_0: + SecretInteger: "45" + result_silu_che_0: + SecretInteger: "1" + result_rec_NR_1: + Integer: "43692" + result_tanh_che_1: + Integer: "65536" + result_gelu_motz_0: + SecretInteger: "103" + result_exp_1: + Integer: "292119" + result_isqrt_1: + Integer: "53510" + result_cos_1: + Integer: "4846" + result_tanh_motz_1: + Integer: "59394" + result_cos_0: + SecretInteger: "65536" + result_sig_che_0: + SecretInteger: "32768" + result_rec_log_0: + SecretInteger: "65040192" + result_sig_motz_0: + SecretInteger: "32686" + result_log_1: + Integer: "26359" + result_tanh_0: + SecretInteger: "-114" + result_sin_1: + Integer: "65710" + result_gelu_1: + Integer: "91839" + result_silu_che_1: + Integer: "80370" + result_tanh_motz_0: + SecretInteger: "-160" + result_tanh_che_0: + SecretInteger: "2" + result_rec_NR_0: + SecretInteger: "451510139" + result_rec_log_1: + Integer: "34575" + result_silu_1: + Integer: "80466" + result_isqrt_0: + SecretInteger: "989647" + result_silu_motz_0: + SecretInteger: "1" + result_sin_0: + SecretInteger: "0" + result_sign_1: + Integer: "65536" + result_sign_0: + SecretInteger: "65536" + result_abs_1: + Integer: "98304" + result_abs_0: + SecretInteger: "3" + diff --git a/tests/nada-tests/tests/fxpmath_funcs.yaml b/tests/nada-tests/tests/fxpmath_funcs.yaml new file mode 100644 index 0000000..bd10daa --- /dev/null +++ b/tests/nada-tests/tests/fxpmath_funcs.yaml @@ -0,0 +1,44 @@ +--- +program: fxpmath_funcs +inputs: + my_input_0: + SecretInteger: "65536" +expected_outputs: + result_gelu: + SecretInteger: "-13" + result_cos: + SecretInteger: "-32566" + result_isqrt: + SecretInteger: "-2341" + result_gelu_motz: + SecretInteger: "0" + result_silu: + SecretInteger: "655340" + result_sin: + SecretInteger: "57328" + result_tanh_che: + SecretInteger: "19089" + result_tanh_motz: + SecretInteger: "24917" + result_silu_che: + SecretInteger: "0" + result_sig_motz: + SecretInteger: "65536" + result_tanh: + SecretInteger: "56624" + result_rec_log: + SecretInteger: "15552" + result_sig_che: + SecretInteger: "31130" + result_sig: + SecretInteger: "34401" + result_tan: + SecretInteger: "-922765" + result_sqrt: + SecretInteger: "262128" + result_log: + SecretInteger: "298843" + result_exp: + SecretInteger: "480249" + result_rec_NR: + SecretInteger: "32768" \ No newline at end of file diff --git a/tests/nada-tests/tests/fxpmath_methods.yaml b/tests/nada-tests/tests/fxpmath_methods.yaml new file mode 100644 index 0000000..42f5f38 --- /dev/null +++ b/tests/nada-tests/tests/fxpmath_methods.yaml @@ -0,0 +1,44 @@ +--- +program: fxpmath_methods +inputs: + my_input_0: + SecretInteger: "65536" +expected_outputs: + result_gelu: + SecretInteger: "-13" + result_cos: + SecretInteger: "-32566" + result_isqrt: + SecretInteger: "-2341" + result_gelu_motz: + SecretInteger: "0" + result_silu: + SecretInteger: "655340" + result_sin: + SecretInteger: "57328" + result_tanh_che: + SecretInteger: "19089" + result_tanh_motz: + SecretInteger: "24917" + result_silu_che: + SecretInteger: "0" + result_sig_motz: + SecretInteger: "65536" + result_tanh: + SecretInteger: "56624" + result_rec_log: + SecretInteger: "15552" + result_sig_che: + SecretInteger: "31130" + result_sig: + SecretInteger: "34401" + result_tan: + SecretInteger: "-922765" + result_sqrt: + SecretInteger: "262128" + result_log: + SecretInteger: "298843" + result_exp: + SecretInteger: "480249" + result_rec_NR: + SecretInteger: "32768" \ No newline at end of file diff --git a/tests/test_all.py b/tests/test_all.py index c27d049..f7cd3e2 100644 --- a/tests/test_all.py +++ b/tests/test_all.py @@ -49,7 +49,10 @@ # Not supported yet # "unsigned_matrix_inverse", "private_inverse", - # "unsigned_matrix_inverse_2" + # "unsigned_matrix_inverse_2", + "fxpmath_funcs", + "fxpmath_arrays", + "fxpmath_methods" ] EXAMPLES = [