From a86e596e6a4f6e74cb2bf00730e6609cde0c6842 Mon Sep 17 00:00:00 2001 From: Michael Staneker Date: Tue, 16 Apr 2024 09:53:40 +0000 Subject: [PATCH] Renaming, improving documentation, improving fortran style numerical literals, use FParser fortran intrinsic procedure names --- loki/expression/parser.py | 98 ++++++++++--------- loki/transform/transform_loop.py | 4 +- tests/test_expression.py | 158 ++++++++++++++++--------------- tests/test_symbolic.py | 16 ++-- tests/test_util_polyhedron.py | 18 ++-- 5 files changed, 151 insertions(+), 143 deletions(-) diff --git a/loki/expression/parser.py b/loki/expression/parser.py index 2c87a9765..c180688b9 100644 --- a/loki/expression/parser.py +++ b/loki/expression/parser.py @@ -18,38 +18,14 @@ _PREC_TIMES, _PREC_PLUS, _times, _plus ) +from loki.frontend.fparser import FORTRAN_INTRINSIC_PROCEDURES from loki.tools.util import CaseInsensitiveDict -from loki.expression import symbols as sym, operations as sym_ops, AttachScopes +import loki.expression.symbols as sym +import loki.expression.operations as sym_ops +from loki.expression.expr_visitors import AttachScopes from loki.scope import Scope -__all__ = ['LokiParser', 'loki_parse'] - - -# Fortran intrinsic procedures -intrinsic_procedures = ['DIM', 'SQRT', 'ADJUSTR', 'DATAN2', 'IEEE_SUPPORT_FLAG', 'MAXVAL', 'MAXVAL', - 'DDIM', 'DMAX1', 'TAN', 'IEEE_SUPPORT_INF', 'CLOG', 'ASIN', 'AMAX1', 'IEEE_LOGB', 'ALLOCATED', 'MIN', - 'IEEE_SUPPORT_DATATYPE', 'IEEE_RINT', 'RRSPACING', 'MAXLOC', 'DINT', 'AIMAG', 'LEN_TRIM', 'UNPACK', - 'CPU_TIME', 'CEXP', 'RANDOM_SEED', 'SIZE', 'MINLOC', 'IEEE_NEXT_AFTER', 'LLE', 'HUGE', 'MATMUL', 'CHAR', - 'ISIGN', 'DATE_AND_TIME', 'IEEE_SELECTED_REAL_KIND', 'SELECTED_REAL_KIND', 'IEEE_SUPPORT_IO', 'NULL', - 'COS', 'ISHFT', 'CSIN', 'BIT_SIZE', 'IEEE_GET_HALTING_MODE', 'DIGITS', 'CEILING', 'ALOG10', 'MINEXPONENT', - 'EXP', 'SUM', 'LOG10', 'IEEE_CLASS', 'DABS', 'SUM', 'RESHAPE', 'IEEE_IS_NEGATIVE', 'MINVAL', 'MAXLOC', - 'REAL', 'SIGN', 'IEEE_SUPPORT_DENORMAL', 'IEEE_SET_ROUNDING_MODE', 'AMIN1', 'MOD', 'SPREAD', 'DEXP', - 'CMPLX', 'SCALE', 'COUNT', 'SHAPE', 'TINY', 'SELECTED_INT_KIND', 'MODULO', 'NEAREST', 'AMOD', 'DNINT', - 'CCOS', 'MIN1', 'DMIN1', 'IBITS', 'COSH', 'DSIGN', 'MAXEXPONENT', 'MAX0', 'IEEE_SET_HALTING_MODE', - 'CSHIFT', 'DASIN', 'ALOG', 'ACHAR', 'IEEE_SET_STATUS', 'SYSTEM_CLOCK', 'MINVAL', 'SIN', 'IEOR', 'DMOD', - 'MALLOC', 'DCOSH', 'IEEE_IS_NORMAL', 'IEEE_SCALB', 'TRIM', 'MPI_SIZEOF', 'IEEE_SUPPORT_STANDARD', - 'IEEE_IS_NAN', 'PACK', 'SNGL', 'DATAN', 'LLT', 'IFIX', 'SCAN', 'KIND', 'RANGE', 'IEEE_IS_FINITE', 'NINT', - 'TRANSFER', 'ABS', 'ACOS', 'ICHAR', 'MIN0', 'AINT', 'RANDOM_NUMBER', 'REPEAT', 'LOG', 'ADJUSTL', 'UBOUND', - 'IEEE_COPY_SIGN', 'IEEE_SUPPORT_SQRT', 'GET_COMMAND', 'TRANSPOSE', 'ANINT', 'DSIN', 'LBOUND', 'EXPONENT', - 'SET_EXPONENT', 'ALL', 'ASSOCIATED', 'IEEE_ARITHMETIC', 'IEEE_GET_FLAG', 'SINH', 'IEEE_GET_STATUS', - 'ISHFTC', 'LEN', 'DPROD', 'NOT', 'DBLE', 'DSQRT', 'MINLOC', 'IOR', 'IEEE_UNORDERED', 'IDIM', 'INDEX', - 'DTANH', 'CMPLX', 'IDINT', 'IAND', 'C_F_POINTER', 'AMIN0', 'INT', 'FRACTION', 'DLOG10', 'ANY', - 'IEEE_SUPPORT_ROUNDING', 'C_ASSOCIATED', 'EOSHIFT', 'DLOG', 'AMAX0', 'DACOS', 'PRECISION', 'SPACING', - 'IDNINT', 'C_LOC', 'CABS', 'COMMAND_ARGUMENT_COUNT', 'IEEE_SUPPORT_NAN', 'EPSILON', 'ATAN2', - 'PRODUCT', 'IBCLR', 'DCOS', 'ATAN', 'IEEE_SET_FLAG', 'DSINH', 'DTAN', 'IEEE_VALUE', 'IBSET', 'MAX1', - 'MERGE', 'BTEST', 'DOT_PRODUCT', 'IACHAR', 'IEEE_SUPPORT_DIVIDE', 'CONJG', 'VERIFY', 'FLOOR', 'MAX', - 'PRODUCT', 'FLOAT', 'LGT', 'LOGICAL', 'MVBITS', 'IABS', 'RADIX', 'CSQRT', 'IEEE_GET_ROUNDING_MODE', - 'IEEE_REM', 'LGE', 'TANH', 'IEEE_SUPPORT_HALTING'] +__all__ = ['ExpressionParser', 'parse_expr'] class PymbolicMapper(Mapper): @@ -142,7 +118,7 @@ def map_algebraic_leaf(self, expr, *args, **kwargs): if isinstance(expr, pmbl.Call): if expr.function.name.lower() in ('real', 'int'): return sym.Cast(expr.function.name, [self.rec(param, *args, **kwargs) for param in expr.parameter][0]) - if expr.function.name.upper() in intrinsic_procedures: + if expr.function.name.upper() in FORTRAN_INTRINSIC_PROCEDURES: return sym.InlineCall(function=sym.Variable(name=expr.function.name), parameters=tuple(self.rec(param, *args, **kwargs) for param in expr.parameters)) return sym.Variable(name=expr.function.name, @@ -172,7 +148,8 @@ def map_list(self, expr, *args, **kwargs): class LokiEvaluationMapper(EvaluationMapper): """ - A mapper for evaluating expressions, based on pymbolic's `EvaluationMapper`. + A mapper for evaluating expressions, based on + :any:`pymbolic.mapper.evaluator.EvaluationMapper`. Parameters ---------- @@ -192,7 +169,7 @@ def map_float_literal(self, expr): map_int_literal = map_float_literal def map_variable(self, expr): - if expr.name.upper() in intrinsic_procedures: + if expr.name.upper() in FORTRAN_INTRINSIC_PROCEDURES: return self.map_call(expr) if self.strict: return super().map_variable(expr) @@ -221,36 +198,55 @@ def map_call(self, expr): return super().map_call(expr) -class LokiParser(ParserBase): +class ExpressionParser(ParserBase): """ - String Parser based on [pymbolic](https://github.com/inducer/pymbolic) 's parser. + String Parser based on `pymbolic's `_ parser for + parsing expressions from strings. The Loki String Parser utilises and extends pymbolic's parser to incorporate Fortran specific syntax and to map pymbolic expressions to Loki expressions, utilising - the mapper ``PymbolicMapper``. + the mapper :any:`PymbolicMapper`. **Further**, in order to ensure correct ordering of Fortran Statements as documented - in ['WD 1539-1 J3/23-007r1 (Draft Fortran 2023)'](https://j3-fortran.org/doc/year/10/10-007.pdf#page=155), + in `'WD 1539-1 J3/23-007r1 (Draft Fortran 2023)' `_, pymbolic's parsing logic needed to be slightly adapted. Pymbolic references: - * [GitHub: pymbolic](https://github.com/inducer/pymbolic) - * [pymbolic/parser.py](https://github.com/inducer/pymbolic/blob/main/pymbolic/parser.py) - * [pymbolic's parser documentation](https://documen.tician.de/pymbolic/utilities.html) + * `GitHub: pymbolic `_ + * `pymbolic/parser.py `_ + * `pymbolic's parser documentation `_ .. note:: **Example:** - Reference a declaration node that contains variable "a" + Using the expression parser and possibly evaluate them .. code-block:: - >>> from loki import loki_parse + >>> from loki import parse_expr + >>> # parse numerical expressions + >>> ir = parse_expr('3 + 2**4') >>> ir Sum((IntLiteral(3, None), Power(IntLiteral(2, None), IntLiteral(4, None)))) - >>> result = loki_parse('((3+2)/(3-2))**4', evaluate=True) - >>> result - FloatLiteral('625.0', None) + >>> # or expressions with variables + >>> ir = parse_expr('a*b') + >>> ir + Product((DeferredTypeSymbol('a', None, None, ),\ + DeferredTypeSymbol('b', None, None, ))) + >>> # and provide a scope e.g, with some routine defining a and b as 'real's + >>> ir = parse_expr('a*b', scope=routine) + >>> ir + Product((Scalar('a', None, None, None), Scalar('b', None, None, None))) + >>> # further, it is possible to evaluate expressions + >>> ir = parse_expr('a*b + 1', evaluate=True, context={'a': 2, 'b': 3}) + >>> ir + >>> IntLiteral(7, None) + >>> # even with functions implemented in Python + >>> def add(a, b): + >>> return a + b + >>> ir = parse_expr('a + add(a, b)', evaluate=True, context={'a': 2, 'b': 3, 'add': add}) + >>> ir + >>> IntLiteral(7, None) """ _f_true = intern("f_true") @@ -282,7 +278,7 @@ class LokiParser(ParserBase): (_f_and, pytools.lex.RE(r"\.and\.", re.IGNORECASE)), (_f_or, pytools.lex.RE(r"\.or\.", re.IGNORECASE)), (_f_not, pytools.lex.RE(r"\.not\.", re.IGNORECASE)), - (_f_float, ("|", pytools.lex.RE(r"[0-9]+\.[0-9]*([eEdD][+-]?[0-9]+)?(_[a-zA-Z]*)", re.IGNORECASE))), + (_f_float, ("|", pytools.lex.RE(r"[0-9]+\.[0-9]*([eEdD][+-]?[0-9]+)?(_([\w$]+|[0-9]+))+$", re.IGNORECASE))), (_f_int, pytools.lex.RE(r"[0-9]+?(_[a-zA-Z]*)", re.IGNORECASE)), (_f_string, ("|", pytools.lex.RE(r'\".*\"', re.IGNORECASE), pytools.lex.RE(r"\'.*\'", re.IGNORECASE))), @@ -388,7 +384,7 @@ def parse_terminal(self, pstate): return sym.LogicLiteral('.FALSE.') return super().parse_terminal(pstate) - def __call__(self, expr_str, scope=None, min_precedence=0, evaluate=False, strict=False, context=None): + def __call__(self, expr_str, scope=None, evaluate=False, strict=False, context=None): """ Call Loki String Parser to convert expression(s) represented in a string to Loki expression(s)/IR. @@ -398,8 +394,6 @@ def __call__(self, expr_str, scope=None, min_precedence=0, evaluate=False, stric The expression as a string scope : :any:`Scope` The scope to which symbol names inside the expression belong - min_precedence : int, optional - Minimum precedence evaluate : bool, optional Whether to evaluate the expression or not (default: `False`) strict : bool, optional @@ -414,7 +408,7 @@ def __call__(self, expr_str, scope=None, min_precedence=0, evaluate=False, stric :any:`Expression` The expression tree corresponding to the expression """ - result = super().__call__(expr_str, min_precedence) + result = super().__call__(expr_str) context = context or {} context = CaseInsensitiveDict(context) if evaluate: @@ -437,4 +431,8 @@ def parse_f_string(self, s): return sym.StringLiteral(s) -loki_parse = LokiParser() +parse_expr = ExpressionParser() +""" +An instance of :any:`ExpressionParser` that allows parsing expression strings into a Loki expression tree. +See :any:`ExpressionParser.__call__` for a description of the available arguments. +""" diff --git a/loki/transform/transform_loop.py b/loki/transform/transform_loop.py index a2ab5fac7..2252820a9 100644 --- a/loki/transform/transform_loop.py +++ b/loki/transform/transform_loop.py @@ -15,7 +15,7 @@ from loki.expression import ( symbols as sym, SubstituteExpressions, FindVariables, - simplify, is_constant, symbolic_op, loki_parse + simplify, is_constant, symbolic_op, parse_expr ) from loki.ir import ( Loop, Conditional, Comment, Pragma, FindNodes, Transformer, @@ -268,7 +268,7 @@ def pragma_ranges_to_loop_ranges(parameters, scope): return None ranges = [] for item in parameters['range'].split(','): - bounds = [loki_parse(bound, scope=scope) for bound in item.split(':')] + bounds = [parse_expr(bound, scope=scope) for bound in item.split(':')] ranges += [sym.LoopRange(as_tuple(bounds))] return as_tuple(ranges) diff --git a/tests/test_expression.py b/tests/test_expression.py index ed7b81b1e..0884210bd 100644 --- a/tests/test_expression.py +++ b/tests/test_expression.py @@ -25,7 +25,7 @@ DeferredTypeSymbol, Module, HAVE_FP, FindExpressions, LiteralList, FindInlineCalls, AttachScopesMapper, FindTypedSymbols, Reference, Dereference ) -from loki.expression import symbols, loki_parse +from loki.expression import symbols, parse_expr from loki.tools import gettempdir, filehash # pylint: disable=too-many-lines @@ -1002,7 +1002,7 @@ def test_string_compare(): ('ansatz(a + 1)', 'a', True), ('ansatz(b + 1)', 'a', False), # Ensure no false positives ]) -@pytest.mark.parametrize('parse', (loki_parse, parse_fparser_expression)) +@pytest.mark.parametrize('parse', (parse_expr, parse_fparser_expression)) def test_subexpression_match(parse, expr, string, ref): """ Test that we can identify individual symbols or sub-expressions in @@ -1026,7 +1026,7 @@ def test_subexpression_match(parse, expr, string, ref): ('5 + (4 + 3) - (2*1)', '5 + (4 + 3) - (2*1)'), ('a*(b*(c+(d+e)))', 'a*(b*(c + (d + e)))'), ]) -@pytest.mark.parametrize('parse', (loki_parse, parse_fparser_expression)) +@pytest.mark.parametrize('parse', (parse_expr, parse_fparser_expression)) def test_parse_expression(parse, source, ref): """ Test the utility function that parses simple expressions. @@ -1560,7 +1560,7 @@ def test_expression_c_de_reference(frontend): assert '(*renamed_var_dereference)=2' in c_str -# utility function to test loki_parse with different case +# utility function to test parse_expr with different case def convert_to_case(_str, mode='upper'): if mode == 'upper': return _str.upper() @@ -1577,7 +1577,7 @@ def convert_to_case(_str, mode='upper'): @pytest.mark.parametrize('case', ('upper', 'lower', 'random')) @pytest.mark.parametrize('frontend', available_frontends()) -def test_parser(frontend, case): +def test_expression_parser(frontend, case): fcode = """ subroutine some_routine() implicit none @@ -1605,139 +1605,139 @@ def to_str(_parsed): routine = Subroutine.from_source(fcode, frontend=frontend) module = Module.from_source(fcode_mod, frontend=frontend) - parsed = loki_parse(convert_to_case('a + b', mode=case)) + parsed = parse_expr(convert_to_case('a + b', mode=case)) assert isinstance(parsed, symbols.Sum) assert all(isinstance(_parsed, symbols.DeferredTypeSymbol) for _parsed in parsed.children) assert to_str(parsed) == 'a+b' - parsed = loki_parse(convert_to_case('a + b', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a + b', mode=case), scope=routine) assert isinstance(parsed, symbols.Sum) assert all(isinstance(_parsed, symbols.Scalar) for _parsed in parsed.children) assert all(_parsed.scope == routine for _parsed in parsed.children) assert to_str(parsed) == 'a+b' - parsed = loki_parse(convert_to_case('a + b + 2 + 10', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a + b + 2 + 10', mode=case), scope=routine) assert isinstance(parsed, symbols.Sum) assert to_str(parsed) == 'a+b+2+10' - parsed = loki_parse(convert_to_case('a - b', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a - b', mode=case), scope=routine) assert isinstance(parsed, symbols.Sum) assert isinstance(parsed.children[0], symbols.Scalar) assert isinstance(parsed.children[1], symbols.Product) assert to_str(parsed) == 'a-b' - parsed = loki_parse(convert_to_case('a * b', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a * b', mode=case), scope=routine) assert isinstance(parsed, symbols.Product) assert all(isinstance(_parsed, symbols.Scalar) for _parsed in parsed.children) assert all(_parsed.scope == routine for _parsed in parsed.children) assert to_str(parsed) == 'a*b' - parsed = loki_parse(convert_to_case('a / b', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a / b', mode=case), scope=routine) assert isinstance(parsed, symbols.Quotient) assert all(isinstance(_parsed, symbols.Scalar) for _parsed in [parsed.numerator, parsed.denominator]) assert all(_parsed.scope == routine for _parsed in [parsed.numerator, parsed.denominator]) assert to_str(parsed) == 'a/b' - parsed = loki_parse(convert_to_case('a ** b', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a ** b', mode=case), scope=routine) assert isinstance(parsed, symbols.Power) assert all(isinstance(_parsed, symbols.Scalar) for _parsed in [parsed.base, parsed.exponent]) assert all(_parsed.scope == routine for _parsed in [parsed.base, parsed.exponent]) assert to_str(parsed) == 'a**b' - parsed = loki_parse(convert_to_case('a:b', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a:b', mode=case), scope=routine) assert isinstance(parsed, symbols.RangeIndex) assert all(isinstance(_parsed, symbols.Scalar) for _parsed in [parsed.lower, parsed.upper]) assert all(_parsed.scope == routine for _parsed in [parsed.lower, parsed.upper]) assert to_str(parsed) == 'a:b' - parsed = loki_parse(convert_to_case('a:b:5', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a:b:5', mode=case), scope=routine) assert isinstance(parsed, symbols.RangeIndex) assert all(isinstance(_parsed, (symbols.Scalar, symbols.IntLiteral)) for _parsed in [parsed.lower, parsed.upper, parsed.step]) assert to_str(parsed) == 'a:b:5' - parsed = loki_parse(convert_to_case('a == b', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a == b', mode=case), scope=routine) assert parsed.operator == '==' assert isinstance(parsed, symbols.Comparison) assert all(isinstance(_parsed, symbols.Scalar) for _parsed in [parsed.left, parsed.right]) assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) assert to_str(parsed) == 'a==b' - parsed = loki_parse(convert_to_case('a.eq.b', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a.eq.b', mode=case), scope=routine) assert parsed.operator == '==' assert isinstance(parsed, symbols.Comparison) assert all(isinstance(_parsed, symbols.Scalar) for _parsed in [parsed.left, parsed.right]) assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) assert to_str(parsed) == 'a==b' - parsed = loki_parse(convert_to_case('a!=b', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a!=b', mode=case), scope=routine) assert parsed.operator == '!=' assert isinstance(parsed, symbols.Comparison) assert all(isinstance(_parsed, symbols.Scalar) for _parsed in [parsed.left, parsed.right]) assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) assert to_str(parsed) == 'a!=b' - parsed = loki_parse(convert_to_case('a.ne.b', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a.ne.b', mode=case), scope=routine) assert parsed.operator == '!=' assert isinstance(parsed, symbols.Comparison) assert all(isinstance(_parsed, symbols.Scalar) for _parsed in [parsed.left, parsed.right]) assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) assert to_str(parsed) == 'a!=b' - parsed = loki_parse(convert_to_case('a>b', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a>b', mode=case), scope=routine) assert parsed.operator == '>' assert isinstance(parsed, symbols.Comparison) assert all(isinstance(_parsed, symbols.Scalar) for _parsed in [parsed.left, parsed.right]) assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) assert to_str(parsed) == 'a>b' - parsed = loki_parse(convert_to_case('a.gt.b', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a.gt.b', mode=case), scope=routine) assert parsed.operator == '>' assert isinstance(parsed, symbols.Comparison) assert all(isinstance(_parsed, symbols.Scalar) for _parsed in [parsed.left, parsed.right]) assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) assert to_str(parsed) == 'a>b' - parsed = loki_parse(convert_to_case('a>=b', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a>=b', mode=case), scope=routine) assert parsed.operator == '>=' assert isinstance(parsed, symbols.Comparison) assert all(isinstance(_parsed, symbols.Scalar) for _parsed in [parsed.left, parsed.right]) assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) assert to_str(parsed) == 'a>=b' - parsed = loki_parse(convert_to_case('a.ge.b', mode=case), scope=routine) + parsed = parse_expr(convert_to_case('a.ge.b', mode=case), scope=routine) assert parsed.operator == '>=' assert isinstance(parsed, symbols.Comparison) assert all(isinstance(_parsed, symbols.Scalar) for _parsed in [parsed.left, parsed.right]) assert all(_parsed.scope == routine for _parsed in [parsed.left, parsed.right]) assert to_str(parsed) == 'a>=b' - parsed = loki_parse(convert_to_case('a