diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..c9162b9 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +charset = utf-8 +indent_style = tab +indent_size = 4 +insert_final_newline = true +end_of_line = lf + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/.github/.templateMarker b/.github/.templateMarker new file mode 100644 index 0000000..5e3a3e0 --- /dev/null +++ b/.github/.templateMarker @@ -0,0 +1 @@ +KOLANICH/python_project_boilerplate.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..89ff339 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + allow: + - dependency-type: "all" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..7fe33b3 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,15 @@ +name: CI +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + build: + runs-on: ubuntu-22.04 + steps: + - name: typical python workflow + uses: KOLANICH-GHActions/typical-python-workflow@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d91cebb --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +__pycache__ +*.pyc +*.pyo +*.egg-info +build +dist +.eggs +/monkeytype.sqlite3 +/*.srctrldb +/*.srctrlbm +/*.srctrlprj \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..e236211 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,14 @@ +#image: pypy:latest +image: registry.gitlab.com/kolanich-subgroups/docker-images/fixed_python:latest + +variables: + DOCKER_DRIVER: overlay2 + SAST_ANALYZER_IMAGE_TAG: latest + SAST_DISABLE_DIND: "true" + SAST_CONFIDENCE_LEVEL: 5 + CODECLIMATE_VERSION: latest + +include: + - template: SAST.gitlab-ci.yml + - template: Code-Quality.gitlab-ci.yml + - template: License-Management.gitlab-ci.yml diff --git a/Code_Of_Conduct.md b/Code_Of_Conduct.md new file mode 100644 index 0000000..c36308b --- /dev/null +++ b/Code_Of_Conduct.md @@ -0,0 +1 @@ +No codes of conduct! diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..20f0fa8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include UNLICENSE +include *.md +include tests +include .editorconfig diff --git a/ReadMe.md b/ReadMe.md new file mode 100644 index 0000000..9450d72 --- /dev/null +++ b/ReadMe.md @@ -0,0 +1,25 @@ +antlrCompile.py [![Unlicensed work](https://raw.githubusercontent.com/unlicense/unlicense.org/master/static/favicon.png)](https://unlicense.org/) +=============== +~~[wheel (GitLab)](https://gitlab.com/KOLANICH/antlrCompile.py/-/jobs/artifacts/master/raw/dist/antlrCompile-0.CI-py3-none-any.whl?job=build)~~ +~~[wheel (GHA via `nightly.link`)](https://nightly.link/UniGrammar-libs/antlrCompile.py/workflows/CI/master/antlrCompile-0.CI-py3-none-any.whl)~~ +~~![GitLab Build Status](https://gitlab.com/KOLANICH/antlrCompile.py/badges/master/pipeline.svg)~~ +~~![GitLab Coverage](https://gitlab.com/UniGrammar/antlrCompile.py/badges/master/coverage.svg)~~ +~~[![GitHub Actions](https://github.com/UniGrammar-libs/antlrCompile.py/workflows/CI/badge.svg)](https://github.com/UniGrammar-libs/antlrCompile.py/actions/)~~ +[![Libraries.io Status](https://img.shields.io/librariesio/github/UniGrammar-libs/antlrCompile.py.svg)](https://libraries.io/github/UniGrammar-libs/antlrCompile.py) +[![Code style: antiflash](https://img.shields.io/badge/code%20style-antiflash-FFF.svg)](https://codeberg.org/KOLANICH-tools/antiflash.py) + +A python wrapper for [ANTLR 4](https://github.com/antlr/antlr4) - a well-known multilanguage LL(*) parser generator. The libs allows you to + +* transpile an ANTLR grammar into the corresponding source code. +* visualize AST resulted from parsing source code with a grammar for debug purposes. + +Currently it lacks docs, but be brave to read the source code, it is only a wrapper that is not that big. + +Our fork is currently needed +------------------------ +You need [our fork of ANTL](https://codeberg.org/UniGrammar/antlr4/tree/tool_refactoring) in order for this to work. We [have tried to upstream our changes](https://github.com/antlr/antlr4/pull/2774), but the project's BDFL is against "big changes". Then I had refactored the changes into several smaller PRs that don't introduce the changes but make it easier for me to introduce them with subclassing. They are still not merged: + +* [Refactor `CodeGenerator`: move target initialization code into a separate method](https://github.com/antlr/antlr4/pull/3925) +* [Extracted getting the interp file name into a separate method `Tool::getInterpFileName`](https://github.com/antlr/antlr4/pull/3924) +* [Move a part of `CodeGenerator.write` into a separate reusable method `writeToWriter` to be able to reuse it later](https://github.com/antlr/antlr4/pull/3923) +* [Convert some members of CodeGenerator from private into protected](https://github.com/antlr/antlr4/pull/3922) diff --git a/UNLICENSE b/UNLICENSE new file mode 100644 index 0000000..efb9808 --- /dev/null +++ b/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/antlrCompile/ANTLRInternalClasses.py b/antlrCompile/ANTLRInternalClasses.py new file mode 100644 index 0000000..9fb70e9 --- /dev/null +++ b/antlrCompile/ANTLRInternalClasses.py @@ -0,0 +1,19 @@ +from abc import ABC, abstractmethod + + +class ANTLRInternalClasses(ABC): + __slots__ = () + + ext = None + + @abstractmethod + def CharStreams(self, src): + raise NotImplementedError + + @abstractmethod + def CommonTokenStream(self, lexers): + raise NotImplementedError + + @abstractmethod + def _toClasses(self, compResult): + raise NotImplementedError diff --git a/antlrCompile/__init__.py b/antlrCompile/__init__.py new file mode 100644 index 0000000..b6367ba --- /dev/null +++ b/antlrCompile/__init__.py @@ -0,0 +1,156 @@ +import typing +from abc import ABC, abstractmethod +from pathlib import Path +from collections import OrderedDict, defaultdict +import threading + +from .antlr import neededAntlrClasses, getAntlrPath, ANTLRLanguage +from .core import CompilationResult + +from UniGrammarRuntimeCore.ICompiler import ICompiler + + +class ANTLR(ICompiler): + __slots__ = ("ji", "t") + + def __init__(self) -> None: + from JAbs import SelectedJVMInitializer + + antlrClassPath = getAntlrPath() + self.ji = SelectedJVMInitializer( + [ + antlrClassPath + ], + neededAntlrClasses + ) + + self.t = self.ji.Tool() + self.t.outputDirectory = None + + def parse(self, grammar: typing.Union[Path, str]): + if isinstance(grammar, Path): + gF = grammar.absolute() + gT = gF.read_text() + elif isinstance(grammar, str): + gT = grammar + + return self.t.parseGrammarFromString(gT) + + def compileStr(self, grammarText: str, target: ANTLRLanguage = None, fileName: typing.Union[Path, str] = "grammar.g4") -> CompilationResult: + """Parses the DSL string and generates internal object from it. `fileName` must not be used for retrieving source, instead it must provide hints, useful for caching or more meaningful error messages and may be just ignored if the backend doesn't support its usage this way.""" + + target = ANTLRLanguage(target) + rAST = self.parse(grammarText) + + if fileName is None: + fileName = "grammar.g4" + else: + fileName = str(fileName) + + rAST.defaultOptions["language"] = target.value + + g = self.t.createGrammar(rAST) + #print(g.ast.toStringTree()) + + g.fileName = fileName + + res = self.t.processInMemory(g, True) + + #r = g.getRule(0) + #mainRuleName = r.name + + return CompilationResult(res, target, g.name) + + def compile(self, grammar: typing.Union[Path, str], target: ANTLRLanguage = ANTLRLanguage.Python3, fileName: typing.Optional[typing.Union[Path, str]] = None) -> CompilationResult: + if isinstance(grammar, Path): + assert fileName is None + return self.compileFile(grammar, target) + return self.compileStr(grammar, target, fileName) + + +class Vis: + __slots__ = ("ji", "MyListener") + def __init__(self) -> None: + from JAbs import SelectedJVMInitializer + + antlrClassPath = getAntlrPath() + self.ji = SelectedJVMInitializer( + [ + antlrClassPath + ], + [ + "org.antlr.v4.gui.Trees", + "org.antlr.v4.gui.TreePostScriptGenerator", + "java.util.ArrayList", + "java.util.Arrays", + "java.awt.event.WindowListener" + ] + ) + + class MyListener(self.ji.WindowListener, metaclass=self.ji._Implements): + # fuck, no __slots__ here + def __init__(self, e: threading.Event): + self.e = e + + @self.ji._Override + def windowClosed(self, e: "java.awt.event.WindowEvent"): + pass + self.e.set() + + @self.ji._Override + def windowClosing(self, e: "java.awt.event.WindowEvent"): + pass + @self.ji._Override + def windowOpened(self, e: "java.awt.event.WindowEvent"): + pass + @self.ji._Override + def windowIconified(self, e: "java.awt.event.WindowEvent"): + pass + @self.ji._Override + def windowDeiconified(self, e: "java.awt.event.WindowEvent"): + pass + @self.ji._Override + def windowActivated(self, e: "java.awt.event.WindowEvent"): + pass + @self.ji._Override + def windowDeactivated(self, e: "java.awt.event.WindowEvent"): + pass + + self.MyListener = MyListener + + def blockOnGUIWindow(self, wind): + """Blocks untill window is closed""" + e = threading.Event() + listn = self.MyListener(e) + wind.addWindowListener(listn) + e.wait() + + def treeGUIVisualization(self, parser: "ANTLRParser", source: str, block: bool=True) -> None: + """Shows a GUI dialog with parse tree""" + fullTree = parser(source) + task = self.ji.Trees.inspect(fullTree, self.ji.Arrays.asList(parser.ruleNames)) + wind = task.get() + + if block: + self.blockOnGUIWindow(wind) + + return wind + + def treePostScriptVisualization(self, parser: "ANTLRParser", source: str, fontName: str = "Helvetica", fontSize: int = 11, gapBetweenLevels: typing.Optional[float] = None, gapBetweenNodes: typing.Optional[float] = None, nodeWidthPadding: typing.Optional[int] = None, nodeHeightPaddingAbove: typing.Optional[int] = None, nodeHeightPaddingBelow: typing.Optional[int] = None) -> str: + """Returns a PostScript string for parse tree image""" + fullTree = parser(source) + + g = self.ji.TreePostScriptGenerator(self.ji.Arrays.asList(parser.ruleNames), fullTree, fontName, fontSize) + + if gapBetweenLevels is not None: + g.gapBetweenLevels = gapBetweenLevels + if gapBetweenNodes is not None: + g.gapBetweenNodes = gapBetweenNodes + if nodeWidthPadding is not None: + g.nodeWidthPadding = nodeWidthPadding + if nodeHeightPaddingAbove is not None: + g.nodeHeightPaddingAbove = nodeHeightPaddingAbove + if nodeHeightPaddingBelow is not None: + g.nodeHeightPaddingBelow = nodeHeightPaddingBelow + + return g.getPS() diff --git a/antlrCompile/antlr.py b/antlrCompile/antlr.py new file mode 100644 index 0000000..b0b63fe --- /dev/null +++ b/antlrCompile/antlr.py @@ -0,0 +1,42 @@ +from enum import Enum +from pathlib import Path + +antlrNs = "org.antlr" +antlrRuntimeNs = antlrNs + ".runtime" +antlrVNs = antlrNs + ".v4" +antlrVRuntimeNs = antlrVNs + ".runtime" + +neededAntlrClasses = [ + antlrVNs + ".tool.Grammar", + antlrVNs + ".parse.ANTLRParser", + antlrVNs + ".Tool", + antlrVNs + ".analysis.AnalysisPipeline", + + antlrRuntimeNs + ".ANTLRFileStream", + antlrRuntimeNs + ".ANTLRStringStream", +] + +neededAntlrRuntimeClasses = [ + antlrVRuntimeNs + ".CharStream", + antlrVRuntimeNs + ".CharStreams", + antlrVRuntimeNs + ".TokenStream", + antlrVRuntimeNs + ".CommonTokenStream", +] + +# todo: get targets via reflection from org.antlr.v4.codegen.target (class names are Target) + + +def getAntlrPath() -> Path: + return Path("./antlr4-4.11.2-SNAPSHOT-complete.jar") + + +class ANTLRLanguage(Enum): + Python3 = "Python3" + Python2 = "Python2" + Java = "Java" + Cpp = "Cpp" + CSharp = "CSharp" + Go = "Go" + ECMAScript = JavaScript = "JavaScript" + PHP = "PHP" + Swift = "Swift" diff --git a/antlrCompile/backends/__init__.py b/antlrCompile/backends/__init__.py new file mode 100644 index 0000000..dfc6288 --- /dev/null +++ b/antlrCompile/backends/__init__.py @@ -0,0 +1,8 @@ +from .java import ANTLRInternalClassesJava +from .python import ANTLRInternalClassesPython +from ..antlr import ANTLRLanguage + +langs2InternalMapping = { + ANTLRLanguage.Java: ANTLRInternalClassesJava, + ANTLRLanguage.Python3: ANTLRInternalClassesPython, +} diff --git a/antlrCompile/backends/java.py b/antlrCompile/backends/java.py new file mode 100644 index 0000000..45550bc --- /dev/null +++ b/antlrCompile/backends/java.py @@ -0,0 +1,53 @@ +import typing + +from ..ANTLRInternalClasses import ANTLRInternalClasses +from ..antlr import neededAntlrRuntimeClasses, getAntlrPath + +class ANTLRInternalClassesJava(ANTLRInternalClasses): + + __slots__ = ("ji", "javaCompile", "antlrClassPath") + + ext = ".java" + extLen = len(ext) + + _necessaryRoles = ("lexer", "parser", "listener") + + def __init__(self): + from JAbs import SelectedJVMInitializer + + self.antlrClassPath = getAntlrPath() + self.ji = SelectedJVMInitializer( + [ + self.antlrClassPath + ], + neededAntlrRuntimeClasses + ["org.antlr.v4.runtime.Parser"] + ) + + from javaMdktCompiler import javaCompile + self.javaCompile = javaCompile + + def CharStreams(self, src): + return self.ji.CharStreams.fromString(src) + + def CommonTokenStream(self, lexer): + return self.ji.CommonTokenStream(lexer) + + def generateArgsForCompiler(self) -> typing.Tuple[str, ...]: + return ("-cp", str(self.antlrClassPath)) + + def _toClasses(self, compResult): + targets = [] + remap = [] + for role in self.__class__._necessaryRoles: + files = getattr(compResult, role) + if len(files) == 1: + for f in files: + pyName = str(f.name) + if pyName.endswith(self.__class__.ext): + stem = pyName[: -self.__class__.extLen] + remap.append((role, stem)) + targets.append((stem, f.content)) + compiled = self.javaCompile(targets, *self.generateArgsForCompiler()) + + for role, stem in remap: + yield (str(role), compiled[stem]) diff --git a/antlrCompile/backends/python.py b/antlrCompile/backends/python.py new file mode 100644 index 0000000..5b9bb60 --- /dev/null +++ b/antlrCompile/backends/python.py @@ -0,0 +1,36 @@ +import typing + +from ..ANTLRInternalClasses import ANTLRInternalClasses + +antlr4 = None + + +class ANTLRInternalClassesPython(ANTLRInternalClasses): + ext = ".py" + extLen = len(ext) + __slots__ = ("antlr4",) + _necessaryRoles = ("lexer", "parser") + + @classmethod + def __init__(self): + import antlr4 + + self.antlr4 = antlr4 + + def CharStreams(self, src): + return self.antlr4.InputStream(src) + + def CommonTokenStream(self, lexer): + return self.antlr4.CommonTokenStream(lexer) + + def _somethingToIterable(self, something: typing.Any, selector: typing.Callable[[typing.Any, str], typing.Union[str, "ast.AST"]]): + for role in self.__class__._necessaryRoles: + className = something.name + role.capitalize() + astOrText = selector(something, role, className) + compiled = compile(astOrText, className + self.__class__.ext, "exec", optimize=2) + globalz = {} + eval(compiled, globalz) + yield (role, globalz[className]) + + def _toClasses(self, compResult): + return self._somethingToIterable(compResult, lambda compResult, role, className: getattr(compResult, role)[0].content) diff --git a/antlrCompile/core.py b/antlrCompile/core.py new file mode 100644 index 0000000..713b46b --- /dev/null +++ b/antlrCompile/core.py @@ -0,0 +1,132 @@ +import typing +from abc import ABC, abstractmethod +from collections import OrderedDict +from pathlib import Path + +from UniGrammarRuntimeCore.IParser import IParser, IParserFactory +from UniGrammarRuntimeCore.PoolManager import PoolManager + +from .backends import langs2InternalMapping + + +class ANTLRParser(IParser): + ext = None + NAME = "antlr4" + + __slots__ = ("lexer", "parser", "listenerClass", "backend", "mainProductionName") + + def __init__(self, backend: typing.Type["ANTLRInternalClasses"], lexerClass=None, parserClass=None, listenerClass=None) -> None: + self.backend = backend + self.lexer = lexerClass(None) + self.parser = parserClass(None) + self.listenerClass = listenerClass + self.mainProductionName = str(next(iter(self.ruleNames))) + + @property + def ruleNames(self): + return self.parser.ruleNames + + def lex(self, src: str) -> "antlr4.CommonTokenStream.CommonTokenStream": + inputStream = self.backend.CharStreams(src) + self.lexer.inputStream = inputStream + tokensStream = self.backend.CommonTokenStream(self.lexer) + return tokensStream + + def _parse(self, s: str, trace: bool=False) -> None: + tokensStream = self.lex(s) + self.parser.setTrace(trace) + self.parser.setInputStream(tokensStream) # fucking crazy shit, for lexer there is a property and no accessor methods, for parser there is no property and there ar accessor methods + + def _getTree(self): + return getattr(self.parser, self.mainProductionName)() + + def __call__(self, s: str, trace: bool=False): + """Tree can be consumed only once!""" + self._parse(s, trace) + return self._getTree() + + +class CompilationResult(ABC): + __slots__ = ("res", "language", "name", "mainProductionName") + _dirRes = None + + def __init__(self, res, language, name: str): + self.res = res + self.language = language + self.name = name + + def __iter__(self): + for elWithRole in self.res: + role = elWithRole.role + el = elWithRole.files + if el is not None: + yield el + + def saveToDisk(self, root: Path = "."): + root = Path(root) + for files in self: + if len(files) == 1: + for f in files: + (root / f.name).write_text(f.content) + else: + for i, f in enumerate(files): + (root / (f.name + "_" + str(i))).write_text(f.content) + + def asRoleFilesItems(self): + for elWithRole in self.res: + role = elWithRole.role + el = elWithRole.files + if el is not None: + yield (role, el) + + def rolesItems(self): + for elWithRole in self.res: + role = elWithRole.role + if elWithRole.files is not None: + yield role + + def roles(self): + return tuple(self.rolesItems()) + + def asNameContentItems(self) -> typing.Iterator[typing.Tuple[str, str]]: + for el in self: + for f in el: + yield (f.name, f.content) + + def asFileNameDict(self) -> OrderedDict: + return OrderedDict(self.asNameContentItems()) + + def asRoleFilesDict(self): + return OrderedDict(self.asRoleFilesItems()) + + def __getattr__(self, k): + return getattr(self.res, k) + + def __dir__(self): + if self.__class__._dirRes is None: + d = set(dir(self.res)) | set(self.__slots__) | set(dir(self.__class__)) + self.__class__._dirRes = frozenset({p for p in d if not (len(p) >= 1 and p[0] == "_")}) + + return self.__class__._dirRes + + def toParser(self): + return ANTLRParserFactory().fromCompResult(self) + + +backendsPool = PoolManager() + +class ANTLRParserFactory(IParserFactory): + __slots__ = () + + PARSER_CLASS = ANTLRParser + + def _fromAttrIterable(self, backend: typing.Type["ANTLRInternalClasses"], something: typing.Iterator[typing.Tuple[str, type]]) -> ANTLRParser: + return self.__class__.PARSER_CLASS(backend, **{(role + "Class"): ctor for role, ctor in something}) + + def fromCompResult(self, compResult: CompilationResult) -> ANTLRParser: + backend = backendsPool(langs2InternalMapping[compResult.language]) + return self._fromAttrIterable(backend, backend._toClasses(compResult)) + + def fromInternal(self, internalRepr: CompilationResult) -> ANTLRParser: + return self.fromCompResult(internalRepr) + diff --git a/antlrCompile/py.typed b/antlrCompile/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0f9363c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,38 @@ +[build-system] +requires = ["setuptools>=61.2.0", "wheel", "toml", "setuptools_scm[toml]>=3.4.3"] +build-backend = "setuptools.build_meta" + +[project] +name = "antlrCompile" +authors = [{name = "KOLANICH"}] +description = "Python bindings for ANTLR parser generator" +readme = "ReadMe.md" +keywords = ["ANTLR", "grammar", "parser", "java", "JVM"] +license = {text = "Unlicense"} +classifiers = [ + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Development Status :: 4 - Beta", + "Environment :: Other Environment", + "Intended Audience :: Developers", + "License :: Public Domain", + "Operating System :: OS Independent", + "Topic :: Software Development :: Libraries :: Python Modules", +] +requires-python = ">=3.4" +dependencies = [ + "JAbs" # @ git+https://codeberg.org/KOLANICH-libs/JAbs.py +] +dynamic = ["version"] + +[project.urls] +Homepage = "https://codeberg.org/UniGrammar/antlrCompile.py" + +[tool.setuptools] +zip-safe = true +include-package-data = false + +[tool.setuptools.packages] +find = {namespaces = false} + +[tool.setuptools_scm]