diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 2d0530b51..fc7086da9 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -20,6 +20,7 @@ jobs: matrix: os: - macos-13 + - macos-latest - ubuntu-latest - windows-latest python-version: @@ -28,6 +29,11 @@ jobs: - "3.10" - "3.11" - "3.12" # Latest supported by ixmp + gams-version: + # Version used until 2024-07; disabled + # - 25.1.1 + # First version including a macOS arm64 distribution + - 43.4.1 # commented: force a specific version of pandas, for e.g. pre-release # testing @@ -35,11 +41,19 @@ jobs: # - "" # - "==2.0.0rc0" - exclude: [ ] - # # Specific version combinations that are invalid, e.g. pandas 2.0 - # # requires Python >= 3.8 - # - python-version: "3.7" - # pandas-version: "==2.0.0rc0" + exclude: + # Specific version combinations that are invalid / not to be used + # No arm64 distribution for this version of GAMS + # - { os: macos-latest, gams-version: 25.1.1} + # No arm64 distributions of JPype for these Pythons + - { os: macos-latest, python-version: "3.8" } + - { os: macos-latest, python-version: "3.9" } + # Redundant with macos-latest + - { os: macos-13, python-version: "3.10" } + - { os: macos-13, python-version: "3.11" } + - { os: macos-13, python-version: "3.12" } + # Example: pandas 2.0 requires Python >= 3.8 + # - { python-version: "3.7", pandas-version: "==2.0.0rc0" } fail-fast: false @@ -81,7 +95,8 @@ jobs: - uses: iiasa/actions/setup-gams@main with: - version: 25.1.1 + version: ${{ matrix.gams-version }} + license: ${{ secrets.GAMS_LICENSE }} - name: Set RETICULATE_PYTHON # Use the environment variable set by the setup-python action, above. diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 51bb1d682..46b14e471 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -1,5 +1,11 @@ -.. Next release -.. ============ +Next release +============ + +- :mod:`ixmp` locates GAMS API libraries needed for the Java code underlying :class:`.JDBCBackend` based on the system GAMS installation (:pull:`532`). + As a result: + + - :class:`.JDBCBackend` is usable on MacOS with newer, ``arm64``-architecture processors and Python/GAMS compiled for ``arm64`` (:issue:`473`, :issue:`531`). + - GAMS API libraries are no longer (re-)packaged with ixmp in the directory :file:`ixmp/backend/jdbc/`. .. _v3.9.0: diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index 671276168..35d471385 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -1,6 +1,7 @@ import gc import logging import os +import platform import re from collections import ChainMap from collections.abc import Iterable, Sequence @@ -1233,18 +1234,32 @@ def start_jvm(jvmargs=None): .. _`JVM documentation`: https://docs.oracle.com/javase/7/docs /technotes/tools/windows/java.html) """ + from ixmp.model.gams import gams_info + if jvmargs is None: jvmargs = [] if jpype.isJVMStarted(): return + # Base directory for the classpath and library path + base = Path(__file__).with_name("jdbc") + # Arguments args = jvmargs if isinstance(jvmargs, list) else [jvmargs] + # Append path to directories containing arch-specific libraries + uname = platform.uname() + paths = [ + gams_info().java_api_dir, # GAMS system directory + base.joinpath(uname.machine), # Subdirectory of ixmp/backend/jdbc + ] + sep = ";" if uname.system == "Windows" else ":" + args.append(f"-Djava.library.path={sep.join(map(str, paths))}") + # Keyword arguments kwargs = dict( - # Glob pattern for ixmp.jar and related Java binaries - classpath=str(Path(__file__).parent.joinpath("jdbc", "*")), + # Use ixmp.jar and related Java JAR files + classpath=str(base.joinpath("*")), # For JPype 0.7 (raises a warning) and 0.8 (default is False). 'True' causes # Java string objects to be converted automatically to Python str(), as expected # by ixmp Python code. diff --git a/ixmp/backend/jdbc/gamsxjni64.dll b/ixmp/backend/jdbc/gamsxjni64.dll deleted file mode 100644 index af8b375aa..000000000 Binary files a/ixmp/backend/jdbc/gamsxjni64.dll and /dev/null differ diff --git a/ixmp/backend/jdbc/gdxjni64.dll b/ixmp/backend/jdbc/gdxjni64.dll deleted file mode 100644 index 9b786b284..000000000 Binary files a/ixmp/backend/jdbc/gdxjni64.dll and /dev/null differ diff --git a/ixmp/backend/jdbc/gevmjni64.dll b/ixmp/backend/jdbc/gevmjni64.dll deleted file mode 100644 index c96aad5d6..000000000 Binary files a/ixmp/backend/jdbc/gevmjni64.dll and /dev/null differ diff --git a/ixmp/backend/jdbc/gmdjni64.dll b/ixmp/backend/jdbc/gmdjni64.dll deleted file mode 100644 index 3bf28f053..000000000 Binary files a/ixmp/backend/jdbc/gmdjni64.dll and /dev/null differ diff --git a/ixmp/backend/jdbc/gmomjni64.dll b/ixmp/backend/jdbc/gmomjni64.dll deleted file mode 100644 index 1699eea34..000000000 Binary files a/ixmp/backend/jdbc/gmomjni64.dll and /dev/null differ diff --git a/ixmp/backend/jdbc/idxjni64.dll b/ixmp/backend/jdbc/idxjni64.dll deleted file mode 100644 index 9d200db03..000000000 Binary files a/ixmp/backend/jdbc/idxjni64.dll and /dev/null differ diff --git a/ixmp/backend/jdbc/libgamsxjni64.dylib b/ixmp/backend/jdbc/libgamsxjni64.dylib deleted file mode 100644 index 9b9bde3da..000000000 Binary files a/ixmp/backend/jdbc/libgamsxjni64.dylib and /dev/null differ diff --git a/ixmp/backend/jdbc/libgamsxjni64.so b/ixmp/backend/jdbc/libgamsxjni64.so deleted file mode 100644 index 979925185..000000000 Binary files a/ixmp/backend/jdbc/libgamsxjni64.so and /dev/null differ diff --git a/ixmp/backend/jdbc/libgdxjni64.dylib b/ixmp/backend/jdbc/libgdxjni64.dylib deleted file mode 100644 index dc7a81355..000000000 Binary files a/ixmp/backend/jdbc/libgdxjni64.dylib and /dev/null differ diff --git a/ixmp/backend/jdbc/libgdxjni64.so b/ixmp/backend/jdbc/libgdxjni64.so deleted file mode 100644 index e4d140619..000000000 Binary files a/ixmp/backend/jdbc/libgdxjni64.so and /dev/null differ diff --git a/ixmp/backend/jdbc/libgevmjni64.dylib b/ixmp/backend/jdbc/libgevmjni64.dylib deleted file mode 100644 index 5bb66262c..000000000 Binary files a/ixmp/backend/jdbc/libgevmjni64.dylib and /dev/null differ diff --git a/ixmp/backend/jdbc/libgevmjni64.so b/ixmp/backend/jdbc/libgevmjni64.so deleted file mode 100644 index 51576fa7a..000000000 Binary files a/ixmp/backend/jdbc/libgevmjni64.so and /dev/null differ diff --git a/ixmp/backend/jdbc/libgmdjni64.dylib b/ixmp/backend/jdbc/libgmdjni64.dylib deleted file mode 100644 index ae4059dae..000000000 Binary files a/ixmp/backend/jdbc/libgmdjni64.dylib and /dev/null differ diff --git a/ixmp/backend/jdbc/libgmdjni64.so b/ixmp/backend/jdbc/libgmdjni64.so deleted file mode 100644 index c46b8548b..000000000 Binary files a/ixmp/backend/jdbc/libgmdjni64.so and /dev/null differ diff --git a/ixmp/backend/jdbc/libgmomjni64.dylib b/ixmp/backend/jdbc/libgmomjni64.dylib deleted file mode 100644 index 01160d3e7..000000000 Binary files a/ixmp/backend/jdbc/libgmomjni64.dylib and /dev/null differ diff --git a/ixmp/backend/jdbc/libgmomjni64.so b/ixmp/backend/jdbc/libgmomjni64.so deleted file mode 100644 index bd2b16612..000000000 Binary files a/ixmp/backend/jdbc/libgmomjni64.so and /dev/null differ diff --git a/ixmp/backend/jdbc/libidxjni64.dylib b/ixmp/backend/jdbc/libidxjni64.dylib deleted file mode 100644 index 7a107dd36..000000000 Binary files a/ixmp/backend/jdbc/libidxjni64.dylib and /dev/null differ diff --git a/ixmp/backend/jdbc/libidxjni64.so b/ixmp/backend/jdbc/libidxjni64.so deleted file mode 100644 index 5ee20602e..000000000 Binary files a/ixmp/backend/jdbc/libidxjni64.so and /dev/null differ diff --git a/ixmp/backend/jdbc/libutiljni64.dylib b/ixmp/backend/jdbc/libutiljni64.dylib deleted file mode 100644 index 8e64461f0..000000000 Binary files a/ixmp/backend/jdbc/libutiljni64.dylib and /dev/null differ diff --git a/ixmp/backend/jdbc/utiljni64.dll b/ixmp/backend/jdbc/utiljni64.dll deleted file mode 100644 index 0c48f3916..000000000 Binary files a/ixmp/backend/jdbc/utiljni64.dll and /dev/null differ diff --git a/ixmp/model/gams.py b/ixmp/model/gams.py index 34d6534b2..20d8eaf35 100644 --- a/ixmp/model/gams.py +++ b/ixmp/model/gams.py @@ -5,7 +5,8 @@ import tempfile from copy import copy from pathlib import Path -from subprocess import CalledProcessError, run +from subprocess import CalledProcessError, check_output, run +from tempfile import TemporaryDirectory from typing import Any, MutableMapping, Optional from ixmp.backend import ItemType @@ -15,39 +16,8 @@ log = logging.getLogger(__name__) -def gams_version() -> Optional[str]: - """Return the GAMS version as a string, for instance "24.7.4".""" - # NB check_output(['gams'], ...) does not work, because GAMS writes directly to the - # console instead of to stdout. check_output(['gams', '-LogOption=3'], ...) does - # not work, because GAMS does not accept options without an input file to - # execute. - import os - from subprocess import check_output - from tempfile import mkdtemp - - # Create a temporary GAMS program that does nothing - tmp_dir = Path(mkdtemp()) - gms = tmp_dir / "null.gms" - gms.write_text("$exit;") - - # Execute, capturing stdout - output = check_output( - ["gams", "null", "-LogOption=3"], - shell=os.name == "nt", - cwd=tmp_dir, - universal_newlines=True, - ) - - # Clean up - gms.unlink() - gms.with_suffix(".lst").unlink() - tmp_dir.rmdir() - - # Find and return the version string - if match := re.search(r"^GAMS ([\d\.]+)\s*Copyright", output, re.MULTILINE): - return match.group(1) - else: # pragma: no cover - return None +# Singleton instance of GAMSInfo. +_GAMS_INFO: Optional["GAMSInfo"] = None #: Return codes used by GAMS, from @@ -94,6 +64,56 @@ def gams_version() -> Optional[str]: RETURN_CODE = {key % 256: value for key, value in RETURN_CODE.items()} +class GAMSInfo: + """Information about the GAMS installation.""" + + #: GAMS version as a string, for instance "24.7.4". + version: Optional[str] + + #: System directory. + system_dir: Path + + def __init__(self) -> None: + # Retrieve some `output` containing GAMS installation info + with TemporaryDirectory() as temp_dir: + # NB the following do not work: + # - check_output(['gams'], ...) —because GAMS writes directly to the console + # instead of to stdout. + # - check_output(['gams', '-LogOption=3'], ...) —because GAMS does not + # accept options without an input file to execute. + # …so instead create a GAMS source file that does nothing: + Path(temp_dir, "null.gms").write_text("$exit;") + + try: + # Execute this no-op file and capture stdout + output = check_output( + ["gams", "null.gms", "-LogOption=3"], + shell=os.name == "nt", + cwd=temp_dir, + universal_newlines=True, + ) + except FileNotFoundError as e: # pragma: no cover + log.warning(f"{e}") + output = "" + + # Parse GAMS version from the copyright line + if match := re.search(r"^GAMS ([\d\.]+)\s*Copyright", output, re.MULTILINE): + self.version = match.group(1) + else: # pragma: no cover + self.version = None + + # Parse GAMS system directory path + if match := re.search(r"^\s*SysDir (.*)", output, re.MULTILINE): + self.system_dir = Path(match.group(1)) + else: # pragma: no cover + self.system_dir = Path.cwd() + + @property + def java_api_dir(self) -> Path: + """Java API files subdirectory of :attr:`.system_dir`.""" + return self.system_dir.joinpath("apifiles", "Java", "api") + + class GAMSModel(Model): """Generic base class for :mod:`ixmp` models using `GAMS `_. @@ -379,3 +399,20 @@ def run(self, scenario): # Finished: remove the temporary directory, if any self.remove_temp_dir() + + +def gams_info() -> GAMSInfo: + """Return an instance of :class:`.GAMSInfo`.""" + # Singleton pattern; ensure there is only one instance of GAMSInfo + global _GAMS_INFO + + if _GAMS_INFO is None: + # Create the singleton + _GAMS_INFO = GAMSInfo() + + return _GAMS_INFO + + +def gams_version() -> Optional[str]: + """Return :attr:`.GAMSInfo.version`.""" + return gams_info().version