Merge pull request #532 from iiasa/issue/531

Support arm64 architecture on macOS

.github/workflows/pytest.yaml

@@ -20,6 +20,7 @@ jobs:
       matrix:
         os:
         - macos-13
+        - macos-latest
         - ubuntu-latest
         - windows-latest
         python-version:
@@ -28,18 +29,31 @@ jobs:
         - "3.10"
         - "3.11"
         - "3.12"  # Latest supported by ixmp
+        gams-version:
+        # Version used until 2024-07; disabled
+        # - 25.1.1
+        # First version including a macOS arm64 distribution
+        - 43.4.1
 
         # commented: force a specific version of pandas, for e.g. pre-release
         # testing
         # pandas-version:
         # - ""
         # - "==2.0.0rc0"
 
-        exclude: [ ]
-        # # Specific version combinations that are invalid, e.g. pandas 2.0
-        # # requires Python >= 3.8
-        # - python-version: "3.7"
-        #   pandas-version: "==2.0.0rc0"
+        exclude:
+        # Specific version combinations that are invalid / not to be used
+        # No arm64 distribution for this version of GAMS
+        # - { os: macos-latest, gams-version: 25.1.1}
+        # No arm64 distributions of JPype for these Pythons
+        - { os: macos-latest, python-version: "3.8" }
+        - { os: macos-latest, python-version: "3.9" }
+        # Redundant with macos-latest
+        - { os: macos-13, python-version: "3.10" }
+        - { os: macos-13, python-version: "3.11" }
+        - { os: macos-13, python-version: "3.12" }
+        # Example: pandas 2.0 requires Python >= 3.8
+        # - { python-version: "3.7", pandas-version: "==2.0.0rc0" }
 
       fail-fast: false
 
@@ -81,7 +95,8 @@ jobs:
 
     - uses: iiasa/actions/setup-gams@main
       with:
-        version: 25.1.1
+        version: ${{ matrix.gams-version }}
+        license: ${{ secrets.GAMS_LICENSE }}
 
     - name: Set RETICULATE_PYTHON
       # Use the environment variable set by the setup-python action, above.

RELEASE_NOTES.rst

@@ -1,5 +1,11 @@
-.. Next release
-.. ============
+Next release
+============
+
+- :mod:`ixmp` locates GAMS API libraries needed for the Java code underlying :class:`.JDBCBackend` based on the system GAMS installation (:pull:`532`).
+  As a result:
+
+  - :class:`.JDBCBackend` is usable on MacOS with newer, ``arm64``-architecture processors and Python/GAMS compiled for ``arm64`` (:issue:`473`, :issue:`531`).
+  - GAMS API libraries are no longer (re-)packaged with ixmp in the directory :file:`ixmp/backend/jdbc/`.
 
 .. _v3.9.0:
 

ixmp/backend/jdbc.py

@@ -1,6 +1,7 @@
 import gc
 import logging
 import os
+import platform
 import re
 from collections import ChainMap
 from collections.abc import Iterable, Sequence
@@ -1233,18 +1234,32 @@ def start_jvm(jvmargs=None):
         .. _`JVM documentation`: https://docs.oracle.com/javase/7/docs
            /technotes/tools/windows/java.html)
     """
+    from ixmp.model.gams import gams_info
+
     if jvmargs is None:
         jvmargs = []
     if jpype.isJVMStarted():
         return
 
+    # Base directory for the classpath and library path
+    base = Path(__file__).with_name("jdbc")
+
     # Arguments
     args = jvmargs if isinstance(jvmargs, list) else [jvmargs]
 
+    # Append path to directories containing arch-specific libraries
+    uname = platform.uname()
+    paths = [
+        gams_info().java_api_dir,  # GAMS system directory
+        base.joinpath(uname.machine),  # Subdirectory of ixmp/backend/jdbc
+    ]
+    sep = ";" if uname.system == "Windows" else ":"
+    args.append(f"-Djava.library.path={sep.join(map(str, paths))}")
+
     # Keyword arguments
     kwargs = dict(
-        # Glob pattern for ixmp.jar and related Java binaries
-        classpath=str(Path(__file__).parent.joinpath("jdbc", "*")),
+        # Use ixmp.jar and related Java JAR files
+        classpath=str(base.joinpath("*")),
         # For JPype 0.7 (raises a warning) and 0.8 (default is False). 'True' causes
         # Java string objects to be converted automatically to Python str(), as expected
         # by ixmp Python code.

ixmp/model/gams.py

@@ -5,7 +5,8 @@
 import tempfile
 from copy import copy
 from pathlib import Path
-from subprocess import CalledProcessError, run
+from subprocess import CalledProcessError, check_output, run
+from tempfile import TemporaryDirectory
 from typing import Any, MutableMapping, Optional
 
 from ixmp.backend import ItemType
@@ -15,39 +16,8 @@
 log = logging.getLogger(__name__)
 
 
-def gams_version() -> Optional[str]:
-    """Return the GAMS version as a string, for instance "24.7.4"."""
-    # NB check_output(['gams'], ...) does not work, because GAMS writes directly to the
-    #    console instead of to stdout. check_output(['gams', '-LogOption=3'], ...) does
-    #    not work, because GAMS does not accept options without an input file to
-    #    execute.
-    import os
-    from subprocess import check_output
-    from tempfile import mkdtemp
-
-    # Create a temporary GAMS program that does nothing
-    tmp_dir = Path(mkdtemp())
-    gms = tmp_dir / "null.gms"
-    gms.write_text("$exit;")
-
-    # Execute, capturing stdout
-    output = check_output(
-        ["gams", "null", "-LogOption=3"],
-        shell=os.name == "nt",
-        cwd=tmp_dir,
-        universal_newlines=True,
-    )
-
-    # Clean up
-    gms.unlink()
-    gms.with_suffix(".lst").unlink()
-    tmp_dir.rmdir()
-
-    # Find and return the version string
-    if match := re.search(r"^GAMS ([\d\.]+)\s*Copyright", output, re.MULTILINE):
-        return match.group(1)
-    else:  # pragma: no cover
-        return None
+# Singleton instance of GAMSInfo.
+_GAMS_INFO: Optional["GAMSInfo"] = None
 
 
 #: Return codes used by GAMS, from
@@ -94,6 +64,56 @@ def gams_version() -> Optional[str]:
 RETURN_CODE = {key % 256: value for key, value in RETURN_CODE.items()}
 
 
+class GAMSInfo:
+    """Information about the GAMS installation."""
+
+    #: GAMS version as a string, for instance "24.7.4".
+    version: Optional[str]
+
+    #: System directory.
+    system_dir: Path
+
+    def __init__(self) -> None:
+        # Retrieve some `output` containing GAMS installation info
+        with TemporaryDirectory() as temp_dir:
+            # NB the following do not work:
+            # - check_output(['gams'], ...) —because GAMS writes directly to the console
+            #   instead of to stdout.
+            # - check_output(['gams', '-LogOption=3'], ...) —because GAMS does not
+            #   accept options without an input file to execute.
+            # …so instead create a GAMS source file that does nothing:
+            Path(temp_dir, "null.gms").write_text("$exit;")
+
+            try:
+                # Execute this no-op file and capture stdout
+                output = check_output(
+                    ["gams", "null.gms", "-LogOption=3"],
+                    shell=os.name == "nt",
+                    cwd=temp_dir,
+                    universal_newlines=True,
+                )
+            except FileNotFoundError as e:  # pragma: no cover
+                log.warning(f"{e}")
+                output = ""
+
+        # Parse GAMS version from the copyright line
+        if match := re.search(r"^GAMS ([\d\.]+)\s*Copyright", output, re.MULTILINE):
+            self.version = match.group(1)
+        else:  # pragma: no cover
+            self.version = None
+
+        # Parse GAMS system directory path
+        if match := re.search(r"^\s*SysDir (.*)", output, re.MULTILINE):
+            self.system_dir = Path(match.group(1))
+        else:  # pragma: no cover
+            self.system_dir = Path.cwd()
+
+    @property
+    def java_api_dir(self) -> Path:
+        """Java API files subdirectory of :attr:`.system_dir`."""
+        return self.system_dir.joinpath("apifiles", "Java", "api")
+
+
 class GAMSModel(Model):
     """Generic base class for :mod:`ixmp` models using `GAMS <https://gams.com>`_.
 
@@ -379,3 +399,20 @@ def run(self, scenario):
 
         # Finished: remove the temporary directory, if any
         self.remove_temp_dir()
+
+
+def gams_info() -> GAMSInfo:
+    """Return an instance of :class:`.GAMSInfo`."""
+    # Singleton pattern; ensure there is only one instance of GAMSInfo
+    global _GAMS_INFO
+
+    if _GAMS_INFO is None:
+        # Create the singleton
+        _GAMS_INFO = GAMSInfo()
+
+    return _GAMS_INFO
+
+
+def gams_version() -> Optional[str]:
+    """Return :attr:`.GAMSInfo.version`."""
+    return gams_info().version