Petab Importer reforge (#1442)

* Moved creation of the amici objective function to a separate class, leaving the importer really independent of amici * Remove check gradients, as this can be done in an objective function * Added functions with a not implementedError and moved most of them to AmiciFactory * Moved import to a factory, allowing a single PEtabImporter to function with different models. Created a basic PetabSimulator supported (inefficent most likely but as a starter) * Resolve potential annotation problems. * Solved the problem of x_names not saving the fixd parameters. similar solution to amici Objective * rewrote petab testsuite to fit new design. * Adjusted text and example code for new changes * import error with petab simulator * updated code * import petab as optional * Adjusted roadrunner example and also solved issue in PetabImporterRR * Another import * Another import and spline error * Changed the documentation that is included * added test for petabSimulator with the example of basico * Fix text for check gradients and fix roadrunner notebook. * added test for petabSimulator with the example of basico * cleared output * Hopeflly renamed everything * Temporary fix for roadrunner fd objective * correct installation of basico as petab simulator in test * Removed try-except in simulator as it is to unspecific. Now requiring the parameter update function * not basico but copasi-basico * not copasi-basico but copasi-basico[petab] * Adjusted some description text * More accurate documentation * Replaced Usage of roadrunner, amici, petab strings with constants * Update doc/example/petab_import.ipynb Co-authored-by: Dilan Pathirana <[email protected]> * Update doc/example/petab_import.ipynb Co-authored-by: Fabian Fröhlich <[email protected]> * Renamed factory mentions as it was not a fitting name. * added option to pass simulator type and simulator to create_objective_creator --------- Co-authored-by: Dilan Pathirana <[email protected]> Co-authored-by: Fabian Fröhlich <[email protected]>
ICB-DCM · Sep 24, 2024 · 0264a1e · 0264a1e
1 parent 2e57687
commit 0264a1e
Show file tree

Hide file tree

Showing 29 changed files with 1,595 additions and 784 deletions.
diff --git a/doc/example/censored_data.ipynb b/doc/example/censored_data.ipynb
@@ -148,16 +148,6 @@
     "As there are no censored data specific inner options, we will pass none to the constructor."
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "model = importer.create_model(verbose=False)\n",
-    "objective = importer.create_objective(model=model)"
-   ]
-  },
   {
    "attachments": {},
    "cell_type": "markdown",
@@ -172,7 +162,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "problem = importer.create_problem(objective)\n",
+    "problem = importer.create_problem()\n",
     "\n",
     "engine = pypesto.engine.MultiProcessEngine(n_procs=3)\n",
     "\n",

diff --git a/doc/example/ordinal_data.ipynb b/doc/example/ordinal_data.ipynb
@@ -542,7 +542,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "objective = importer.create_objective(verbose=False)"
+    "factory = importer.create_objective_creator()\n",
+    "objective = factory.create_objective(verbose=False)"
    ]
   },
   {
@@ -559,7 +560,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "objective = importer.create_objective(\n",
+    "objective = factory.create_objective(\n",
     "    inner_options={\n",
     "        \"method\": \"reduced\",\n",
     "        \"reparameterized\": True,\n",

diff --git a/doc/example/petab_import.ipynb b/doc/example/petab_import.ipynb
@@ -105,7 +105,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "The model must be imported to pyPESTO and AMICI. Therefore, we create a `pypesto.PetabImporter` from the problem, and create an AMICI model."
+    "In order to import the model into pyPESTO, we additionally need a simulator. We can specify the simulator through the `simulator_type` argument. Supported simulators are e.g.`amici` and `roadrunner`. We will use AMICI as our example simulator. Therefore, we create a `pypesto.PetabImporter` from the problem. The importer itself creates a `pypesto.petab.Factory`, which is used to create the AMICI objective and model."
    ]
   },
   {
@@ -116,9 +116,10 @@
    },
    "outputs": [],
    "source": [
-    "importer = pypesto.petab.PetabImporter(petab_problem)\n",
+    "importer = pypesto.petab.PetabImporter(petab_problem, simulator_type=\"amici\")\n",
+    "factory = importer.create_objective_creator()\n",
     "\n",
-    "model = importer.create_model(verbose=False)\n",
+    "model = factory.create_model(verbose=False)\n",
     "\n",
     "# some model properties\n",
     "print(\"Model parameters:\", list(model.getParameterIds()), \"\\n\")\n",
@@ -154,7 +155,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "importer = pypesto.petab.PetabImporter.from_yaml(yaml_config)\n",
+    "importer = pypesto.petab.PetabImporter.from_yaml(yaml_config, simulator_type=\"amici\")\n",
     "problem = importer.create_problem()  # creating the problem from the importer. The objective can be found at problem.objective"
    ]
   },
@@ -178,7 +179,8 @@
     "converter_config = libsbml.SBMLLocalParameterConverter().getDefaultProperties()\n",
     "petab_problem.sbml_document.convert(converter_config)\n",
     "\n",
-    "obj = importer.create_objective()\n",
+    "factory = importer.create_objective_creator()\n",
+    "obj = factory.create_objective()\n",
     "\n",
     "# for some models, hyperparameters need to be adjusted\n",
     "# obj.amici_solver.setMaxSteps(10000)\n",

diff --git a/doc/example/roadrunner.ipynb b/doc/example/roadrunner.ipynb
@@ -49,10 +49,9 @@
     "import petab\n",
     "import pypesto.objective\n",
     "import pypesto.optimize as optimize\n",
-    "import pypesto.objective.roadrunner as pypesto_rr\n",
-    "import pypesto.sample as sample\n",
+    "import pypesto.petab\n",
     "import pypesto.visualize as visualize\n",
-    "import pypesto.profile as profile\n",
+    "import pypesto.objective.roadrunner as pypesto_rr\n",
     "from IPython.display import Markdown, display\n",
     "from pprint import pprint\n",
     "\n",
@@ -80,7 +79,7 @@
    "source": [
     "## Creating pyPESTO problem from PEtab\n",
     "\n",
-    "The [PEtab file format](https://petab.readthedocs.io/en/latest/documentation_data_format.html) stores all the necessary information to define a parameter estimation problem. This includes the model, the experimental data, the parameters to estimate, and the experimental conditions. Using the `pypesto_rr.PetabImporterRR` class, we can create a pyPESTO problem directly from a PEtab problem."
+    "The [PEtab file format](https://petab.readthedocs.io/en/latest/documentation_data_format.html) stores all the necessary information to define a parameter estimation problem. This includes the model, the experimental data, the parameters to estimate, and the experimental conditions. Using the `pypesto.petab.PetabImporter` class, we can create a pyPESTO problem directly from a PEtab problem."
    ]
   },
   {
@@ -97,7 +96,7 @@
     "petab_yaml = f\"./{model_name}/{model_name}.yaml\"\n",
     "\n",
     "petab_problem = petab.Problem.from_yaml(petab_yaml)\n",
-    "importer = pypesto_rr.PetabImporterRR(petab_problem)\n",
+    "importer = pypesto.petab.PetabImporter(petab_problem, simulator_type=\"roadrunner\")\n",
     "problem = importer.create_problem()"
    ]
   },
@@ -329,7 +328,7 @@
     "# no support for sensitivities\n",
     "try:\n",
     "    ret = problem.objective(\n",
-    "        petab_problem.get_x_nominal(fixed=False,scaled=True),\n",
+    "        petab_problem.x_nominal_free_scaled,\n",
     "        mode=\"mode_fun\",\n",
     "        return_dict=True,\n",
     "        sensi_orders=(1,),\n",
@@ -354,7 +353,7 @@
     "# support through finite differences\n",
     "try:\n",
     "    ret = objective_fd(\n",
-    "        petab_problem.get_x_nominal(fixed=False,scaled=True),\n",
+    "        petab_problem.x_nominal_scaled,\n",
     "        mode=\"mode_fun\",\n",
     "        return_dict=True,\n",
     "        sensi_orders=(1,),\n",

diff --git a/doc/example/semiquantitative_data.ipynb b/doc/example/semiquantitative_data.ipynb
@@ -193,7 +193,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "objective = importer.create_objective(verbose=False)"
+    "factory = importer.create_objective_creator()\n",
+    "objective = factory.create_objective(verbose=False)"
    ]
   },
   {
@@ -210,7 +211,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "objective = importer.create_objective(\n",
+    "objective = factory.create_objective(\n",
     "    inner_options={\n",
     "        \"spline_ratio\": 1 / 2,\n",
     "        \"min_diff_factor\": 1 / 2,\n",

diff --git a/pypesto/C.py b/pypesto/C.py
@@ -84,6 +84,10 @@ class EnsembleType(Enum):
 X0 = "x0"
 ID = "id"
 
+AMICI = "amici"
+ROADRUNNER = "roadrunner"
+PETAB = "petab"
+
 
 ###############################################################################
 # HIERARCHICAL SCALING + OFFSET

diff --git a/pypesto/objective/__init__.py b/pypesto/objective/__init__.py
@@ -8,8 +8,10 @@
 from .base import ObjectiveBase
 from .finite_difference import FD, FDDelta
 from .function import Objective
+from .petab import PetabSimulatorObjective
 from .priors import (
     NegLogParameterPriors,
     NegLogPriors,
     get_parameter_prior_dict,
 )
+from .roadrunner import RoadRunnerObjective
diff --git a/pypesto/objective/petab.py b/pypesto/objective/petab.py
@@ -0,0 +1,102 @@
+"""Objective function for PEtab models using the PEtab simulator."""
+from __future__ import annotations
+
+try:
+    import petab.v1 as petab
+    from petab.v1.simulate import Simulator as PetabSimulator
+except ImportError:
+    petab = None
+from collections import OrderedDict
+from collections.abc import Sequence
+
+import numpy as np
+
+from ..C import FVAL, MODE_FUN, MODE_RES, RES, ModeType
+from .base import ObjectiveBase, ResultDict
+
+
+class PetabSimulatorObjective(ObjectiveBase):
+    """Objective function for PEtab models using the PEtab simulator."""
+
+    def __init__(
+        self,
+        simulator: PetabSimulator,
+        x_names: Sequence[str] | None = None,
+    ):
+        """Initialize the PEtab simulator objective function.
+
+        Parameters
+        ----------
+        petab_problem:
+            The PEtab problem.
+        simulator:
+            The PEtab simulator.
+        x_names:
+            Names of optimization parameters.
+        """
+        if petab is None:
+            raise ImportError(
+                "The `petab` package is required for this objective function."
+            )
+        self.simulator = simulator
+        self.petab_problem = self.simulator.petab_problem
+        if x_names is None:
+            x_names = list(self.petab_problem.get_x_ids())
+        super().__init__(x_names=x_names)
+
+    def replace_parameters(self, x: np.ndarray):
+        """Replace the parameters in the PEtab problem with the given values.
+
+        Parameters
+        ----------
+        x:
+            Parameter vector for optimization.
+        """
+        x_dict = OrderedDict(zip(self._x_names, x))
+        x_unscaled = self.petab_problem.unscale_parameters(x_dict)
+        par_df = self.petab_problem.parameter_df
+        par_df["nominalValue"] = par_df.index.map(x_unscaled)
+        self.simulator.set_parameters(x_unscaled)
+
+    def call_unprocessed(
+        self,
+        x: np.ndarray,
+        sensi_orders: tuple[int, ...],
+        mode: ModeType,
+        return_dict: bool,
+        **kwargs,
+    ) -> ResultDict:
+        """See :meth:`ObjectiveBase.call_unprocessed`."""
+
+        self.replace_parameters(x)
+        sim_df = self.simulator.simulate(noise=False, as_measurement=False)
+        result = {}
+        result["simulations"] = sim_df
+        if mode == MODE_FUN:
+            result[FVAL] = -petab.calculate_llh(
+                measurement_dfs=self.petab_problem.measurement_df,
+                simulation_dfs=sim_df,
+                observable_dfs=self.petab_problem.observable_df,
+                parameter_dfs=self.petab_problem.parameter_df,
+            )
+        elif mode == MODE_RES:
+            result[RES] = petab.calculate_residuals(
+                measurement_dfs=self.petab_problem.measurement_df,
+                simulation_dfs=sim_df,
+                observable_dfs=self.petab_problem.observable_df,
+                parameter_dfs=self.petab_problem.parameter_df,
+            )
+        return result
+
+    def check_sensi_orders(
+        self,
+        sensi_orders: tuple[int, ...],
+        mode: ModeType,
+    ) -> bool:
+        """See :class:`ObjectiveBase` documentation."""
+        if not sensi_orders:
+            return True
+        sensi_order = max(sensi_orders)
+        max_sensi_order = 0
+
+        return sensi_order <= max_sensi_order
diff --git a/pypesto/objective/roadrunner/__init__.py b/pypesto/objective/roadrunner/__init__.py
@@ -2,6 +2,12 @@
 RoadRunner objective
 ====================
 """
+__all__ = [
+    "PetabImporterRR",
+    "RoadRunnerCalculator",
+    "ExpData",
+    "SolverOptions",
+]
 
 from .petab_importer_roadrunner import PetabImporterRR
 from .road_runner import RoadRunnerObjective

diff --git a/pypesto/objective/roadrunner/petab_importer_roadrunner.py b/pypesto/objective/roadrunner/petab_importer_roadrunner.py
@@ -9,24 +9,25 @@
 import logging
 import numbers
 import re
+import warnings
 from collections.abc import Iterable
 from pathlib import Path
 from typing import Any
 
-import libsbml
-import petab.v1 as petab
-import roadrunner
-from petab.v1.C import (
-    OBSERVABLE_FORMULA,
-    PREEQUILIBRATION_CONDITION_ID,
-    SIMULATION_CONDITION_ID,
-)
-from petab.v1.models.sbml_model import SbmlModel
-from petab.v1.parameter_mapping import ParMappingDictQuadruple
+try:
+    import petab.v1 as petab
+    from petab.v1.C import (
+        OBSERVABLE_FORMULA,
+        PREEQUILIBRATION_CONDITION_ID,
+        SIMULATION_CONDITION_ID,
+    )
+    from petab.v1.models.sbml_model import SbmlModel
+    from petab.v1.parameter_mapping import ParMappingDictQuadruple
+except ImportError:
+    petab = None
 
 import pypesto.C
 
-from ...petab.importer import PetabStartpoints
 from ...problem import Problem
 from ...startpoint import StartpointMethod
 from ..aggregated import AggregatedObjective
@@ -35,6 +36,13 @@
 from .roadrunner_calculator import RoadRunnerCalculator
 from .utils import ExpData
 
+try:
+    import libsbml
+    import roadrunner
+except ImportError:
+    roadrunner = None
+    libsbml = None
+
 logger = logging.getLogger(__name__)
 
 
@@ -62,6 +70,14 @@ def __init__(
         validate_petab:
             Flag indicating if the PEtab problem shall be validated.
         """
+        warnings.warn(
+            "The RoadRunner importer is deprecated and will be removed in "
+            "future versions. Please use the generic PetabImporter instead "
+            "with `simulator_type='roadrunner'`. Everything else will stay "
+            "same.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         self.petab_problem = petab_problem
         if validate_petab:
             if petab.lint_problem(petab_problem):
@@ -288,6 +304,7 @@ def create_objective(
             petab_problem=self.petab_problem,
             calculator=calculator,
             x_names=x_names,
+            x_ids=x_names,
         )
 
     def create_prior(self) -> NegLogParameterPriors | None:
@@ -344,6 +361,8 @@ def create_startpoint_method(self, **kwargs) -> StartpointMethod:
             Additional keyword arguments passed on to
             :meth:`pypesto.startpoint.FunctionStartpoints.__init__`.
         """
+        from ...petab.util import PetabStartpoints
+
         return PetabStartpoints(petab_problem=self.petab_problem, **kwargs)
 
     def create_problem(