diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e91f8e5..8b899a83 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -135,8 +135,8 @@ jobs: . /opt/conda/etc/profile.d/conda.sh conda activate tiktorch-server-env ./scripts/conda_build.sh conda-recipe-client - ./scripts/conda_build.sh pybio-core-recipe - ./scripts/conda_build.sh pybio-torch-recipe + ./scripts/conda_build.sh bioimageio-core-recipe + ./scripts/conda_build.sh bioimageio-torch-recipe ./scripts/conda_build.sh conda-recipe ./scripts/conda_build.sh conda-recipe-cpu diff --git a/environment.yml b/environment.yml index fd0e1303..2bcaeaf4 100644 --- a/environment.yml +++ b/environment.yml @@ -16,7 +16,7 @@ dependencies: - inferno=v0.4.* - pillow=6.2.1 - pyyaml=5.3 - - marshmallow=3.4.0 + - marshmallow=3.12.* - pytest=4.3.0 - grpcio>=1.31 - grpcio-tools @@ -34,6 +34,9 @@ dependencies: - scikit-learn - pip>=19 - pip: - - tensorflow + - marshmallow_union - onnxruntime - pytest-grpc==0.7.0 + - python-stdnum + - spdx-license-list + - tensorflow diff --git a/pybio-core-recipe/meta.yaml b/pybio-core-recipe/meta.yaml index d53d427d..19d69b4c 100644 --- a/pybio-core-recipe/meta.yaml +++ b/pybio-core-recipe/meta.yaml @@ -28,14 +28,14 @@ requirements: test: imports: - - pybio.spec + - bioimageio.spec about: home: https://github.com/bioimage-io/python-bioimage-io license: MIT license_family: MIT license_file: LICENSE - summary: 'Tools for parsing pybio model specification' + summary: 'Tools for parsing bioimageio model specification' doc_url: https://github.com/bioimage-io/python-bioimage-io dev_url: https://github.com/bioimage-io/python-bioimage-io diff --git a/pybio-torch-recipe/meta.yaml b/pybio-torch-recipe/meta.yaml index fbef67f2..6ba9694d 100644 --- a/pybio-torch-recipe/meta.yaml +++ b/pybio-torch-recipe/meta.yaml @@ -27,15 +27,15 @@ requirements: test: imports: - - pybio.torch - - pybio.torch.training + - bioimageio.torch + - bioimageio.torch.training about: home: https://github.com/bioimage-io/pytorch-bioimage-io license: MIT license_family: MIT license_file: LICENSE - summary: 'Pytorch related components of pybio' + summary: 'Pytorch related components of bioimageio' doc_url: https://github.com/bioimage-io/pytorch-bioimage-io dev_url: https://github.com/bioimage-io/pytorch-bioimage-io diff --git a/setup.py b/setup.py index a0ae41aa..e9044bc4 100644 --- a/setup.py +++ b/setup.py @@ -30,8 +30,8 @@ "numpy", "pyyaml", "torch", - "pybio.core @ git+ssh://git@github.com/bioimage-io/python-bioimage-io#egg=pybio.core", - "pybio.torch @ git+ssh://git@github.com/bioimage-io/pytorch-bioimage-io#egg=pybio.torch", + "bioimageio.core @ git+ssh://git@github.com/bioimage-io/python-bioimage-io#egg=bioimageio.core", + "bioimageio.torch @ git+ssh://git@github.com/bioimage-io/pytorch-bioimage-io#egg=bioimageio.torch", ], entry_points={"console_scripts": ["tiktorch=tiktorch.server.base:main"]}, # extras_require={"test": ["pytest"]}, diff --git a/tests/conftest.py b/tests/conftest.py index c2ee4cfd..d3c46e0c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,11 +15,11 @@ import pytest TEST_DATA = "data" -TEST_PYBIO_ZIPFOLDER = "unet2d" -TEST_PYBIO_ONNX = "unet2d_onnx" -TEST_PYBIO_DUMMY = "dummy" -TEST_PYBIO_TENSORFLOW_DUMMY = "dummy_tensorflow" -TEST_PYBIO_TORCHSCRIPT = "unet2d_torchscript" +TEST_BIOIMAGEIO_ZIPFOLDER = "unet2d" +TEST_BIOIMAGEIO_ONNX = "unet2d_onnx" +TEST_BIOIMAGEIO_DUMMY = "dummy" +TEST_BIOIMAGEIO_TENSORFLOW_DUMMY = "dummy_tensorflow" +TEST_BIOIMAGEIO_TORCHSCRIPT = "unet2d_torchscript" NNModel = namedtuple("NNModel", ["model", "state"]) @@ -82,8 +82,8 @@ def assert_threads_cleanup(): @pytest.fixture -def pybio_model_bytes(data_path): - zip_folder = data_path / TEST_PYBIO_ZIPFOLDER +def bioimageio_model_bytes(data_path): + zip_folder = data_path / TEST_BIOIMAGEIO_ZIPFOLDER data = io.BytesIO() with ZipFile(data, mode="w") as zip_model: for f_path in zip_folder.iterdir(): @@ -97,18 +97,18 @@ def pybio_model_bytes(data_path): @pytest.fixture -def pybio_model_zipfile(pybio_model_bytes): - with ZipFile(pybio_model_bytes, mode="r") as zf: +def bioimageio_model_zipfile(bioimageio_model_bytes): + with ZipFile(bioimageio_model_bytes, mode="r") as zf: yield zf @pytest.fixture -def pybio_dummy_model_filepath(data_path, tmpdir): - pybio_net_dir = Path(data_path) / TEST_PYBIO_DUMMY +def bioimageio_dummy_model_filepath(data_path, tmpdir): + bioimageio_net_dir = Path(data_path) / TEST_BIOIMAGEIO_DUMMY path = tmpdir / "dummy_model.zip" with ZipFile(path, mode="w") as zip_model: - for f_path in pybio_net_dir.iterdir(): + for f_path in bioimageio_net_dir.iterdir(): if str(f_path.name).startswith("__"): continue @@ -119,11 +119,11 @@ def pybio_dummy_model_filepath(data_path, tmpdir): @pytest.fixture -def pybio_dummy_model_bytes(data_path): - pybio_net_dir = Path(data_path) / TEST_PYBIO_DUMMY +def bioimageio_dummy_model_bytes(data_path): + bioimageio_net_dir = Path(data_path) / TEST_BIOIMAGEIO_DUMMY data = io.BytesIO() with ZipFile(data, mode="w") as zip_model: - for f_path in pybio_net_dir.iterdir(): + for f_path in bioimageio_net_dir.iterdir(): if str(f_path.name).startswith("__"): continue @@ -156,22 +156,22 @@ def _archive(path_to_archive): @pytest.fixture -def pybio_dummy_tensorflow_model_bytes(data_path): - pybio_net_dir = Path(data_path) / TEST_PYBIO_TENSORFLOW_DUMMY - return archive(pybio_net_dir) +def bioimageio_dummy_tensorflow_model_bytes(data_path): + bioimageio_net_dir = Path(data_path) / TEST_BIOIMAGEIO_TENSORFLOW_DUMMY + return archive(bioimageio_net_dir) @pytest.fixture -def pybio_unet2d_onnx_bytes(data_path): - pybio_net_dir = Path(data_path) / TEST_PYBIO_ONNX - return archive(pybio_net_dir) +def bioimageio_unet2d_onnx_bytes(data_path): + bioimageio_net_dir = Path(data_path) / TEST_BIOIMAGEIO_ONNX + return archive(bioimageio_net_dir) @pytest.fixture -def pybio_unet2d_onnx_test_data(data_path): - pybio_net_dir = Path(data_path) / TEST_PYBIO_ONNX - test_input = pybio_net_dir / "test_input.npy" - test_output = pybio_net_dir / "test_output.npy" +def bioimageio_unet2d_onnx_test_data(data_path): + bioimageio_net_dir = Path(data_path) / TEST_BIOIMAGEIO_ONNX + test_input = bioimageio_net_dir / "test_input.npy" + test_output = bioimageio_net_dir / "test_output.npy" return {"test_input": test_input, "test_output": test_output} @@ -184,19 +184,14 @@ def npy_zeros_file(tmpdir): @pytest.fixture -def pybio_unet2d_torchscript_bytes(data_path): - pybio_net_dir = Path(data_path) / TEST_PYBIO_TORCHSCRIPT - return archive(pybio_net_dir) +def bioimageio_unet2d_torchscript_bytes(data_path): + bioimageio_net_dir = Path(data_path) / TEST_BIOIMAGEIO_TORCHSCRIPT + return archive(bioimageio_net_dir) @pytest.fixture -def pybio_unet2d_torchscript_test_data(data_path): - pybio_net_dir = Path(data_path) / TEST_PYBIO_TORCHSCRIPT - test_input = pybio_net_dir / "test_input.npy" - test_output = pybio_net_dir / "test_output.npy" +def bioimageio_unet2d_torchscript_test_data(data_path): + bioimageio_net_dir = Path(data_path) / TEST_BIOIMAGEIO_TORCHSCRIPT + test_input = bioimageio_net_dir / "test_input.npy" + test_output = bioimageio_net_dir / "test_output.npy" return {"test_input": test_input, "test_output": test_output} - - -@pytest.fixture -def cache_path(tmp_path): - return Path(getenv("PYBIO_CACHE_PATH", tmp_path)) diff --git a/tests/data/unet2d_onnx/UNet2DNucleiBroad.model.yaml b/tests/data/unet2d_onnx/UNet2DNucleiBroad.model.yaml index 39593edc..7225c75d 100644 --- a/tests/data/unet2d_onnx/UNet2DNucleiBroad.model.yaml +++ b/tests/data/unet2d_onnx/UNet2DNucleiBroad.model.yaml @@ -57,7 +57,7 @@ outputs: language: python framework: pytorch -source: pybio.torch.models.unet2d.UNet2d +source: bioimageio.torch.models.unet2d.UNet2d kwargs: {input_channels: 1, output_channels: 1} dependencies: conda:../environment.yaml diff --git a/tests/data/unet2d_torchscript/UNet2DNucleiBroad.model.yaml b/tests/data/unet2d_torchscript/UNet2DNucleiBroad.model.yaml index c8b7a8cd..eeca5837 100644 --- a/tests/data/unet2d_torchscript/UNet2DNucleiBroad.model.yaml +++ b/tests/data/unet2d_torchscript/UNet2DNucleiBroad.model.yaml @@ -50,7 +50,7 @@ outputs: language: python framework: pytorch -source: pybio.torch.models.unet2d.UNet2d +source: bioimageio.torch.models.unet2d.UNet2d kwargs: {input_channels: 1, output_channels: 1} dependencies: conda:../environment.yaml diff --git a/tests/data/unet2d_torchscript/weights.pt b/tests/data/unet2d_torchscript/weights.pt index bfa4c536..abe68864 100644 Binary files a/tests/data/unet2d_torchscript/weights.pt and b/tests/data/unet2d_torchscript/weights.pt differ diff --git a/tests/test_server/test_grpc/test_inference_servicer.py b/tests/test_server/test_grpc/test_inference_servicer.py index 80a86e4f..ee353119 100644 --- a/tests/test_server/test_grpc/test_inference_servicer.py +++ b/tests/test_server/test_grpc/test_inference_servicer.py @@ -44,13 +44,13 @@ def method_requiring_session(self, request, grpc_stub): method_name, req = request.param return getattr(grpc_stub, method_name), req - def test_model_session_creation(self, grpc_stub, pybio_model_bytes): - model = grpc_stub.CreateModelSession(valid_model_request(pybio_model_bytes)) + def test_model_session_creation(self, grpc_stub, bioimageio_model_bytes): + model = grpc_stub.CreateModelSession(valid_model_request(bioimageio_model_bytes)) assert model.id grpc_stub.CloseModelSession(model) - def test_model_session_creation_using_upload_id(self, grpc_stub, data_store, pybio_dummy_model_bytes): - id_ = data_store.put(pybio_dummy_model_bytes.getvalue()) + def test_model_session_creation_using_upload_id(self, grpc_stub, data_store, bioimageio_dummy_model_bytes): + id_ = data_store.put(bioimageio_dummy_model_bytes.getvalue()) rq = inference_pb2.CreateModelSessionRequest(model_uri=f"upload://{id_}", deviceIds=["cpu"]) model = grpc_stub.CreateModelSession(rq) @@ -103,12 +103,12 @@ def test_if_model_create_fails_devices_are_released(self, grpc_stub): if model: grpc_stub.CloseModelSession(model) - def test_use_device(self, grpc_stub, pybio_model_bytes): + def test_use_device(self, grpc_stub, bioimageio_model_bytes): device_by_id = self._query_devices(grpc_stub) assert "cpu" in device_by_id assert inference_pb2.Device.Status.AVAILABLE == device_by_id["cpu"].status - model = grpc_stub.CreateModelSession(valid_model_request(pybio_model_bytes, device_ids=["cpu"])) + model = grpc_stub.CreateModelSession(valid_model_request(bioimageio_model_bytes, device_ids=["cpu"])) device_by_id = self._query_devices(grpc_stub) assert "cpu" in device_by_id @@ -116,15 +116,15 @@ def test_use_device(self, grpc_stub, pybio_model_bytes): grpc_stub.CloseModelSession(model) - def test_using_same_device_fails(self, grpc_stub, pybio_model_bytes): - model = grpc_stub.CreateModelSession(valid_model_request(pybio_model_bytes, device_ids=["cpu"])) + def test_using_same_device_fails(self, grpc_stub, bioimageio_model_bytes): + model = grpc_stub.CreateModelSession(valid_model_request(bioimageio_model_bytes, device_ids=["cpu"])) with pytest.raises(grpc.RpcError): - model = grpc_stub.CreateModelSession(valid_model_request(pybio_model_bytes, device_ids=["cpu"])) + model = grpc_stub.CreateModelSession(valid_model_request(bioimageio_model_bytes, device_ids=["cpu"])) grpc_stub.CloseModelSession(model) - def test_closing_session_releases_devices(self, grpc_stub, pybio_model_bytes): - model = grpc_stub.CreateModelSession(valid_model_request(pybio_model_bytes, device_ids=["cpu"])) + def test_closing_session_releases_devices(self, grpc_stub, bioimageio_model_bytes): + model = grpc_stub.CreateModelSession(valid_model_request(bioimageio_model_bytes, device_ids=["cpu"])) device_by_id = self._query_devices(grpc_stub) assert "cpu" in device_by_id @@ -138,8 +138,8 @@ def test_closing_session_releases_devices(self, grpc_stub, pybio_model_bytes): class TestGetLogs: - def test_returns_ack_message(self, pybio_model_bytes, grpc_stub): - model = grpc_stub.CreateModelSession(valid_model_request(pybio_model_bytes)) + def test_returns_ack_message(self, bioimageio_model_bytes, grpc_stub): + model = grpc_stub.CreateModelSession(valid_model_request(bioimageio_model_bytes)) resp = grpc_stub.GetLogs(inference_pb2.Empty()) record = next(resp) assert inference_pb2.LogEntry.Level.INFO == record.level @@ -154,8 +154,8 @@ def test_call_fails_with_unknown_model_session_id(self, grpc_stub): assert grpc.StatusCode.FAILED_PRECONDITION == e.value.code() assert "model-session with id myid1 doesn't exist" in e.value.details() - def test_call_predict(self, grpc_stub, pybio_dummy_model_bytes): - model = grpc_stub.CreateModelSession(valid_model_request(pybio_dummy_model_bytes)) + def test_call_predict(self, grpc_stub, bioimageio_dummy_model_bytes): + model = grpc_stub.CreateModelSession(valid_model_request(bioimageio_dummy_model_bytes)) arr = xr.DataArray(np.arange(32 * 32).reshape(1, 32, 32), dims=("c", "x", "y")) expected = arr + 1 diff --git a/tests/test_server/test_predict.py b/tests/test_server/test_predict.py index 6762b1a3..cd7a1113 100644 --- a/tests/test_server/test_predict.py +++ b/tests/test_server/test_predict.py @@ -19,13 +19,13 @@ def npy_zeros_file(tmpdir): return path -def test_running_predict_with_valid_arguments(testdir, pybio_dummy_model_filepath, npy_zeros_file, output_path): +def test_running_predict_with_valid_arguments(testdir, bioimageio_dummy_model_filepath, npy_zeros_file, output_path): result = testdir.run( "python", "-m", "tiktorch.server.predict", "--model", - pybio_dummy_model_filepath, + bioimageio_dummy_model_filepath, npy_zeros_file, "--output", output_path, @@ -41,20 +41,20 @@ def test_running_predict_fails_when_model_unspecified(testdir, npy_zeros_file, o assert result.ret != 0 -def test_running_predict_fails_when_no_images_specified(testdir, pybio_dummy_model_filepath, output_path): +def test_running_predict_fails_when_no_images_specified(testdir, bioimageio_dummy_model_filepath, output_path): result = testdir.run( - "python", "-m", "tiktorch.server.predict", "--model", pybio_dummy_model_filepath, "--output", output_path + "python", "-m", "tiktorch.server.predict", "--model", bioimageio_dummy_model_filepath, "--output", output_path ) assert result.ret != 0 -def test_running_predict_fails_when_invalid_image_specified(testdir, pybio_dummy_model_filepath, output_path): +def test_running_predict_fails_when_invalid_image_specified(testdir, bioimageio_dummy_model_filepath, output_path): result = testdir.run( "python", "-m", "tiktorch.server.predict", "--model", - pybio_dummy_model_filepath, + bioimageio_dummy_model_filepath, "nonexisting", "--output", output_path, @@ -77,9 +77,9 @@ def test_running_predict_fails_without_when_model_file_does_not_exist(testdir, t def test_running_predict_failes_when_output_is_unspecified( - testdir, pybio_dummy_model_filepath, npy_zeros_file, output_path + testdir, bioimageio_dummy_model_filepath, npy_zeros_file, output_path ): result = testdir.run( - "python", "-m", "tiktorch.server.predict", "--model", pybio_dummy_model_filepath, npy_zeros_file + "python", "-m", "tiktorch.server.predict", "--model", bioimageio_dummy_model_filepath, npy_zeros_file ) assert result.ret != 0 diff --git a/tests/test_server/test_prediction_pipeline/test_prediction_pipeline.py b/tests/test_server/test_prediction_pipeline/test_prediction_pipeline.py index 6a327474..426015f1 100644 --- a/tests/test_server/test_prediction_pipeline/test_prediction_pipeline.py +++ b/tests/test_server/test_prediction_pipeline/test_prediction_pipeline.py @@ -7,32 +7,34 @@ from tiktorch.server.reader import eval_model_zip -def test_eval_onnx_model_zip_predict(pybio_unet2d_onnx_bytes, pybio_unet2d_onnx_test_data, cache_path): - with ZipFile(pybio_unet2d_onnx_bytes) as zf: - adapter = eval_model_zip(zf, devices=["cpu"], cache_path=cache_path, preserve_batch_dim=True) - test_input = xarray.DataArray(np.load(pybio_unet2d_onnx_test_data["test_input"]), dims=("b", "c", "x", "y")) +def test_eval_onnx_model_zip_predict(bioimageio_unet2d_onnx_bytes, bioimageio_unet2d_onnx_test_data): + with ZipFile(bioimageio_unet2d_onnx_bytes) as zf: + adapter = eval_model_zip(zf, devices=["cpu"], preserve_batch_dim=True) + test_input = xarray.DataArray( + np.load(bioimageio_unet2d_onnx_test_data["test_input"]), dims=("b", "c", "x", "y") + ) # TODO: Figure out why test output doesn't match result adapter.forward(test_input) def test_eval_torchscript_model_zip_predict( - pybio_unet2d_torchscript_bytes, pybio_unet2d_torchscript_test_data, cache_path + bioimageio_unet2d_torchscript_bytes, bioimageio_unet2d_torchscript_test_data ): - with ZipFile(pybio_unet2d_torchscript_bytes) as zf: - pipeline = eval_model_zip(zf, devices=["cpu"], cache_path=cache_path, preserve_batch_dim=True) + with ZipFile(bioimageio_unet2d_torchscript_bytes) as zf: + pipeline = eval_model_zip(zf, devices=["cpu"], preserve_batch_dim=True) test_input = xarray.DataArray( - np.load(pybio_unet2d_torchscript_test_data["test_input"]).astype(np.float32), dims=("b", "c", "x", "y") + np.load(bioimageio_unet2d_torchscript_test_data["test_input"]).astype(np.float32), dims=("b", "c", "x", "y") ) - test_output = np.load(pybio_unet2d_torchscript_test_data["test_output"]) + test_output = np.load(bioimageio_unet2d_torchscript_test_data["test_output"]) result = pipeline.forward(test_input) assert_array_almost_equal(result.data, test_output, decimal=4) def test_eval_model_zip_metadata_no_batch_dim( - pybio_unet2d_torchscript_bytes, pybio_unet2d_torchscript_test_data, cache_path + bioimageio_unet2d_torchscript_bytes, bioimageio_unet2d_torchscript_test_data ): - with ZipFile(pybio_unet2d_torchscript_bytes) as zf: - pipeline = eval_model_zip(zf, devices=["cpu"], cache_path=cache_path, preserve_batch_dim=False) + with ZipFile(bioimageio_unet2d_torchscript_bytes) as zf: + pipeline = eval_model_zip(zf, devices=["cpu"], preserve_batch_dim=False) assert pipeline.name == "UNet 2D Nuclei Broad" assert pipeline.input_axes == "cyx" assert pipeline.output_axes == "cyx" @@ -40,9 +42,9 @@ def test_eval_model_zip_metadata_no_batch_dim( assert pipeline.halo == [("c", 0), ("y", 32), ("x", 32)] -def test_eval_model_zip(pybio_model_bytes, cache_path): - with ZipFile(pybio_model_bytes) as zf: - pipeline = eval_model_zip(zf, devices=["cpu"], cache_path=cache_path, preserve_batch_dim=True) +def test_eval_model_zip(bioimageio_model_bytes): + with ZipFile(bioimageio_model_bytes) as zf: + pipeline = eval_model_zip(zf, devices=["cpu"], preserve_batch_dim=True) assert pipeline.input_axes == "bcyx" assert pipeline.output_axes == "bcyx" @@ -51,10 +53,10 @@ def test_eval_model_zip(pybio_model_bytes, cache_path): def test_eval_model_zip_metadata_with_batch_dim( - pybio_unet2d_torchscript_bytes, pybio_unet2d_torchscript_test_data, cache_path + bioimageio_unet2d_torchscript_bytes, bioimageio_unet2d_torchscript_test_data ): - with ZipFile(pybio_unet2d_torchscript_bytes) as zf: - pipeline = eval_model_zip(zf, devices=["cpu"], cache_path=cache_path, preserve_batch_dim=True) + with ZipFile(bioimageio_unet2d_torchscript_bytes) as zf: + pipeline = eval_model_zip(zf, devices=["cpu"], preserve_batch_dim=True) assert pipeline.input_axes == "bcyx" assert pipeline.output_axes == "bcyx" assert pipeline.input_shape == [("b", 1), ("c", 1), ("y", 512), ("x", 512)] diff --git a/tests/test_server/test_prediction_pipeline/test_preprocessing.py b/tests/test_server/test_prediction_pipeline/test_preprocessing.py index 01a8ca8d..709eed2d 100644 --- a/tests/test_server/test_prediction_pipeline/test_preprocessing.py +++ b/tests/test_server/test_prediction_pipeline/test_preprocessing.py @@ -1,7 +1,7 @@ import numpy as np import pytest import xarray as xr -from pybio.spec.nodes import Preprocessing +from bioimageio.spec.nodes import Preprocessing from tiktorch.server.prediction_pipeline._preprocessing import ADD_BATCH_DIM, make_preprocessing diff --git a/tests/test_server/test_reader.py b/tests/test_server/test_reader.py index 5dbbad8f..b23154b9 100644 --- a/tests/test_server/test_reader.py +++ b/tests/test_server/test_reader.py @@ -19,15 +19,15 @@ def test_guess_model_path_without_model_file(paths): assert guess_model_path(paths) is None -def test_eval_model_zip(pybio_model_bytes, cache_path): - with ZipFile(pybio_model_bytes) as zf: - exemplum = eval_model_zip(zf, devices=["cpu"], cache_path=cache_path) +def test_eval_model_zip(bioimageio_model_bytes): + with ZipFile(bioimageio_model_bytes) as zf: + exemplum = eval_model_zip(zf, devices=["cpu"]) assert isinstance(exemplum, PredictionPipeline) -def test_eval_tensorflow_model_zip(pybio_dummy_tensorflow_model_bytes, cache_path): - with ZipFile(pybio_dummy_tensorflow_model_bytes) as zf: - pipeline = eval_model_zip(zf, devices=["cpu"], cache_path=cache_path) +def test_eval_tensorflow_model_zip(bioimageio_dummy_tensorflow_model_bytes): + with ZipFile(bioimageio_dummy_tensorflow_model_bytes) as zf: + pipeline = eval_model_zip(zf, devices=["cpu"]) assert isinstance(pipeline, PredictionPipeline) test_input = xr.DataArray(np.zeros(shape=(1, 128, 128)), dims=("c", "y", "x")) @@ -38,13 +38,13 @@ def test_eval_tensorflow_model_zip(pybio_dummy_tensorflow_model_bytes, cache_pat xr.testing.assert_equal(result, test_output) -def test_eval_torchscript_model_zip(pybio_unet2d_torchscript_bytes, cache_path): - with ZipFile(pybio_unet2d_torchscript_bytes) as zf: - adapter = eval_model_zip(zf, devices=["cpu"], cache_path=cache_path) +def test_eval_torchscript_model_zip(bioimageio_unet2d_torchscript_bytes): + with ZipFile(bioimageio_unet2d_torchscript_bytes) as zf: + adapter = eval_model_zip(zf, devices=["cpu"]) assert isinstance(adapter, PredictionPipeline) -def test_eval_onnx_model_zip(pybio_unet2d_onnx_bytes, cache_path): - with ZipFile(pybio_unet2d_onnx_bytes) as zf: - adapter = eval_model_zip(zf, devices=["cpu"], cache_path=cache_path) +def test_eval_onnx_model_zip(bioimageio_unet2d_onnx_bytes): + with ZipFile(bioimageio_unet2d_onnx_bytes) as zf: + adapter = eval_model_zip(zf, devices=["cpu"]) assert isinstance(adapter, PredictionPipeline) diff --git a/tiktorch/server/prediction_pipeline/_model_adapters/_model_adapter.py b/tiktorch/server/prediction_pipeline/_model_adapters/_model_adapter.py index 1dc8c658..0790e6e5 100644 --- a/tiktorch/server/prediction_pipeline/_model_adapters/_model_adapter.py +++ b/tiktorch/server/prediction_pipeline/_model_adapters/_model_adapter.py @@ -2,7 +2,7 @@ from typing import Callable, List, Optional, Type import xarray as xr -from pybio.spec import nodes +from bioimageio.spec import nodes #: Known weigh types in order of priority #: First match wins @@ -15,12 +15,7 @@ class ModelAdapter(abc.ABC): """ @abc.abstractmethod - def __init__( - self, - *, - pybio_model: nodes.Model, - devices=List[str], - ): + def __init__(self, *, bioimageio_model: nodes.Model, devices=List[str]): ... @abc.abstractmethod @@ -56,15 +51,15 @@ def get_weight_formats() -> List[str]: def create_model_adapter( - *, pybio_model: nodes.Model, devices=List[str], weight_format: Optional[str] = None + *, bioimageio_model: nodes.Model, devices=List[str], weight_format: Optional[str] = None ) -> ModelAdapter: """ Creates model adapter based on the passed spec Note: All specific adapters should happen inside this function to prevent different framework initializations interfering with each other """ - spec = pybio_model - weights = pybio_model.weights + spec = bioimageio_model + weights = bioimageio_model.weights weight_formats = get_weight_formats() if weight_format is not None: @@ -75,7 +70,7 @@ def create_model_adapter( for weight in weight_formats: if weight in weights: adapter_cls = _get_model_adapter(weight) - return adapter_cls(pybio_model=pybio_model, devices=devices) + return adapter_cls(bioimageio_model=bioimageio_model, devices=devices) raise NotImplementedError(f"No supported weight_formats in {spec.weights.keys()}") diff --git a/tiktorch/server/prediction_pipeline/_model_adapters/_onnx_model_adapter.py b/tiktorch/server/prediction_pipeline/_model_adapters/_onnx_model_adapter.py index f12aab7a..53c7cc16 100644 --- a/tiktorch/server/prediction_pipeline/_model_adapters/_onnx_model_adapter.py +++ b/tiktorch/server/prediction_pipeline/_model_adapters/_onnx_model_adapter.py @@ -3,7 +3,7 @@ import onnxruntime as rt import xarray as xr -from pybio.spec import nodes +from bioimageio.spec import nodes from tiktorch.server.prediction_pipeline._model_adapters._model_adapter import ModelAdapter @@ -11,13 +11,8 @@ class ONNXModelAdapter(ModelAdapter): - def __init__( - self, - *, - pybio_model: nodes.Model, - devices=List[str], - ): - spec = pybio_model + def __init__(self, *, bioimageio_model: nodes.Model, devices=List[str]): + spec = bioimageio_model self.name = spec.name if len(spec.inputs) != 1 or len(spec.outputs) != 1: diff --git a/tiktorch/server/prediction_pipeline/_model_adapters/_pytorch_model_adapter.py b/tiktorch/server/prediction_pipeline/_model_adapters/_pytorch_model_adapter.py index d14ecd59..a170be8b 100644 --- a/tiktorch/server/prediction_pipeline/_model_adapters/_pytorch_model_adapter.py +++ b/tiktorch/server/prediction_pipeline/_model_adapters/_pytorch_model_adapter.py @@ -3,8 +3,8 @@ import torch import xarray as xr -from pybio.spec import nodes -from pybio.spec.utils import get_instance +from bioimageio.spec import nodes +from bioimageio.spec.utils import get_instance from ._model_adapter import ModelAdapter @@ -12,15 +12,10 @@ class PytorchModelAdapter(ModelAdapter): - def __init__( - self, - *, - pybio_model: nodes.Model, - devices=Sequence[str], - ): - self._internal_output_axes = pybio_model.outputs[0].axes - spec = pybio_model - self.model = get_instance(pybio_model) + def __init__(self, *, bioimageio_model: nodes.Model, devices=Sequence[str]): + self._internal_output_axes = bioimageio_model.outputs[0].axes + spec = bioimageio_model + self.model = get_instance(bioimageio_model) self.devices = [torch.device(d) for d in devices] self.model.to(self.devices[0]) assert isinstance(self.model, torch.nn.Module) diff --git a/tiktorch/server/prediction_pipeline/_model_adapters/_tensorflow_model_adapter.py b/tiktorch/server/prediction_pipeline/_model_adapters/_tensorflow_model_adapter.py index 5c0e1e73..6b51affc 100644 --- a/tiktorch/server/prediction_pipeline/_model_adapters/_tensorflow_model_adapter.py +++ b/tiktorch/server/prediction_pipeline/_model_adapters/_tensorflow_model_adapter.py @@ -3,20 +3,15 @@ import numpy as np import tensorflow as tf import xarray as xr -from pybio.spec import nodes -from pybio.spec.utils import get_instance +from bioimageio.spec import nodes +from bioimageio.spec.utils import get_instance from ._model_adapter import ModelAdapter class TensorflowModelAdapter(ModelAdapter): - def __init__( - self, - *, - pybio_model: nodes.Model, - devices=List[str], - ): - spec = pybio_model + def __init__(self, *, bioimageio_model: nodes.Model, devices=List[str]): + spec = bioimageio_model self.name = spec.name spec.inputs[0] @@ -24,7 +19,7 @@ def __init__( # FIXME: TF probably uses different axis names self._internal_output_axes = _output.axes - self.model = get_instance(pybio_model) + self.model = get_instance(bioimageio_model) self.devices = [] tf_model = tf.keras.models.load_model(spec.weights["tensorflow_saved_model_bundle"].source) self.model.set_model(tf_model) diff --git a/tiktorch/server/prediction_pipeline/_model_adapters/_torchscript_model_adapter.py b/tiktorch/server/prediction_pipeline/_model_adapters/_torchscript_model_adapter.py index e61f4a0d..5708c0d9 100644 --- a/tiktorch/server/prediction_pipeline/_model_adapters/_torchscript_model_adapter.py +++ b/tiktorch/server/prediction_pipeline/_model_adapters/_torchscript_model_adapter.py @@ -3,19 +3,14 @@ import numpy as np import torch import xarray as xr -from pybio.spec import nodes +from bioimageio.spec import nodes from ._model_adapter import ModelAdapter class TorchscriptModelAdapter(ModelAdapter): - def __init__( - self, - *, - pybio_model: nodes.Model, - devices=List[str], - ): - spec = pybio_model + def __init__(self, *, bioimageio_model: nodes.Model, devices=List[str]): + spec = bioimageio_model self.name = spec.name _input = spec.inputs[0] diff --git a/tiktorch/server/prediction_pipeline/_postprocessing.py b/tiktorch/server/prediction_pipeline/_postprocessing.py index 4bb343e0..d12983b0 100644 --- a/tiktorch/server/prediction_pipeline/_postprocessing.py +++ b/tiktorch/server/prediction_pipeline/_postprocessing.py @@ -1,7 +1,7 @@ from typing import List import xarray as xr -from pybio.spec.nodes import Postprocessing +from bioimageio.spec.nodes import Postprocessing from ._preprocessing import chain from ._types import Transform @@ -21,10 +21,7 @@ def sigmoid(tensor: xr.DataArray, **kwargs): return 1 / (1 + xr.ufuncs.exp(-tensor)) -KNOWN_POSTPROCESSING = { - "__tiktorch_remove_batch_dim": remove_batch_dim, - "sigmoid": sigmoid, -} +KNOWN_POSTPROCESSING = {"__tiktorch_remove_batch_dim": remove_batch_dim, "sigmoid": sigmoid} def make_postprocessing(spec: List[Postprocessing]) -> Transform: diff --git a/tiktorch/server/prediction_pipeline/_prediction_pipeline.py b/tiktorch/server/prediction_pipeline/_prediction_pipeline.py index 7ad78873..5807d002 100644 --- a/tiktorch/server/prediction_pipeline/_prediction_pipeline.py +++ b/tiktorch/server/prediction_pipeline/_prediction_pipeline.py @@ -2,7 +2,7 @@ from typing import Callable, List, Optional, Tuple import xarray as xr -from pybio.spec import nodes +from bioimageio.spec import nodes from ._model_adapters import ModelAdapter, create_model_adapter from ._postprocessing import REMOVE_BATCH_DIM, make_postprocessing @@ -129,11 +129,7 @@ def set_max_num_iterations(self, val: int) -> None: def create_prediction_pipeline( - *, - pybio_model: nodes.Model, - devices=List[str], - preserve_batch_dim=False, - weight_format: Optional[str] = None, + *, bioimageio_model: nodes.Model, devices=List[str], preserve_batch_dim=False, weight_format: Optional[str] = None ) -> PredictionPipeline: """ Creates prediction pipeline which includes: @@ -141,14 +137,14 @@ def create_prediction_pipeline( * model prediction * postprocessing """ - if len(pybio_model.inputs) != 1 or len(pybio_model.outputs) != 1: + if len(bioimageio_model.inputs) != 1 or len(bioimageio_model.outputs) != 1: raise NotImplementedError("Only models with single input and output are supported") model_adapter: ModelAdapter = create_model_adapter( - pybio_model=pybio_model, devices=devices, weight_format=weight_format + bioimageio_model=bioimageio_model, devices=devices, weight_format=weight_format ) - input = pybio_model.inputs[0] + input = bioimageio_model.inputs[0] input_shape = input.shape input_axes = input.axes preprocessing_spec = input.preprocessing.copy() @@ -161,9 +157,9 @@ def create_prediction_pipeline( input_named_shape = list(zip(input_axes, input_shape)) preprocessing: Transform = make_preprocessing(preprocessing_spec) - output = pybio_model.outputs[0] + output = bioimageio_model.outputs[0] halo_shape = output.halo or [0 for _ in output.axes] - output_axes = pybio_model.outputs[0].axes + output_axes = bioimageio_model.outputs[0].axes postprocessing_spec = output.postprocessing.copy() if has_batch_dim(output_axes) and not preserve_batch_dim: postprocessing_spec.append(REMOVE_BATCH_DIM) @@ -174,7 +170,7 @@ def create_prediction_pipeline( postprocessing: Transform = make_postprocessing(postprocessing_spec) return _PredictionPipelineImpl( - name=pybio_model.name, + name=bioimageio_model.name, input_axes=input_axes, input_shape=input_named_shape, output_axes=output_axes, diff --git a/tiktorch/server/prediction_pipeline/_preprocessing.py b/tiktorch/server/prediction_pipeline/_preprocessing.py index 4c0b819e..c779da69 100644 --- a/tiktorch/server/prediction_pipeline/_preprocessing.py +++ b/tiktorch/server/prediction_pipeline/_preprocessing.py @@ -1,7 +1,7 @@ from typing import List import xarray as xr -from pybio.spec.nodes import Preprocessing +from bioimageio.spec.nodes import Preprocessing from ._types import Transform diff --git a/tiktorch/server/reader.py b/tiktorch/server/reader.py index de816e64..37039583 100644 --- a/tiktorch/server/reader.py +++ b/tiktorch/server/reader.py @@ -5,7 +5,7 @@ from typing import List, Optional, Sequence from zipfile import ZipFile -from pybio import spec +from bioimageio import spec from tiktorch.server.prediction_pipeline import PredictionPipeline, create_prediction_pipeline @@ -21,12 +21,8 @@ def guess_model_path(file_names: List[str]) -> Optional[str]: return None -def eval_model_zip( - model_zip: ZipFile, devices: Sequence[str], cache_path: Optional[Path] = None, *, preserve_batch_dim=False -) -> PredictionPipeline: +def eval_model_zip(model_zip: ZipFile, devices: Sequence[str], *, preserve_batch_dim=False) -> PredictionPipeline: temp_path = Path(tempfile.mkdtemp(prefix="tiktorch_")) - if cache_path is None: - cache_path = temp_path / "cache" model_zip.extractall(temp_path) @@ -36,8 +32,10 @@ def eval_model_zip( "Model config file not found, make sure that .model.yaml file in the root of your model archive" ) - pybio_model = spec.load_and_resolve_spec(spec_file_str) - ret = create_prediction_pipeline(pybio_model=pybio_model, devices=devices, preserve_batch_dim=preserve_batch_dim) + bioimageio_model = spec.load_and_resolve_spec(Path(spec_file_str)) + ret = create_prediction_pipeline( + bioimageio_model=bioimageio_model, devices=devices, preserve_batch_dim=preserve_batch_dim + ) def _on_error(function, path, exc_info): logger.warning("Failed to delete temp directory %s", path) diff --git a/tiktorch/server/session/process.py b/tiktorch/server/session/process.py index ed4e2c03..9157bb72 100644 --- a/tiktorch/server/session/process.py +++ b/tiktorch/server/session/process.py @@ -1,12 +1,10 @@ import dataclasses import io import multiprocessing as _mp -import os import uuid import zipfile from concurrent.futures import Future from multiprocessing.connection import Connection -from pathlib import Path from typing import List, Optional, Tuple import numpy @@ -33,12 +31,8 @@ class ModelInfo: class ModelSessionProcess(IRPCModelSession): def __init__(self, model_zip: bytes, devices: List[str]) -> None: - cache_path = os.getenv("PYBIO_CACHE_PATH", None) - if cache_path is not None: - cache_path = Path(cache_path) - with zipfile.ZipFile(io.BytesIO(model_zip)) as model_file: - self._model = eval_model_zip(model_file, devices, cache_path=cache_path) + self._model = eval_model_zip(model_file, devices) self._datasets = {} self._worker = base.SessionBackend(self._model) diff --git a/tiktorch/server/test_model.py b/tiktorch/server/test_model.py index b9fc5a68..6ab37c7d 100644 --- a/tiktorch/server/test_model.py +++ b/tiktorch/server/test_model.py @@ -10,8 +10,8 @@ import numpy as np import xarray as xr +from bioimageio import spec from numpy.testing import assert_array_almost_equal -from pybio import spec from .prediction_pipeline import create_prediction_pipeline, get_weight_formats from .reader import guess_model_path @@ -47,20 +47,22 @@ def main(): # try opening model from model.zip try: with ZipFile(args.model, "r") as model_zip: - pybio_model, cache_path = _load_from_zip(model_zip) + bioimageio_model, cache_path = _load_from_zip(model_zip) # otherwise open from model.yaml except BadZipFile: spec_path = os.path.abspath(args.model) - pybio_model = spec.load_and_resolve_spec(spec_path) + bioimageio_model = spec.load_and_resolve_spec(spec_path) cache_path = None model = create_prediction_pipeline( - pybio_model=pybio_model, devices=["cpu"], weight_format=args.weight_format, preserve_batch_dim=True + bioimageio_model=bioimageio_model, devices=["cpu"], weight_format=args.weight_format, preserve_batch_dim=True ) - input_args = [load_data(inp, inp_spec) for inp, inp_spec in zip(pybio_model.test_inputs, pybio_model.inputs)] + input_args = [ + load_data(inp, inp_spec) for inp, inp_spec in zip(bioimageio_model.test_inputs, bioimageio_model.inputs) + ] expected_outputs = [ - load_data(out, out_spec) for out, out_spec in zip(pybio_model.test_outputs, pybio_model.outputs) + load_data(out, out_spec) for out, out_spec in zip(bioimageio_model.test_outputs, bioimageio_model.outputs) ] results = [model.forward(*input_args)] diff --git a/vendor/python-bioimage-io b/vendor/python-bioimage-io index 9716ad46..7d47a8cd 160000 --- a/vendor/python-bioimage-io +++ b/vendor/python-bioimage-io @@ -1 +1 @@ -Subproject commit 9716ad46d8b357dbb76647d857191d135a118ba1 +Subproject commit 7d47a8cdf311348d9ab2ceca13fd925f708c4218 diff --git a/vendor/pytorch-bioimage-io b/vendor/pytorch-bioimage-io index 53373c45..25f6bac5 160000 --- a/vendor/pytorch-bioimage-io +++ b/vendor/pytorch-bioimage-io @@ -1 +1 @@ -Subproject commit 53373c45d3833b35657914953aaac544ec794a7e +Subproject commit 25f6bac5a22d8a76553bd4484a515f634bcb9ee2