diff --git a/CHANGELOG.md b/CHANGELOG.md index b674e8f2..6b48e975 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Open Prediction Service HUB +## 2.8.0 _2022/11/28_ + +* [DBACLD-47417] Extend OPS to support downloading of Ruleset Models + ## 2.7.1 _2022/09/15_ * [DBACLD-58320] Fix ads ml service rule import diff --git a/open-prediction-service.yaml b/open-prediction-service.yaml index a6ddbfcc..40273f2b 100644 --- a/open-prediction-service.yaml +++ b/open-prediction-service.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Open Prediction Service - version: 2.7.4-SNAPSHOT + version: 2.8.0-SNAPSHOT description: The Open Prediction Service API is an effort to provide an Open API that enables unsupported native ML Providers in Decision Designer or Decision Runtime. tags: - name: info @@ -213,6 +213,47 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + /models/{model_id}/download: + get: + tags: + - discover + summary: Download model binary + operationId: get_binary_by_id + parameters: + - $ref: '#/components/parameters/ModelIDParam' + responses: + '200': + description: Successful Response + content: + # A binary file: + application/octet-stream: {} + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /models/{model_id}/metadata: + get: + tags: + - discover + summary: Get parsed model metadata + operationId: get_metadata_by_id + parameters: + - $ref: '#/components/parameters/ModelIDParam' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/AdditionalModelInfo' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' /endpoints: get: tags: @@ -823,13 +864,15 @@ components: unknown_file_size: type: boolean default: false - description: Whether the user can upload a file whose length is unknown at the time of the request. + description: Whether the user can upload a file whose length(Content-Length in HTTP header) is unknown at the time of the request. example: capabilities: - info - discover - manage - prediction + - download + - metadata managed_capabilities: supported_input_data_structure: - "auto" @@ -854,6 +897,10 @@ components: - discover - manage - run + # Download model binary + - download + # Inspect model specific metadata, for example for PMML model return subType (Scorecard, Ruleset) + - metadata Parameter: title: Parameter description: Parameter for ml model invocation @@ -907,6 +954,21 @@ components: target: - rel: endpoint href: 'http://open-prediction-service.org/endpoints/8c2af534-cdce-11ea-87d0-0242ac130003' + AdditionalModelInfo: + type: object + title: Additional Model Information + required: + - modelPackage + - modelType + properties: + modelPackage: + type: string + description: The file format of binary model + example: pmml + modelType: + type: string + description: Model type of binary model + example: RuleSetModel parameters: ModelIDParam: in: path diff --git a/ops-client-sdk/pom.xml b/ops-client-sdk/pom.xml index 8820b1d1..cf4777a9 100644 --- a/ops-client-sdk/pom.xml +++ b/ops-client-sdk/pom.xml @@ -6,7 +6,7 @@ com.ibm.decision ops-client-sdk - 2.7.4-SNAPSHOT + 2.8.0-SNAPSHOT 2020 diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/capabilities.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/capabilities.py index b3f9f0b5..6e9e92fb 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/capabilities.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/capabilities.py @@ -35,7 +35,9 @@ async def server_capabilities() -> typing.Dict[typing.Text, typing.Any]: ops_model.Capability.info, ops_model.Capability.discover, ops_model.Capability.manage, - ops_model.Capability.run + ops_model.Capability.run, + ops_model.Capability.download, + ops_model.Capability.metadata ], 'managed_capabilities': { 'supported_input_data_structure': ['auto', 'DataFrame', 'ndarray', 'DMatrix', 'list'], diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index 0d8cb612..7eebb10f 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -17,6 +17,7 @@ import logging import typing +import io import fastapi import fastapi.encoders as encoders @@ -28,10 +29,12 @@ import app.crud as crud import app.gen.schemas.ops_schemas as ops_schemas import app.runtime.model_upload as app_model_upload +import app.runtime.inspection as app_runtime_inspection import app.schemas as schemas import app.schemas.binary_config as app_binary_config import app.schemas.impl as impl import app.core.configuration as app_conf +import app.models as models router = fastapi.APIRouter() LOGGER = logging.getLogger(__name__) @@ -182,3 +185,76 @@ async def add_binary( model_id=model_id ) return impl.EndpointImpl.from_database(crud.endpoint.get(db, id=m)) + + +@router.get( + path='/models/{model_id}/metadata', + response_model=ops_schemas.AdditionalModelInfo, + tags=['discover']) +def get_model_metadata( + model_id: int, + db: saorm.Session = fastapi.Depends(deps.get_db)): + LOGGER.info('Retrieving model metadata for id: %s', model_id) + match crud.binary_ml_model.get(db=db, id=model_id): + case None: + raise fastapi.HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f'Model with id {model_id} is not found') + case models.BinaryMlModel(format=app_binary_config.ModelWrapper.PICKLE | app_binary_config.ModelWrapper.JOBLIB as format): + return ops_schemas.AdditionalModelInfo( + modelPackage=format.value, + modelType='other') + case models.BinaryMlModel(format=app_binary_config.ModelWrapper.PMML, model_b64=binary): + match app_runtime_inspection.inspect_pmml_subtype(binary): + case 'Scorecard' | 'RuleSetModel' as typ: + return ops_schemas.AdditionalModelInfo( + modelPackage='pmml', + modelType=typ) + case _: + return ops_schemas.AdditionalModelInfo( + modelPackage='pmml', + modelType='other') + case _: + return ops_schemas.AdditionalModelInfo( + modelPackage='other', + modelType='other') + + +@router.get( + path='/models/{model_id}/download', + response_class=responses.StreamingResponse, + tags=['discover']) +def get_model_binary( + model_id: int, + db: saorm.Session = fastapi.Depends(deps.get_db)): + LOGGER.info('Retrieving model binary for id: %s', model_id) + + model = crud.model.get(db=db, id=model_id) + + if model is None: + raise fastapi.HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f'Model with id {model_id} is not found') + + filename = model.config.configuration['name'] + + try: + binary = model.endpoint.binary + except AttributeError: + raise fastapi.HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f'Model binary with id {model_id} is not found') + + if binary.format == app_binary_config.ModelWrapper.PMML: + file_extension = 'pmml' + elif binary.format == app_binary_config.ModelWrapper.PICKLE: + file_extension = 'pickle' + elif binary.format == app_binary_config.ModelWrapper.JOBLIB: + file_extension = 'joblib' + else: + file_extension = 'bin' + + LOGGER.info('Downloading model binary with name %s and format %s', filename, file_extension) + + return responses.StreamingResponse( + content=io.BytesIO(binary.model_b64), + media_type='application/octet-stream', + headers={'Content-Disposition': 'attachment; filename="{basename}.{extension}"'.format( + basename=filename, extension=file_extension)}) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/upload.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/upload.py index 60cb18a7..7986364e 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/upload.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/upload.py @@ -77,7 +77,7 @@ async def upload( model_binary, input_data_structure, output_data_structure, - format_, + file_format, name=model_name ) diff --git a/ops-implementations/ads-ml-service/app/db/session.py b/ops-implementations/ads-ml-service/app/db/session.py index 750c2654..521ffd29 100644 --- a/ops-implementations/ads-ml-service/app/db/session.py +++ b/ops-implementations/ads-ml-service/app/db/session.py @@ -43,7 +43,11 @@ def get_db_opts() -> typing.Dict[str, typing.Any]: return opt +def get_engine(): + return create_engine(get_db_url(), **get_db_opts()) + + SessionLocal = sessionmaker( autocommit=False, autoflush=False, - bind=create_engine(get_db_url(), **get_db_opts())) + bind=get_engine()) diff --git a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml index 4e0eae84..40273f2b 100644 --- a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml +++ b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Open Prediction Service - version: 2.7.0-SNAPSHOT + version: 2.8.0-SNAPSHOT description: The Open Prediction Service API is an effort to provide an Open API that enables unsupported native ML Providers in Decision Designer or Decision Runtime. tags: - name: info @@ -213,6 +213,47 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + /models/{model_id}/download: + get: + tags: + - discover + summary: Download model binary + operationId: get_binary_by_id + parameters: + - $ref: '#/components/parameters/ModelIDParam' + responses: + '200': + description: Successful Response + content: + # A binary file: + application/octet-stream: {} + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /models/{model_id}/metadata: + get: + tags: + - discover + summary: Get parsed model metadata + operationId: get_metadata_by_id + parameters: + - $ref: '#/components/parameters/ModelIDParam' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/AdditionalModelInfo' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' /endpoints: get: tags: @@ -823,13 +864,15 @@ components: unknown_file_size: type: boolean default: false - description: Whether the user can upload a file whose length is unknown at the time of the request. + description: Whether the user can upload a file whose length(Content-Length in HTTP header) is unknown at the time of the request. example: capabilities: - info - discover - manage - prediction + - download + - metadata managed_capabilities: supported_input_data_structure: - "auto" @@ -854,6 +897,10 @@ components: - discover - manage - run + # Download model binary + - download + # Inspect model specific metadata, for example for PMML model return subType (Scorecard, Ruleset) + - metadata Parameter: title: Parameter description: Parameter for ml model invocation @@ -907,6 +954,21 @@ components: target: - rel: endpoint href: 'http://open-prediction-service.org/endpoints/8c2af534-cdce-11ea-87d0-0242ac130003' + AdditionalModelInfo: + type: object + title: Additional Model Information + required: + - modelPackage + - modelType + properties: + modelPackage: + type: string + description: The file format of binary model + example: pmml + modelType: + type: string + description: Model type of binary model + example: RuleSetModel parameters: ModelIDParam: in: path diff --git a/ops-implementations/ads-ml-service/app/runtime/inspection.py b/ops-implementations/ads-ml-service/app/runtime/inspection.py index e53755ab..6f635241 100644 --- a/ops-implementations/ads-ml-service/app/runtime/inspection.py +++ b/ops-implementations/ads-ml-service/app/runtime/inspection.py @@ -17,6 +17,7 @@ import logging import typing +import pickletools import pypmml.base as pypmml_base @@ -65,3 +66,8 @@ def inspect_pmml_model_name(model_file: bytes) -> typing.Optional[str]: return wrapper.model.modelName else: return None + + +def inspect_pmml_subtype(model_file: bytes) -> typing.Optional[str]: + wrapper = load_pmml_model(model_file).loaded_model + return wrapper.model.modelElement diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_capabilities.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_capabilities.py index b9fb2415..aeb6f13f 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_capabilities.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_capabilities.py @@ -34,6 +34,8 @@ def test_get_server_capabilities( assert 'discover' in content['capabilities'] assert 'manage' in content['capabilities'] assert 'run' in content['capabilities'] + assert 'download' in content['capabilities'] + assert 'metadata' in content['capabilities'] def test_get_managed_capabilities( diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py index f2bd56cf..2b565ac9 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py @@ -20,6 +20,7 @@ import time import typing import typing as typ +import re import pytest import fastapi.testclient as tstc @@ -31,6 +32,8 @@ import app.schemas as schemas import app.tests.predictors.identity.model as app_tests_identity import app.tests.predictors.scikit_learn.model as app_test_skl +import app.models as models +import app.tests.predictors.pmml_sample.model as app_test_pmml def test_get_model( @@ -233,3 +236,67 @@ def test_update_binary( assert response.status_code == 201 assert response.json()['status'] == 'in_service' assert response_1.status_code == 422 + + +def test_not_supported_metadata( + client: tstc.TestClient, + xgboost_endpoint: models.Endpoint +) -> typ.NoReturn: + # When + resp = client.get(url=f'/models/{xgboost_endpoint.id}/metadata') + + # Assert + assert resp.ok + assert resp.json()['modelType'] == 'other' + + +def test_pickle_metadata( + client: tstc.TestClient +) -> typ.NoReturn: + # When + model = client.post( + url='/upload', + data={'format': 'pickle'}, + files={'file': ('model.pkl', pickle.dumps(app_test_skl.get_classification_predictor()))}).json() + model_id = model['id'] + resp = client.get(url=f'/models/{model_id}/metadata') + + # Assert + assert resp.ok + assert resp.json()['modelPackage'] == 'pickle' + assert resp.json()['modelType'] == 'other' + + +def test_pmml_metadata( + client: tstc.TestClient +) -> typ.NoReturn: + # When + model = client.post( + url='/upload', + data={'format': 'pmml'}, + files={'file': ('scorecard.pmml', app_test_pmml.get_pmml_scorecard_file().read_text())}).json() + model_id = model['id'] + resp = client.get(url=f'/models/{model_id}/metadata') + + # Assert + assert resp.ok + assert resp.json()['modelPackage'] == 'pmml' + assert resp.json()['modelType'] == 'Scorecard' + + +def test_download_binary( + client: tstc.TestClient +) -> typ.NoReturn: + # When + model_content = app_test_pmml.get_pmml_scorecard_file().read_text() + model = client.post( + url='/upload', + files={'file': ('scorecard.pmml', model_content)}).json() + model_id = model['id'] + resp = client.get(url=f'/models/{model_id}/download') + + # Assert + assert resp.ok + assert resp.content == str.encode(model_content) + # filename + assert re.findall("filename=\"(.+)\"", resp.headers['content-disposition'])[0] == 'scorecard.pmml' diff --git a/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/model.py b/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/model.py index 1473e5a0..6187c34b 100644 --- a/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/model.py +++ b/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/model.py @@ -22,5 +22,9 @@ def get_pmml_file() -> pathlib.Path: return pathlib.Path(__file__).resolve().parent.joinpath('model.pmml') +def get_pmml_scorecard_file() -> pathlib.Path: + return pathlib.Path(__file__).resolve().parent.joinpath('scorecard.pmml') + + def get_pmml_no_output_schema_file() -> pathlib.Path: return pathlib.Path(__file__).resolve().parent.joinpath('model-no-output-schema.pmml') diff --git a/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/scorecard.pmml b/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/scorecard.pmml new file mode 100644 index 00000000..643949c4 --- /dev/null +++ b/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/scorecard.pmml @@ -0,0 +1,708 @@ + + +
+ + 2022-11-25 05:46:41.942928 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + "F" + + + + + "M" + + + + + + + "D" + + + + + "M" + + + + + "S" + + + + + + + "N" + + + + + "Y" + + + + + + + "Auto" + + + + + "CC" + + + + + "CH" + + + + + + diff --git a/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py b/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py index 99d6c45f..135dee02 100644 --- a/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py +++ b/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py @@ -16,6 +16,7 @@ import pathlib +import pickle import app.runtime.inspection as app_signature_inspection import app.tests.predictors.pmml_sample.model as app_test_pmml @@ -52,3 +53,15 @@ def test_pmml_output_schema_inspection( 'probability_1': 'double', 'predicted_paymentDefault': 'integer' } + + +def test_inspect_pmml_subtype(): + # When + sub_type_regression = app_signature_inspection.inspect_pmml_subtype( + str.encode(app_test_pmml.get_pmml_file().read_text())) + sub_type_scorecard = app_signature_inspection.inspect_pmml_subtype( + str.encode(app_test_pmml.get_pmml_scorecard_file().read_text())) + + # Assert + assert sub_type_regression == 'RegressionModel' + assert sub_type_scorecard == 'Scorecard' diff --git a/ops-implementations/ads-ml-service/app/version.py b/ops-implementations/ads-ml-service/app/version.py index 1d8e1393..bb4701d5 100644 --- a/ops-implementations/ads-ml-service/app/version.py +++ b/ops-implementations/ads-ml-service/app/version.py @@ -1 +1 @@ -__version__ = '2.7.4-SNAPSHOT' +__version__ = '2.8.0-SNAPSHOT' diff --git a/ops-implementations/ads-ml-service/pom.xml b/ops-implementations/ads-ml-service/pom.xml index 3347a872..e721157f 100644 --- a/ops-implementations/ads-ml-service/pom.xml +++ b/ops-implementations/ads-ml-service/pom.xml @@ -5,7 +5,7 @@ com.ibm.decision.ops ml-service-implementations - 2.7.4-SNAPSHOT + 2.8.0-SNAPSHOT .. diff --git a/ops-implementations/ads-ml-service/tox.ini b/ops-implementations/ads-ml-service/tox.ini index d76f3d9a..c0cc3c98 100755 --- a/ops-implementations/ads-ml-service/tox.ini +++ b/ops-implementations/ads-ml-service/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38 +envlist = py310, py311 skipsdist = True [testenv] diff --git a/ops-implementations/pom.xml b/ops-implementations/pom.xml index 130c9849..a6e0fc6d 100644 --- a/ops-implementations/pom.xml +++ b/ops-implementations/pom.xml @@ -4,7 +4,7 @@ com.ibm.decision.ops ml-service-implementations - 2.7.4-SNAPSHOT + 2.8.0-SNAPSHOT pom diff --git a/ops-implementations/sagemaker-service/openapi_server/version.py b/ops-implementations/sagemaker-service/openapi_server/version.py index 1d8e1393..bb4701d5 100644 --- a/ops-implementations/sagemaker-service/openapi_server/version.py +++ b/ops-implementations/sagemaker-service/openapi_server/version.py @@ -1 +1 @@ -__version__ = '2.7.4-SNAPSHOT' +__version__ = '2.8.0-SNAPSHOT' diff --git a/ops-implementations/sagemaker-service/pom.xml b/ops-implementations/sagemaker-service/pom.xml index 1e9b29fc..555e3cf4 100644 --- a/ops-implementations/sagemaker-service/pom.xml +++ b/ops-implementations/sagemaker-service/pom.xml @@ -5,7 +5,7 @@ com.ibm.decision.ops ml-service-implementations - 2.7.4-SNAPSHOT + 2.8.0-SNAPSHOT .. diff --git a/ops-implementations/wml-service/pom.xml b/ops-implementations/wml-service/pom.xml index 8f3a7e44..b15d539f 100644 --- a/ops-implementations/wml-service/pom.xml +++ b/ops-implementations/wml-service/pom.xml @@ -5,7 +5,7 @@ com.ibm.decision.ops ml-service-implementations - 2.7.4-SNAPSHOT + 2.8.0-SNAPSHOT .. diff --git a/ops-implementations/wml-service/swagger_server/version.py b/ops-implementations/wml-service/swagger_server/version.py index 1d8e1393..bb4701d5 100644 --- a/ops-implementations/wml-service/swagger_server/version.py +++ b/ops-implementations/wml-service/swagger_server/version.py @@ -1 +1 @@ -__version__ = '2.7.4-SNAPSHOT' +__version__ = '2.8.0-SNAPSHOT'