From 3dd400127e9cca217e4188f7529068bef3d85d25 Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 10:28:57 +0100 Subject: [PATCH 01/26] [DBACLD-47417] Update OPS spec --- open-prediction-service.yaml | 79 +++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/open-prediction-service.yaml b/open-prediction-service.yaml index a6ddbfc..5c4e9bd 100644 --- a/open-prediction-service.yaml +++ b/open-prediction-service.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Open Prediction Service - version: 2.7.4-SNAPSHOT + version: 2.8.0-SNAPSHOT description: The Open Prediction Service API is an effort to provide an Open API that enables unsupported native ML Providers in Decision Designer or Decision Runtime. tags: - name: info @@ -213,6 +213,47 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + /models/{model_id}/binary: + get: + tags: + - discover + summary: Download model binary + operationId: get_binary_by_id + parameters: + - $ref: '#/components/parameters/ModelIDParam' + responses: + '200': + description: Successful Response + content: + # A binary file: + application/octet-stream: {} + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /models/{model_id}/metadata: + get: + tags: + - discover + summary: Get parsed model metadata + operationId: get_metadata_by_id + parameters: + - $ref: '#/components/parameters/ModelIDParam' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/AdditionalModelInfo' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' /endpoints: get: tags: @@ -823,7 +864,7 @@ components: unknown_file_size: type: boolean default: false - description: Whether the user can upload a file whose length is unknown at the time of the request. + description: Whether the user can upload a file whose length(Content-Length in HTTP header) is unknown at the time of the request. example: capabilities: - info @@ -907,6 +948,40 @@ components: target: - rel: endpoint href: 'http://open-prediction-service.org/endpoints/8c2af534-cdce-11ea-87d0-0242ac130003' + AdditionalModelInfo: + type: object + title: Additional Model Information + required: + - modelType + properties: + modelType: + type: string + description: the type of imported binary + example: pmml + discriminator: + propertyName: modelType + mapping: + pmml: '#/components/schemas/AdditionalPMMLModelInfo' + pickle: '#/components/schemas/AdditionalPickleModelInfo' + oneOf: + - $ref: '#/components/schemas/AdditionalPMMLModelInfo' + - $ref: '#/components/schemas/AdditionalPickleModelInfo' + AdditionalPMMLModelInfo: + allOf: + - $ref: '#/components/schemas/AdditionalModelInfo' + - properties: + modelSubType: + type: string + description: model sub type + example: ruleset + AdditionalPickleModelInfo: + allOf: + - $ref: '#/components/schemas/AdditionalModelInfo' + - properties: + pickleProtoVersion: + type: string + description: the version of pickle protocol + example: 4 parameters: ModelIDParam: in: path From b4ea755110e05c2f5c127c6aafc909e7cfa7c38f Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 10:30:35 +0100 Subject: [PATCH 02/26] [DBACLD-47417] Update OPS spec --- .../app/gen/tmp.schemas.ops.yaml | 79 ++++++++++++++++++- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml index 4e0eae8..5c4e9bd 100644 --- a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml +++ b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Open Prediction Service - version: 2.7.0-SNAPSHOT + version: 2.8.0-SNAPSHOT description: The Open Prediction Service API is an effort to provide an Open API that enables unsupported native ML Providers in Decision Designer or Decision Runtime. tags: - name: info @@ -213,6 +213,47 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + /models/{model_id}/binary: + get: + tags: + - discover + summary: Download model binary + operationId: get_binary_by_id + parameters: + - $ref: '#/components/parameters/ModelIDParam' + responses: + '200': + description: Successful Response + content: + # A binary file: + application/octet-stream: {} + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /models/{model_id}/metadata: + get: + tags: + - discover + summary: Get parsed model metadata + operationId: get_metadata_by_id + parameters: + - $ref: '#/components/parameters/ModelIDParam' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/AdditionalModelInfo' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' /endpoints: get: tags: @@ -823,7 +864,7 @@ components: unknown_file_size: type: boolean default: false - description: Whether the user can upload a file whose length is unknown at the time of the request. + description: Whether the user can upload a file whose length(Content-Length in HTTP header) is unknown at the time of the request. example: capabilities: - info @@ -907,6 +948,40 @@ components: target: - rel: endpoint href: 'http://open-prediction-service.org/endpoints/8c2af534-cdce-11ea-87d0-0242ac130003' + AdditionalModelInfo: + type: object + title: Additional Model Information + required: + - modelType + properties: + modelType: + type: string + description: the type of imported binary + example: pmml + discriminator: + propertyName: modelType + mapping: + pmml: '#/components/schemas/AdditionalPMMLModelInfo' + pickle: '#/components/schemas/AdditionalPickleModelInfo' + oneOf: + - $ref: '#/components/schemas/AdditionalPMMLModelInfo' + - $ref: '#/components/schemas/AdditionalPickleModelInfo' + AdditionalPMMLModelInfo: + allOf: + - $ref: '#/components/schemas/AdditionalModelInfo' + - properties: + modelSubType: + type: string + description: model sub type + example: ruleset + AdditionalPickleModelInfo: + allOf: + - $ref: '#/components/schemas/AdditionalModelInfo' + - properties: + pickleProtoVersion: + type: string + description: the version of pickle protocol + example: 4 parameters: ModelIDParam: in: path From 10c4e212179068b653a865bdea2fe4aaf04815bb Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 15:25:45 +0100 Subject: [PATCH 03/26] [DBACLD-47417] Update runtime.inspection for model metadata --- .../ads-ml-service/app/runtime/inspection.py | 10 + .../app/tests/predictors/pmml_sample/model.py | 4 + .../predictors/pmml_sample/scorecard.pmml | 708 ++++++++++++++++++ .../runtime/test_signature_inspection.py | 36 + 4 files changed, 758 insertions(+) create mode 100644 ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/scorecard.pmml diff --git a/ops-implementations/ads-ml-service/app/runtime/inspection.py b/ops-implementations/ads-ml-service/app/runtime/inspection.py index e53755a..59a8343 100644 --- a/ops-implementations/ads-ml-service/app/runtime/inspection.py +++ b/ops-implementations/ads-ml-service/app/runtime/inspection.py @@ -17,6 +17,7 @@ import logging import typing +import pickletools import pypmml.base as pypmml_base @@ -65,3 +66,12 @@ def inspect_pmml_model_name(model_file: bytes) -> typing.Optional[str]: return wrapper.model.modelName else: return None + + +def inspect_pmml_subtype(model_file: bytes) -> typing.Optional[str]: + wrapper = load_pmml_model(model_file).loaded_model + return wrapper.model.modelElement + + +def inspect_pickle_version(model_file: bytes) -> int: + return max(op[0].proto for op in pickletools.genops(model_file)) diff --git a/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/model.py b/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/model.py index 1473e5a..6187c34 100644 --- a/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/model.py +++ b/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/model.py @@ -22,5 +22,9 @@ def get_pmml_file() -> pathlib.Path: return pathlib.Path(__file__).resolve().parent.joinpath('model.pmml') +def get_pmml_scorecard_file() -> pathlib.Path: + return pathlib.Path(__file__).resolve().parent.joinpath('scorecard.pmml') + + def get_pmml_no_output_schema_file() -> pathlib.Path: return pathlib.Path(__file__).resolve().parent.joinpath('model-no-output-schema.pmml') diff --git a/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/scorecard.pmml b/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/scorecard.pmml new file mode 100644 index 0000000..643949c --- /dev/null +++ b/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/scorecard.pmml @@ -0,0 +1,708 @@ + + +
+ + 2022-11-25 05:46:41.942928 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + "F" + + + + + "M" + + + + + + + "D" + + + + + "M" + + + + + "S" + + + + + + + "N" + + + + + "Y" + + + + + + + "Auto" + + + + + "CC" + + + + + "CH" + + + + + + diff --git a/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py b/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py index 99d6c45..1ae4dfd 100644 --- a/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py +++ b/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py @@ -16,6 +16,7 @@ import pathlib +import pickle import app.runtime.inspection as app_signature_inspection import app.tests.predictors.pmml_sample.model as app_test_pmml @@ -52,3 +53,38 @@ def test_pmml_output_schema_inspection( 'probability_1': 'double', 'predicted_paymentDefault': 'integer' } + + +def test_inspect_pickle_version(): + # Arrange + p0 = pickle.dumps('test_0', protocol=0) + p1 = pickle.dumps('test_1', protocol=1) + p2 = pickle.dumps('test_2', protocol=2) + p3 = pickle.dumps(str.encode('test_3'), protocol=3) + p4 = pickle.dumps('test_4', protocol=4) + + # When + ver_0 = app_signature_inspection.inspect_pickle_version(p0) + ver_1 = app_signature_inspection.inspect_pickle_version(p1) + ver_2 = app_signature_inspection.inspect_pickle_version(p2) + ver_3 = app_signature_inspection.inspect_pickle_version(p3) + ver_4 = app_signature_inspection.inspect_pickle_version(p4) + + # Assert + assert ver_0 == 0 + assert ver_1 == 1 + assert ver_2 == 2 + assert ver_3 == 3 + assert ver_4 == 4 + + +def test_inspect_pmml_subtype(): + # When + sub_type_regression = app_signature_inspection.inspect_pmml_subtype( + str.encode(app_test_pmml.get_pmml_file().read_text())) + sub_type_scorecard = app_signature_inspection.inspect_pmml_subtype( + str.encode(app_test_pmml.get_pmml_scorecard_file().read_text())) + + # Assert + assert sub_type_regression == 'RegressionModel' + assert sub_type_scorecard == 'Scorecard' From 354cf044ac07b69971b4dbdd41f9fd3ee59ea542 Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 16:59:20 +0100 Subject: [PATCH 04/26] [DBACLD-47417] Add /models/id/metadata --- .../app/api/api_v2/endpoints/models.py | 28 +++++++++++ .../app/tests/api/api_v2/test_models.py | 48 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index 0d8cb61..20221e5 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -28,6 +28,7 @@ import app.crud as crud import app.gen.schemas.ops_schemas as ops_schemas import app.runtime.model_upload as app_model_upload +import app.runtime.inspection as app_runtime_inspection import app.schemas as schemas import app.schemas.binary_config as app_binary_config import app.schemas.impl as impl @@ -182,3 +183,30 @@ async def add_binary( model_id=model_id ) return impl.EndpointImpl.from_database(crud.endpoint.get(db, id=m)) + + +@router.get( + path='/models/{model_id}/metadata', + response_model=typing.Union[ops_schemas.AdditionalPickleModelInfo, ops_schemas.AdditionalPMMLModelInfo], + tags=['discover'] +) +def get_model_metadata( + model_id: int, + db: saorm.Session = fastapi.Depends(deps.get_db)): + LOGGER.info('Retrieving model metadata for id: %s', model_id) + model = crud.binary_ml_model.get(db=db, id=model_id) + + if model is None: + raise fastapi.HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f'Model with id {model_id} is not found') + + if model.format == app_binary_config.ModelWrapper.PICKLE: + return ops_schemas.AdditionalPickleModelInfo( + modelType='pickle', pickleProtoVersion=str(app_runtime_inspection.inspect_pickle_version(model.model_b64))) + elif model.format == app_binary_config.ModelWrapper.PMML: + return ops_schemas.AdditionalPMMLModelInfo( + modelType='pmml', modelSubType=str(app_runtime_inspection.inspect_pmml_subtype(model.model_b64))) + else: + raise fastapi.HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f'Format {model.format} is not supported for metadata') diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py index f2bd56c..d7a01e2 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py @@ -31,6 +31,8 @@ import app.schemas as schemas import app.tests.predictors.identity.model as app_tests_identity import app.tests.predictors.scikit_learn.model as app_test_skl +import app.models as models +import app.tests.predictors.pmml_sample.model as app_test_pmml def test_get_model( @@ -233,3 +235,49 @@ def test_update_binary( assert response.status_code == 201 assert response.json()['status'] == 'in_service' assert response_1.status_code == 422 + + +def test_not_supported_metadata( + client: tstc.TestClient, + xgboost_endpoint: models.Endpoint +) -> typ.NoReturn: + # When + resp = client.get(url=f'/models/{xgboost_endpoint.id}/metadata') + + # Assert + assert resp.status_code == 422 + assert resp.json()['detail'] == 'Format bst is not supported for metadata' + + +def test_pickle_metadata( + client: tstc.TestClient +) -> typ.NoReturn: + # When + model = client.post( + url='/upload', + data={'format': 'pickle'}, + files={'file': ('model.pkl', pickle.dumps(app_test_skl.get_classification_predictor()))}).json() + model_id = model['id'] + resp = client.get(url=f'/models/{model_id}/metadata') + + # Assert + assert resp.ok + assert resp.json()['modelType'] == 'pickle' + assert resp.json()['pickleProtoVersion'] == '4' + + +def test_pmml_metadata( + client: tstc.TestClient +) -> typ.NoReturn: + # When + model = client.post( + url='/upload', + data={'format': 'pmml'}, + files={'file': ('scorecard.pmml', app_test_pmml.get_pmml_scorecard_file().read_text())}).json() + model_id = model['id'] + resp = client.get(url=f'/models/{model_id}/metadata') + + # Assert + assert resp.ok + assert resp.json()['modelType'] == 'pmml' + assert resp.json()['modelSubType'] == 'Scorecard' From cf47ec6eed54195a5543f0e2633d9853c49c0701 Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 17:08:19 +0100 Subject: [PATCH 05/26] [DBACLD-47417] Fix discriminator --- .../app/api/api_v2/endpoints/models.py | 6 +++--- .../ads-ml-service/app/schemas/impl.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index 20221e5..95e6beb 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -187,7 +187,7 @@ async def add_binary( @router.get( path='/models/{model_id}/metadata', - response_model=typing.Union[ops_schemas.AdditionalPickleModelInfo, ops_schemas.AdditionalPMMLModelInfo], + response_model=impl.AdditionalModelInfo, tags=['discover'] ) def get_model_metadata( @@ -201,10 +201,10 @@ def get_model_metadata( status_code=status.HTTP_404_NOT_FOUND, detail=f'Model with id {model_id} is not found') if model.format == app_binary_config.ModelWrapper.PICKLE: - return ops_schemas.AdditionalPickleModelInfo( + return impl.AdditionalPickleModelInfo( modelType='pickle', pickleProtoVersion=str(app_runtime_inspection.inspect_pickle_version(model.model_b64))) elif model.format == app_binary_config.ModelWrapper.PMML: - return ops_schemas.AdditionalPMMLModelInfo( + return impl.AdditionalPMMLModelInfo( modelType='pmml', modelSubType=str(app_runtime_inspection.inspect_pmml_subtype(model.model_b64))) else: raise fastapi.HTTPException( diff --git a/ops-implementations/ads-ml-service/app/schemas/impl.py b/ops-implementations/ads-ml-service/app/schemas/impl.py index 7faddd9..3e84f5d 100644 --- a/ops-implementations/ads-ml-service/app/schemas/impl.py +++ b/ops-implementations/ads-ml-service/app/schemas/impl.py @@ -17,6 +17,7 @@ import enum import typing +import typing_extensions import datetime as dt import numpy @@ -158,3 +159,17 @@ class PredictionImpl(pydt.BaseModel): parameters: typing.List[typing.Union[typing.List[ParameterImpl], ParameterImpl]] = pydt.Field( ..., description='Model parameters', title='Parameters' ) + + +class AdditionalPMMLModelInfo(pydt.BaseModel): + modelType: typing.Literal['pmml'] + modelSubType: typing.Optional[str] + + +class AdditionalPickleModelInfo(pydt.BaseModel): + modelType: typing.Literal['pickle'] + pickleProtoVersion: typing.Optional[str] + + +AdditionalModelInfo = typing_extensions.Annotated[ + typing.Union[AdditionalPMMLModelInfo, AdditionalPickleModelInfo], pydt.Field(discriminator='modelType')] From 3c0ec47038f15a03890c456dd79067d2bd2ac7e0 Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 17:33:29 +0100 Subject: [PATCH 06/26] [DBACLD-47417] Add binary download --- .../app/api/api_v2/endpoints/models.py | 23 +++++++++++++++++-- .../app/tests/api/api_v2/test_models.py | 17 ++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index 95e6beb..fe16a76 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -17,6 +17,7 @@ import logging import typing +import io import fastapi import fastapi.encoders as encoders @@ -188,8 +189,7 @@ async def add_binary( @router.get( path='/models/{model_id}/metadata', response_model=impl.AdditionalModelInfo, - tags=['discover'] -) + tags=['discover']) def get_model_metadata( model_id: int, db: saorm.Session = fastapi.Depends(deps.get_db)): @@ -210,3 +210,22 @@ def get_model_metadata( raise fastapi.HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f'Format {model.format} is not supported for metadata') + + +@router.get( + path='/models/{model_id}/binary', + tags=['discover']) +def get_model_binary( + model_id: int, + db: saorm.Session = fastapi.Depends(deps.get_db)): + LOGGER.info('Retrieving model binary for id: %s', model_id) + model = crud.binary_ml_model.get(db=db, id=model_id) + + if model is None: + raise fastapi.HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f'Model binary with id {model_id} is not found') + + filelike = io.BytesIO(model.model_b64) + filelike.seek(0) + + return responses.StreamingResponse(content=filelike, media_type='application/octet-stream') diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py index d7a01e2..9b267d3 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py @@ -281,3 +281,20 @@ def test_pmml_metadata( assert resp.ok assert resp.json()['modelType'] == 'pmml' assert resp.json()['modelSubType'] == 'Scorecard' + + +def test_download_binary( + client: tstc.TestClient +) -> typ.NoReturn: + # When + model_content = app_test_pmml.get_pmml_scorecard_file().read_text() + model = client.post( + url='/upload', + data={'format': 'pmml'}, + files={'file': ('scorecard.pmml', model_content)}).json() + model_id = model['id'] + resp = client.get(url=f'/models/{model_id}/binary') + + # Assert + assert resp.ok + assert resp.content == str.encode(model_content) From eb9480dc203b698a376695c357f92e97eb0953dc Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 17:52:18 +0100 Subject: [PATCH 07/26] [DBACLD-47417] Add response_class for binary download --- .../ads-ml-service/app/api/api_v2/endpoints/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index fe16a76..c7b2ada 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -214,6 +214,7 @@ def get_model_metadata( @router.get( path='/models/{model_id}/binary', + response_class=responses.StreamingResponse, tags=['discover']) def get_model_binary( model_id: int, From 0826404fa5e3e2ba6886204f7afee1039737f0ba Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 17:54:36 +0100 Subject: [PATCH 08/26] [DBACLD-47417] Bump ver --- ops-client-sdk/pom.xml | 2 +- ops-implementations/ads-ml-service/app/version.py | 2 +- ops-implementations/ads-ml-service/pom.xml | 2 +- ops-implementations/pom.xml | 2 +- ops-implementations/sagemaker-service/openapi_server/version.py | 2 +- ops-implementations/sagemaker-service/pom.xml | 2 +- ops-implementations/wml-service/pom.xml | 2 +- ops-implementations/wml-service/swagger_server/version.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ops-client-sdk/pom.xml b/ops-client-sdk/pom.xml index 9927c82..09337fb 100644 --- a/ops-client-sdk/pom.xml +++ b/ops-client-sdk/pom.xml @@ -6,7 +6,7 @@ com.ibm.decision ops-client-sdk - 2.7.4-SNAPSHOT + 2.8.0-SNAPSHOT 2020 diff --git a/ops-implementations/ads-ml-service/app/version.py b/ops-implementations/ads-ml-service/app/version.py index 1d8e139..bb4701d 100644 --- a/ops-implementations/ads-ml-service/app/version.py +++ b/ops-implementations/ads-ml-service/app/version.py @@ -1 +1 @@ -__version__ = '2.7.4-SNAPSHOT' +__version__ = '2.8.0-SNAPSHOT' diff --git a/ops-implementations/ads-ml-service/pom.xml b/ops-implementations/ads-ml-service/pom.xml index 3347a87..e721157 100644 --- a/ops-implementations/ads-ml-service/pom.xml +++ b/ops-implementations/ads-ml-service/pom.xml @@ -5,7 +5,7 @@ com.ibm.decision.ops ml-service-implementations - 2.7.4-SNAPSHOT + 2.8.0-SNAPSHOT .. diff --git a/ops-implementations/pom.xml b/ops-implementations/pom.xml index 130c984..a6e0fc6 100644 --- a/ops-implementations/pom.xml +++ b/ops-implementations/pom.xml @@ -4,7 +4,7 @@ com.ibm.decision.ops ml-service-implementations - 2.7.4-SNAPSHOT + 2.8.0-SNAPSHOT pom diff --git a/ops-implementations/sagemaker-service/openapi_server/version.py b/ops-implementations/sagemaker-service/openapi_server/version.py index 1d8e139..bb4701d 100644 --- a/ops-implementations/sagemaker-service/openapi_server/version.py +++ b/ops-implementations/sagemaker-service/openapi_server/version.py @@ -1 +1 @@ -__version__ = '2.7.4-SNAPSHOT' +__version__ = '2.8.0-SNAPSHOT' diff --git a/ops-implementations/sagemaker-service/pom.xml b/ops-implementations/sagemaker-service/pom.xml index 1e9b29f..555e3cf 100644 --- a/ops-implementations/sagemaker-service/pom.xml +++ b/ops-implementations/sagemaker-service/pom.xml @@ -5,7 +5,7 @@ com.ibm.decision.ops ml-service-implementations - 2.7.4-SNAPSHOT + 2.8.0-SNAPSHOT .. diff --git a/ops-implementations/wml-service/pom.xml b/ops-implementations/wml-service/pom.xml index 8f3a7e4..b15d539 100644 --- a/ops-implementations/wml-service/pom.xml +++ b/ops-implementations/wml-service/pom.xml @@ -5,7 +5,7 @@ com.ibm.decision.ops ml-service-implementations - 2.7.4-SNAPSHOT + 2.8.0-SNAPSHOT .. diff --git a/ops-implementations/wml-service/swagger_server/version.py b/ops-implementations/wml-service/swagger_server/version.py index 1d8e139..bb4701d 100644 --- a/ops-implementations/wml-service/swagger_server/version.py +++ b/ops-implementations/wml-service/swagger_server/version.py @@ -1 +1 @@ -__version__ = '2.7.4-SNAPSHOT' +__version__ = '2.8.0-SNAPSHOT' From 180a9f75e4ba6bf732ee8dc45f4abfcbb7c17738 Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 17:59:15 +0100 Subject: [PATCH 09/26] [DBACLD-47417] Add changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b674e8f..6b48e97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Open Prediction Service HUB +## 2.8.0 _2022/11/28_ + +* [DBACLD-47417] Extend OPS to support downloading of Ruleset Models + ## 2.7.1 _2022/09/15_ * [DBACLD-58320] Fix ads ml service rule import From 181420a270fbaecea56fdba7d1c162cb32a4ad7e Mon Sep 17 00:00:00 2001 From: ke li Date: Wed, 30 Nov 2022 11:40:32 +0100 Subject: [PATCH 10/26] [DBACLD-47417] Add filename support for model download --- .../app/api/api_v2/endpoints/models.py | 27 ++++++++++++++++--- .../app/tests/api/api_v2/test_models.py | 3 +++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index c7b2ada..b478887 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -220,13 +220,32 @@ def get_model_binary( model_id: int, db: saorm.Session = fastapi.Depends(deps.get_db)): LOGGER.info('Retrieving model binary for id: %s', model_id) - model = crud.binary_ml_model.get(db=db, id=model_id) + + model = crud.model.get(db=db, id=model_id) if model is None: + raise fastapi.HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f'Model with id {model_id} is not found') + + filename = model.config.configuration['name'] + + try: + binary = model.endpoint.binary + except AttributeError: raise fastapi.HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f'Model binary with id {model_id} is not found') - filelike = io.BytesIO(model.model_b64) - filelike.seek(0) + if binary.format == app_binary_config.ModelWrapper.PMML: + file_extension = 'pmml' + elif binary.format == app_binary_config.ModelWrapper.PICKLE: + file_extension = 'pickle' + elif binary.format == app_binary_config.ModelWrapper.JOBLIB: + file_extension = 'pickle' + else: + file_extension = 'bin' - return responses.StreamingResponse(content=filelike, media_type='application/octet-stream') + return responses.StreamingResponse( + content=io.BytesIO(binary.model_b64), + media_type='application/octet-stream', + headers={'Content-Disposition': 'attachment; filename="{basename}.{extension}"'.format( + basename=filename, extension=file_extension)}) diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py index 9b267d3..d878577 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py @@ -20,6 +20,7 @@ import time import typing import typing as typ +import re import pytest import fastapi.testclient as tstc @@ -294,7 +295,9 @@ def test_download_binary( files={'file': ('scorecard.pmml', model_content)}).json() model_id = model['id'] resp = client.get(url=f'/models/{model_id}/binary') + received_filename = re.findall("filename=\"(.+)\"", resp.headers['content-disposition'])[0] # Assert assert resp.ok assert resp.content == str.encode(model_content) + assert received_filename == 'scorecard.pmml' From 6d62f418c60ea70042202a782d1f5f902d5b0d2d Mon Sep 17 00:00:00 2001 From: ke li Date: Thu, 1 Dec 2022 13:42:33 +0100 Subject: [PATCH 11/26] [DBACLD-47417] Add log for model download --- .../ads-ml-service/app/api/api_v2/endpoints/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index b478887..353712e 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -244,6 +244,8 @@ def get_model_binary( else: file_extension = 'bin' + LOGGER.info('Downloading model binary with name %s and format %s', filename, file_extension) + return responses.StreamingResponse( content=io.BytesIO(binary.model_b64), media_type='application/octet-stream', From f6624a08d9485d162af596b37bfd999b0eb49b1c Mon Sep 17 00:00:00 2001 From: ke li Date: Thu, 1 Dec 2022 13:42:55 +0100 Subject: [PATCH 12/26] [DBACLD-47417] Fix model type --- .../ads-ml-service/app/api/api_v2/endpoints/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index 353712e..4b57f62 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -240,7 +240,7 @@ def get_model_binary( elif binary.format == app_binary_config.ModelWrapper.PICKLE: file_extension = 'pickle' elif binary.format == app_binary_config.ModelWrapper.JOBLIB: - file_extension = 'pickle' + file_extension = 'joblib' else: file_extension = 'bin' From 5cdd71f9f613b84ea020f8b494611c87cc8c6341 Mon Sep 17 00:00:00 2001 From: ke li Date: Thu, 1 Dec 2022 14:06:11 +0100 Subject: [PATCH 13/26] [DBACLD-47417] Update spec for cap --- open-prediction-service.yaml | 8 +++++++- .../ads-ml-service/app/gen/tmp.schemas.ops.yaml | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/open-prediction-service.yaml b/open-prediction-service.yaml index 5c4e9bd..4b3c125 100644 --- a/open-prediction-service.yaml +++ b/open-prediction-service.yaml @@ -213,7 +213,7 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' - /models/{model_id}/binary: + /models/{model_id}/download: get: tags: - discover @@ -871,6 +871,8 @@ components: - discover - manage - prediction + - download + - metadata managed_capabilities: supported_input_data_structure: - "auto" @@ -895,6 +897,10 @@ components: - discover - manage - run + # Download model binary + - download + # Inspect model specific metadata, for example for PMML model return subType (Scorecard, Ruleset) + - metadata Parameter: title: Parameter description: Parameter for ml model invocation diff --git a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml index 5c4e9bd..4b3c125 100644 --- a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml +++ b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml @@ -213,7 +213,7 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' - /models/{model_id}/binary: + /models/{model_id}/download: get: tags: - discover @@ -871,6 +871,8 @@ components: - discover - manage - prediction + - download + - metadata managed_capabilities: supported_input_data_structure: - "auto" @@ -895,6 +897,10 @@ components: - discover - manage - run + # Download model binary + - download + # Inspect model specific metadata, for example for PMML model return subType (Scorecard, Ruleset) + - metadata Parameter: title: Parameter description: Parameter for ml model invocation From 3a08e84bd8284e5a91cbefa4c9026628ed5c9462 Mon Sep 17 00:00:00 2001 From: ke li Date: Thu, 1 Dec 2022 14:18:03 +0100 Subject: [PATCH 14/26] [DBACLD-47417] Add cap --- .../ads-ml-service/app/api/api_v2/endpoints/capabilities.py | 4 +++- .../ads-ml-service/app/tests/api/api_v2/test_capabilities.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/capabilities.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/capabilities.py index b3f9f0b..6e9e92f 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/capabilities.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/capabilities.py @@ -35,7 +35,9 @@ async def server_capabilities() -> typing.Dict[typing.Text, typing.Any]: ops_model.Capability.info, ops_model.Capability.discover, ops_model.Capability.manage, - ops_model.Capability.run + ops_model.Capability.run, + ops_model.Capability.download, + ops_model.Capability.metadata ], 'managed_capabilities': { 'supported_input_data_structure': ['auto', 'DataFrame', 'ndarray', 'DMatrix', 'list'], diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_capabilities.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_capabilities.py index b9fb241..aeb6f13 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_capabilities.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_capabilities.py @@ -34,6 +34,8 @@ def test_get_server_capabilities( assert 'discover' in content['capabilities'] assert 'manage' in content['capabilities'] assert 'run' in content['capabilities'] + assert 'download' in content['capabilities'] + assert 'metadata' in content['capabilities'] def test_get_managed_capabilities( From 91435a790e51ec300040bec1de5457aabdd931a2 Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 13 Dec 2022 10:10:15 +0100 Subject: [PATCH 15/26] [DBACLD-47417] Update endpoint name --- .../ads-ml-service/app/api/api_v2/endpoints/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index 4b57f62..6631356 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -213,7 +213,7 @@ def get_model_metadata( @router.get( - path='/models/{model_id}/binary', + path='/models/{model_id}/download', response_class=responses.StreamingResponse, tags=['discover']) def get_model_binary( From 7cc92094bc787877aa6652b9bd5f675d1b9b2219 Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 13 Dec 2022 10:13:21 +0100 Subject: [PATCH 16/26] [DBACLD-47417] Fix upload format check --- .../ads-ml-service/app/api/api_v2/endpoints/upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/upload.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/upload.py index 60cb18a..7986364 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/upload.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/upload.py @@ -77,7 +77,7 @@ async def upload( model_binary, input_data_structure, output_data_structure, - format_, + file_format, name=model_name ) From 98afc61ccc99dc4665dc87f4aa56d9e3e5dad5a8 Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 13 Dec 2022 10:14:28 +0100 Subject: [PATCH 17/26] [DBACLD-47417] Fix db init for debug launcher --- ops-implementations/ads-ml-service/app/db/session.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/app/db/session.py b/ops-implementations/ads-ml-service/app/db/session.py index 750c265..521ffd2 100644 --- a/ops-implementations/ads-ml-service/app/db/session.py +++ b/ops-implementations/ads-ml-service/app/db/session.py @@ -43,7 +43,11 @@ def get_db_opts() -> typing.Dict[str, typing.Any]: return opt +def get_engine(): + return create_engine(get_db_url(), **get_db_opts()) + + SessionLocal = sessionmaker( autocommit=False, autoflush=False, - bind=create_engine(get_db_url(), **get_db_opts())) + bind=get_engine()) From e20507b897f07c059d24c82ee512cffe99095d67 Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 13 Dec 2022 11:11:33 +0100 Subject: [PATCH 18/26] [DBACLD-47417] Update tests for new download endpoint --- .../ads-ml-service/app/tests/api/api_v2/test_models.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py index d878577..5701869 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py @@ -291,13 +291,12 @@ def test_download_binary( model_content = app_test_pmml.get_pmml_scorecard_file().read_text() model = client.post( url='/upload', - data={'format': 'pmml'}, files={'file': ('scorecard.pmml', model_content)}).json() model_id = model['id'] - resp = client.get(url=f'/models/{model_id}/binary') - received_filename = re.findall("filename=\"(.+)\"", resp.headers['content-disposition'])[0] + resp = client.get(url=f'/models/{model_id}/download') # Assert assert resp.ok assert resp.content == str.encode(model_content) - assert received_filename == 'scorecard.pmml' + # filename + assert re.findall("filename=\"(.+)\"", resp.headers['content-disposition'])[0] == 'scorecard.pmml' From fd6b07a7dc607eb00b457d8e31d69094aa72e094 Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 13 Dec 2022 14:56:40 +0100 Subject: [PATCH 19/26] [DBACLD-47417] Fix merge issue --- .../ads-ml-service/app/api/api_v2/endpoints/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index 6631356..65faea2 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -209,7 +209,7 @@ def get_model_metadata( else: raise fastapi.HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=f'Format {model.format} is not supported for metadata') + detail=f'Format {model.format.value} is not supported for metadata') @router.get( From 9663da0a9ecb3f4a63c44f9bb101fd870cde2e59 Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 20 Dec 2022 09:39:58 +0100 Subject: [PATCH 20/26] [DBACLD-47417] Update spec for better description and required --- open-prediction-service.yaml | 8 +++++++- .../ads-ml-service/app/gen/tmp.schemas.ops.yaml | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/open-prediction-service.yaml b/open-prediction-service.yaml index 4b3c125..6e8ff84 100644 --- a/open-prediction-service.yaml +++ b/open-prediction-service.yaml @@ -973,14 +973,20 @@ components: - $ref: '#/components/schemas/AdditionalPMMLModelInfo' - $ref: '#/components/schemas/AdditionalPickleModelInfo' AdditionalPMMLModelInfo: + required: + - modelType + - modelSubType allOf: - $ref: '#/components/schemas/AdditionalModelInfo' - properties: modelSubType: type: string - description: model sub type + description: PMML MODEL-ELEMENT. Could be RuleSetModel, Scorecard... example: ruleset AdditionalPickleModelInfo: + required: + - modelType + - pickleProtoVersion allOf: - $ref: '#/components/schemas/AdditionalModelInfo' - properties: diff --git a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml index 4b3c125..6e8ff84 100644 --- a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml +++ b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml @@ -973,14 +973,20 @@ components: - $ref: '#/components/schemas/AdditionalPMMLModelInfo' - $ref: '#/components/schemas/AdditionalPickleModelInfo' AdditionalPMMLModelInfo: + required: + - modelType + - modelSubType allOf: - $ref: '#/components/schemas/AdditionalModelInfo' - properties: modelSubType: type: string - description: model sub type + description: PMML MODEL-ELEMENT. Could be RuleSetModel, Scorecard... example: ruleset AdditionalPickleModelInfo: + required: + - modelType + - pickleProtoVersion allOf: - $ref: '#/components/schemas/AdditionalModelInfo' - properties: From 8e8688eb9d2eef9862837f2bd1b8df414e126ee3 Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 20 Dec 2022 11:45:10 +0100 Subject: [PATCH 21/26] [DBACLD-47417] Update spec for AdditionalOtherModelInfo --- open-prediction-service.yaml | 3 +++ .../ads-ml-service/app/api/api_v2/endpoints/models.py | 5 ++--- .../ads-ml-service/app/gen/tmp.schemas.ops.yaml | 3 +++ ops-implementations/ads-ml-service/app/schemas/impl.py | 10 +++++++--- .../ads-ml-service/app/tests/api/api_v2/test_models.py | 4 ++-- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/open-prediction-service.yaml b/open-prediction-service.yaml index 6e8ff84..edffbd8 100644 --- a/open-prediction-service.yaml +++ b/open-prediction-service.yaml @@ -994,6 +994,9 @@ components: type: string description: the version of pickle protocol example: 4 + AdditionalOtherModelInfo: + allOf: + - $ref: '#/components/schemas/AdditionalModelInfo' parameters: ModelIDParam: in: path diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index 65faea2..bd2566e 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -207,9 +207,8 @@ def get_model_metadata( return impl.AdditionalPMMLModelInfo( modelType='pmml', modelSubType=str(app_runtime_inspection.inspect_pmml_subtype(model.model_b64))) else: - raise fastapi.HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=f'Format {model.format.value} is not supported for metadata') + return impl.AdditionalOtherModelInfo( + modelType='other') @router.get( diff --git a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml index 6e8ff84..edffbd8 100644 --- a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml +++ b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml @@ -994,6 +994,9 @@ components: type: string description: the version of pickle protocol example: 4 + AdditionalOtherModelInfo: + allOf: + - $ref: '#/components/schemas/AdditionalModelInfo' parameters: ModelIDParam: in: path diff --git a/ops-implementations/ads-ml-service/app/schemas/impl.py b/ops-implementations/ads-ml-service/app/schemas/impl.py index 3e84f5d..e56db0d 100644 --- a/ops-implementations/ads-ml-service/app/schemas/impl.py +++ b/ops-implementations/ads-ml-service/app/schemas/impl.py @@ -163,13 +163,17 @@ class PredictionImpl(pydt.BaseModel): class AdditionalPMMLModelInfo(pydt.BaseModel): modelType: typing.Literal['pmml'] - modelSubType: typing.Optional[str] + modelSubType: str class AdditionalPickleModelInfo(pydt.BaseModel): modelType: typing.Literal['pickle'] - pickleProtoVersion: typing.Optional[str] + pickleProtoVersion: str + + +class AdditionalOtherModelInfo(pydt.BaseModel): + modelType: typing.Literal['other'] AdditionalModelInfo = typing_extensions.Annotated[ - typing.Union[AdditionalPMMLModelInfo, AdditionalPickleModelInfo], pydt.Field(discriminator='modelType')] + typing.Union[AdditionalPMMLModelInfo, AdditionalPickleModelInfo, AdditionalOtherModelInfo], pydt.Field(discriminator='modelType')] diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py index 5701869..8d39c51 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py @@ -246,8 +246,8 @@ def test_not_supported_metadata( resp = client.get(url=f'/models/{xgboost_endpoint.id}/metadata') # Assert - assert resp.status_code == 422 - assert resp.json()['detail'] == 'Format bst is not supported for metadata' + assert resp.ok + assert resp.json()['modelType'] == 'other' def test_pickle_metadata( From c587ac3f7f5002ef414f02231f9b12a47304821e Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 20 Dec 2022 11:48:30 +0100 Subject: [PATCH 22/26] [DBACLD-47417] Update spec for AdditionalOtherModelInfo --- open-prediction-service.yaml | 2 ++ ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/open-prediction-service.yaml b/open-prediction-service.yaml index edffbd8..63ffbb4 100644 --- a/open-prediction-service.yaml +++ b/open-prediction-service.yaml @@ -969,9 +969,11 @@ components: mapping: pmml: '#/components/schemas/AdditionalPMMLModelInfo' pickle: '#/components/schemas/AdditionalPickleModelInfo' + other: '#/components/schemas/AdditionalOtherModelInfo' oneOf: - $ref: '#/components/schemas/AdditionalPMMLModelInfo' - $ref: '#/components/schemas/AdditionalPickleModelInfo' + - $ref: '#/components/schemas/AdditionalOtherModelInfo' AdditionalPMMLModelInfo: required: - modelType diff --git a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml index edffbd8..63ffbb4 100644 --- a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml +++ b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml @@ -969,9 +969,11 @@ components: mapping: pmml: '#/components/schemas/AdditionalPMMLModelInfo' pickle: '#/components/schemas/AdditionalPickleModelInfo' + other: '#/components/schemas/AdditionalOtherModelInfo' oneOf: - $ref: '#/components/schemas/AdditionalPMMLModelInfo' - $ref: '#/components/schemas/AdditionalPickleModelInfo' + - $ref: '#/components/schemas/AdditionalOtherModelInfo' AdditionalPMMLModelInfo: required: - modelType From e89bc2e20f186866a78f1783a8d17fb6b4d1f624 Mon Sep 17 00:00:00 2001 From: ke li Date: Wed, 18 Jan 2023 14:41:52 +0100 Subject: [PATCH 23/26] [DBACLD-47417] Update spec --- open-prediction-service.yaml | 55 ++++++------------- .../app/api/api_v2/endpoints/models.py | 48 ++++++++++------ .../app/gen/tmp.schemas.ops.yaml | 55 ++++++------------- .../ads-ml-service/app/schemas/impl.py | 18 ------ .../app/tests/api/api_v2/test_models.py | 8 +-- ops-implementations/ads-ml-service/tox.ini | 2 +- 6 files changed, 73 insertions(+), 113 deletions(-) diff --git a/open-prediction-service.yaml b/open-prediction-service.yaml index 63ffbb4..715d35a 100644 --- a/open-prediction-service.yaml +++ b/open-prediction-service.yaml @@ -958,47 +958,28 @@ components: type: object title: Additional Model Information required: + - modelPackage - modelType properties: - modelType: + modelPackage: type: string - description: the type of imported binary + description: The file format of binary model example: pmml - discriminator: - propertyName: modelType - mapping: - pmml: '#/components/schemas/AdditionalPMMLModelInfo' - pickle: '#/components/schemas/AdditionalPickleModelInfo' - other: '#/components/schemas/AdditionalOtherModelInfo' - oneOf: - - $ref: '#/components/schemas/AdditionalPMMLModelInfo' - - $ref: '#/components/schemas/AdditionalPickleModelInfo' - - $ref: '#/components/schemas/AdditionalOtherModelInfo' - AdditionalPMMLModelInfo: - required: - - modelType - - modelSubType - allOf: - - $ref: '#/components/schemas/AdditionalModelInfo' - - properties: - modelSubType: - type: string - description: PMML MODEL-ELEMENT. Could be RuleSetModel, Scorecard... - example: ruleset - AdditionalPickleModelInfo: - required: - - modelType - - pickleProtoVersion - allOf: - - $ref: '#/components/schemas/AdditionalModelInfo' - - properties: - pickleProtoVersion: - type: string - description: the version of pickle protocol - example: 4 - AdditionalOtherModelInfo: - allOf: - - $ref: '#/components/schemas/AdditionalModelInfo' + enum: + - pmml + - joblib + - pickle + - other + modelType: + type: string + description: Model type of binary model + example: RuleSetModel + enum: + # pmml ruleset + - RuleSetModel + # pmml scorecard + - Scorecard + - other parameters: ModelIDParam: in: path diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index bd2566e..f2838f8 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -34,6 +34,7 @@ import app.schemas.binary_config as app_binary_config import app.schemas.impl as impl import app.core.configuration as app_conf +import app.models as models router = fastapi.APIRouter() LOGGER = logging.getLogger(__name__) @@ -188,27 +189,42 @@ async def add_binary( @router.get( path='/models/{model_id}/metadata', - response_model=impl.AdditionalModelInfo, + response_model=ops_schemas.AdditionalModelInfo, tags=['discover']) def get_model_metadata( model_id: int, db: saorm.Session = fastapi.Depends(deps.get_db)): LOGGER.info('Retrieving model metadata for id: %s', model_id) - model = crud.binary_ml_model.get(db=db, id=model_id) - - if model is None: - raise fastapi.HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail=f'Model with id {model_id} is not found') - - if model.format == app_binary_config.ModelWrapper.PICKLE: - return impl.AdditionalPickleModelInfo( - modelType='pickle', pickleProtoVersion=str(app_runtime_inspection.inspect_pickle_version(model.model_b64))) - elif model.format == app_binary_config.ModelWrapper.PMML: - return impl.AdditionalPMMLModelInfo( - modelType='pmml', modelSubType=str(app_runtime_inspection.inspect_pmml_subtype(model.model_b64))) - else: - return impl.AdditionalOtherModelInfo( - modelType='other') + match crud.binary_ml_model.get(db=db, id=model_id): + case None: + raise fastapi.HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f'Model with id {model_id} is not found') + case models.BinaryMlModel(format=app_binary_config.ModelWrapper.PICKLE): + return ops_schemas.AdditionalModelInfo( + modelPackage=ops_schemas.ModelPackage.pickle, + modelType=ops_schemas.ModelType.other) + case models.BinaryMlModel(format=app_binary_config.ModelWrapper.JOBLIB): + return ops_schemas.AdditionalModelInfo( + modelPackage=ops_schemas.ModelPackage.joblib, + modelType=ops_schemas.ModelType.other) + case models.BinaryMlModel(format=app_binary_config.ModelWrapper.PMML, model_b64=binary): + match app_runtime_inspection.inspect_pmml_subtype(binary): + case 'RuleSetModel': + return ops_schemas.AdditionalModelInfo( + modelPackage=ops_schemas.ModelPackage.pmml, + modelType=ops_schemas.ModelType.RuleSetModel) + case 'Scorecard': + return ops_schemas.AdditionalModelInfo( + modelPackage=ops_schemas.ModelPackage.pmml, + modelType=ops_schemas.ModelType.Scorecard) + case _: + return ops_schemas.AdditionalModelInfo( + modelPackage=ops_schemas.ModelPackage.pmml, + modelType=ops_schemas.ModelType.other) + case _: + return ops_schemas.AdditionalModelInfo( + modelPackage=ops_schemas.ModelPackage.other, + modelType=ops_schemas.ModelType.other) @router.get( diff --git a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml index 63ffbb4..715d35a 100644 --- a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml +++ b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml @@ -958,47 +958,28 @@ components: type: object title: Additional Model Information required: + - modelPackage - modelType properties: - modelType: + modelPackage: type: string - description: the type of imported binary + description: The file format of binary model example: pmml - discriminator: - propertyName: modelType - mapping: - pmml: '#/components/schemas/AdditionalPMMLModelInfo' - pickle: '#/components/schemas/AdditionalPickleModelInfo' - other: '#/components/schemas/AdditionalOtherModelInfo' - oneOf: - - $ref: '#/components/schemas/AdditionalPMMLModelInfo' - - $ref: '#/components/schemas/AdditionalPickleModelInfo' - - $ref: '#/components/schemas/AdditionalOtherModelInfo' - AdditionalPMMLModelInfo: - required: - - modelType - - modelSubType - allOf: - - $ref: '#/components/schemas/AdditionalModelInfo' - - properties: - modelSubType: - type: string - description: PMML MODEL-ELEMENT. Could be RuleSetModel, Scorecard... - example: ruleset - AdditionalPickleModelInfo: - required: - - modelType - - pickleProtoVersion - allOf: - - $ref: '#/components/schemas/AdditionalModelInfo' - - properties: - pickleProtoVersion: - type: string - description: the version of pickle protocol - example: 4 - AdditionalOtherModelInfo: - allOf: - - $ref: '#/components/schemas/AdditionalModelInfo' + enum: + - pmml + - joblib + - pickle + - other + modelType: + type: string + description: Model type of binary model + example: RuleSetModel + enum: + # pmml ruleset + - RuleSetModel + # pmml scorecard + - Scorecard + - other parameters: ModelIDParam: in: path diff --git a/ops-implementations/ads-ml-service/app/schemas/impl.py b/ops-implementations/ads-ml-service/app/schemas/impl.py index e56db0d..04512c4 100644 --- a/ops-implementations/ads-ml-service/app/schemas/impl.py +++ b/ops-implementations/ads-ml-service/app/schemas/impl.py @@ -159,21 +159,3 @@ class PredictionImpl(pydt.BaseModel): parameters: typing.List[typing.Union[typing.List[ParameterImpl], ParameterImpl]] = pydt.Field( ..., description='Model parameters', title='Parameters' ) - - -class AdditionalPMMLModelInfo(pydt.BaseModel): - modelType: typing.Literal['pmml'] - modelSubType: str - - -class AdditionalPickleModelInfo(pydt.BaseModel): - modelType: typing.Literal['pickle'] - pickleProtoVersion: str - - -class AdditionalOtherModelInfo(pydt.BaseModel): - modelType: typing.Literal['other'] - - -AdditionalModelInfo = typing_extensions.Annotated[ - typing.Union[AdditionalPMMLModelInfo, AdditionalPickleModelInfo, AdditionalOtherModelInfo], pydt.Field(discriminator='modelType')] diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py index 8d39c51..2b565ac 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py @@ -263,8 +263,8 @@ def test_pickle_metadata( # Assert assert resp.ok - assert resp.json()['modelType'] == 'pickle' - assert resp.json()['pickleProtoVersion'] == '4' + assert resp.json()['modelPackage'] == 'pickle' + assert resp.json()['modelType'] == 'other' def test_pmml_metadata( @@ -280,8 +280,8 @@ def test_pmml_metadata( # Assert assert resp.ok - assert resp.json()['modelType'] == 'pmml' - assert resp.json()['modelSubType'] == 'Scorecard' + assert resp.json()['modelPackage'] == 'pmml' + assert resp.json()['modelType'] == 'Scorecard' def test_download_binary( diff --git a/ops-implementations/ads-ml-service/tox.ini b/ops-implementations/ads-ml-service/tox.ini index d76f3d9..79716e3 100755 --- a/ops-implementations/ads-ml-service/tox.ini +++ b/ops-implementations/ads-ml-service/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38 +envlist = py311 skipsdist = True [testenv] From 24d2de415440bd275fa24141468e1bee0a544e00 Mon Sep 17 00:00:00 2001 From: ke li Date: Wed, 18 Jan 2023 15:31:42 +0100 Subject: [PATCH 24/26] [DBACLD-47417] Update spec --- open-prediction-service.yaml | 11 -------- .../app/api/api_v2/endpoints/models.py | 28 +++++++------------ .../app/gen/tmp.schemas.ops.yaml | 11 -------- 3 files changed, 10 insertions(+), 40 deletions(-) diff --git a/open-prediction-service.yaml b/open-prediction-service.yaml index 715d35a..40273f2 100644 --- a/open-prediction-service.yaml +++ b/open-prediction-service.yaml @@ -965,21 +965,10 @@ components: type: string description: The file format of binary model example: pmml - enum: - - pmml - - joblib - - pickle - - other modelType: type: string description: Model type of binary model example: RuleSetModel - enum: - # pmml ruleset - - RuleSetModel - # pmml scorecard - - Scorecard - - other parameters: ModelIDParam: in: path diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index f2838f8..7eebb10 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -199,32 +199,24 @@ def get_model_metadata( case None: raise fastapi.HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f'Model with id {model_id} is not found') - case models.BinaryMlModel(format=app_binary_config.ModelWrapper.PICKLE): + case models.BinaryMlModel(format=app_binary_config.ModelWrapper.PICKLE | app_binary_config.ModelWrapper.JOBLIB as format): return ops_schemas.AdditionalModelInfo( - modelPackage=ops_schemas.ModelPackage.pickle, - modelType=ops_schemas.ModelType.other) - case models.BinaryMlModel(format=app_binary_config.ModelWrapper.JOBLIB): - return ops_schemas.AdditionalModelInfo( - modelPackage=ops_schemas.ModelPackage.joblib, - modelType=ops_schemas.ModelType.other) + modelPackage=format.value, + modelType='other') case models.BinaryMlModel(format=app_binary_config.ModelWrapper.PMML, model_b64=binary): match app_runtime_inspection.inspect_pmml_subtype(binary): - case 'RuleSetModel': - return ops_schemas.AdditionalModelInfo( - modelPackage=ops_schemas.ModelPackage.pmml, - modelType=ops_schemas.ModelType.RuleSetModel) - case 'Scorecard': + case 'Scorecard' | 'RuleSetModel' as typ: return ops_schemas.AdditionalModelInfo( - modelPackage=ops_schemas.ModelPackage.pmml, - modelType=ops_schemas.ModelType.Scorecard) + modelPackage='pmml', + modelType=typ) case _: return ops_schemas.AdditionalModelInfo( - modelPackage=ops_schemas.ModelPackage.pmml, - modelType=ops_schemas.ModelType.other) + modelPackage='pmml', + modelType='other') case _: return ops_schemas.AdditionalModelInfo( - modelPackage=ops_schemas.ModelPackage.other, - modelType=ops_schemas.ModelType.other) + modelPackage='other', + modelType='other') @router.get( diff --git a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml index 715d35a..40273f2 100644 --- a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml +++ b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml @@ -965,21 +965,10 @@ components: type: string description: The file format of binary model example: pmml - enum: - - pmml - - joblib - - pickle - - other modelType: type: string description: Model type of binary model example: RuleSetModel - enum: - # pmml ruleset - - RuleSetModel - # pmml scorecard - - Scorecard - - other parameters: ModelIDParam: in: path From 6de1e39e852eadd778a09695fb4be351bf911446 Mon Sep 17 00:00:00 2001 From: ke li Date: Thu, 19 Jan 2023 14:35:08 +0100 Subject: [PATCH 25/26] [DBACLD-47417] Refine tox.ini --- ops-implementations/ads-ml-service/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/tox.ini b/ops-implementations/ads-ml-service/tox.ini index 79716e3..c0cc3c9 100755 --- a/ops-implementations/ads-ml-service/tox.ini +++ b/ops-implementations/ads-ml-service/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py311 +envlist = py310, py311 skipsdist = True [testenv] From 087f8dd5f6e232acd6daa0bebf47b0f0312a9bd7 Mon Sep 17 00:00:00 2001 From: ke li Date: Thu, 19 Jan 2023 14:35:34 +0100 Subject: [PATCH 26/26] [DBACLD-47417] Remove unused inspect_pickle_version --- .../ads-ml-service/app/runtime/inspection.py | 4 ---- .../ads-ml-service/app/schemas/impl.py | 1 - .../runtime/test_signature_inspection.py | 23 ------------------- 3 files changed, 28 deletions(-) diff --git a/ops-implementations/ads-ml-service/app/runtime/inspection.py b/ops-implementations/ads-ml-service/app/runtime/inspection.py index 59a8343..6f63524 100644 --- a/ops-implementations/ads-ml-service/app/runtime/inspection.py +++ b/ops-implementations/ads-ml-service/app/runtime/inspection.py @@ -71,7 +71,3 @@ def inspect_pmml_model_name(model_file: bytes) -> typing.Optional[str]: def inspect_pmml_subtype(model_file: bytes) -> typing.Optional[str]: wrapper = load_pmml_model(model_file).loaded_model return wrapper.model.modelElement - - -def inspect_pickle_version(model_file: bytes) -> int: - return max(op[0].proto for op in pickletools.genops(model_file)) diff --git a/ops-implementations/ads-ml-service/app/schemas/impl.py b/ops-implementations/ads-ml-service/app/schemas/impl.py index 04512c4..7faddd9 100644 --- a/ops-implementations/ads-ml-service/app/schemas/impl.py +++ b/ops-implementations/ads-ml-service/app/schemas/impl.py @@ -17,7 +17,6 @@ import enum import typing -import typing_extensions import datetime as dt import numpy diff --git a/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py b/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py index 1ae4dfd..135dee0 100644 --- a/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py +++ b/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py @@ -55,29 +55,6 @@ def test_pmml_output_schema_inspection( } -def test_inspect_pickle_version(): - # Arrange - p0 = pickle.dumps('test_0', protocol=0) - p1 = pickle.dumps('test_1', protocol=1) - p2 = pickle.dumps('test_2', protocol=2) - p3 = pickle.dumps(str.encode('test_3'), protocol=3) - p4 = pickle.dumps('test_4', protocol=4) - - # When - ver_0 = app_signature_inspection.inspect_pickle_version(p0) - ver_1 = app_signature_inspection.inspect_pickle_version(p1) - ver_2 = app_signature_inspection.inspect_pickle_version(p2) - ver_3 = app_signature_inspection.inspect_pickle_version(p3) - ver_4 = app_signature_inspection.inspect_pickle_version(p4) - - # Assert - assert ver_0 == 0 - assert ver_1 == 1 - assert ver_2 == 2 - assert ver_3 == 3 - assert ver_4 == 4 - - def test_inspect_pmml_subtype(): # When sub_type_regression = app_signature_inspection.inspect_pmml_subtype(