From 64f763ec67f2536b3287b2350357573082788c57 Mon Sep 17 00:00:00 2001 From: "whitesource-ets[bot]" Date: Tue, 25 Oct 2022 00:57:54 +0000 Subject: [PATCH 01/42] Update dependency python to v3.11 --- ops-implementations/ads-ml-service/Dockerfile | 2 +- ops-implementations/sagemaker-service/Dockerfile | 2 +- ops-implementations/wml-service/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ops-implementations/ads-ml-service/Dockerfile b/ops-implementations/ads-ml-service/Dockerfile index a040487..0c06f74 100644 --- a/ops-implementations/ads-ml-service/Dockerfile +++ b/ops-implementations/ads-ml-service/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=${BUILDPLATFORM} python:3.8-slim AS python-base +FROM --platform=${BUILDPLATFORM} python:3.11-slim AS python-base ENV \ OPS_HOME="/usr/src/ads-ml-service" \ diff --git a/ops-implementations/sagemaker-service/Dockerfile b/ops-implementations/sagemaker-service/Dockerfile index b109d57..20fd9c3 100755 --- a/ops-implementations/sagemaker-service/Dockerfile +++ b/ops-implementations/sagemaker-service/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=${BUILDPLATFORM} python:3.7-slim +FROM --platform=${BUILDPLATFORM} python:3.11-slim RUN mkdir -p /usr/src/app WORKDIR /usr/src/app diff --git a/ops-implementations/wml-service/Dockerfile b/ops-implementations/wml-service/Dockerfile index 40492e4..0aa1555 100755 --- a/ops-implementations/wml-service/Dockerfile +++ b/ops-implementations/wml-service/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=${BUILDPLATFORM} python:3.7-slim +FROM --platform=${BUILDPLATFORM} python:3.11-slim RUN pip3 install connexion[swagger-ui] From 9520234f1defa809f417337ba7a68d2de4681087 Mon Sep 17 00:00:00 2001 From: "whitesource-ets[bot]" Date: Fri, 11 Nov 2022 14:21:34 +0000 Subject: [PATCH 02/42] chore(deps): update dependency postgres to v15.1 --- .../ads-ml-service/doc/installation/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/doc/installation/docker-compose.yaml b/ops-implementations/ads-ml-service/doc/installation/docker-compose.yaml index 63b1a91..b6dfd4b 100644 --- a/ops-implementations/ads-ml-service/doc/installation/docker-compose.yaml +++ b/ops-implementations/ads-ml-service/doc/installation/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3' services: db: - image: postgres:15.0 + image: postgres:15.1 restart: always environment: POSTGRES_DB: 'serving' From c09610097c124c96d4a27c91d831f279784ad27d Mon Sep 17 00:00:00 2001 From: "whitesource-ets[bot]" Date: Mon, 14 Nov 2022 10:14:53 +0000 Subject: [PATCH 03/42] chore(deps): update dependency sqlalchemy to v1.4.44 --- ops-implementations/ads-ml-service/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/requirements.txt b/ops-implementations/ads-ml-service/requirements.txt index ed62491..d37d7c7 100644 --- a/ops-implementations/ads-ml-service/requirements.txt +++ b/ops-implementations/ads-ml-service/requirements.txt @@ -2,7 +2,7 @@ ## Web interface python-multipart==0.0.5 -sqlalchemy==1.4.43 +sqlalchemy==1.4.44 alembic==1.8.1 psycopg2-binary==2.9.5 fastapi==0.86.0 From aafb14af58c3067af2c688f9fdeb512e9c56bdb5 Mon Sep 17 00:00:00 2001 From: "whitesource-ets[bot]" Date: Thu, 17 Nov 2022 22:29:04 +0000 Subject: [PATCH 04/42] fix(deps): update dependency io.swagger:swagger-annotations to v1.6.9 --- ops-client-sdk/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-client-sdk/pom.xml b/ops-client-sdk/pom.xml index 9927c82..8820b1d 100644 --- a/ops-client-sdk/pom.xml +++ b/ops-client-sdk/pom.xml @@ -224,7 +224,7 @@ limitations under the License.IBM Confidential]]> ${java.version} ${java.version} 1.8.5 - 1.6.8 + 1.6.9 4.10.0 2.10 3.12.0 From 77eb6b5632980410145d44bc327e1582578f6f97 Mon Sep 17 00:00:00 2001 From: "whitesource-ets[bot]" Date: Thu, 24 Nov 2022 06:39:28 +0000 Subject: [PATCH 05/42] chore(deps): update dependency setuptools to v65.6.3 --- ops-implementations/explanation-service/requirements.txt | 2 +- ops-implementations/wml-service/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ops-implementations/explanation-service/requirements.txt b/ops-implementations/explanation-service/requirements.txt index 0e1c7fb..674d959 100644 --- a/ops-implementations/explanation-service/requirements.txt +++ b/ops-implementations/explanation-service/requirements.txt @@ -7,7 +7,7 @@ connexion[swagger-ui] ==2.14.1; python_version=="3.5" or python_version=="3.4" werkzeug ==2.2.2; python_version=="3.5" or python_version=="3.4" swagger-ui-bundle ==0.0.9 python_dateutil ==2.8.2 -setuptools ==65.5.1 +setuptools ==65.6.3 numpy pandas sklearn diff --git a/ops-implementations/wml-service/requirements.txt b/ops-implementations/wml-service/requirements.txt index 0c94687..6a8fb3a 100755 --- a/ops-implementations/wml-service/requirements.txt +++ b/ops-implementations/wml-service/requirements.txt @@ -1,5 +1,5 @@ connexion[swagger-ui] ==2.14.1 python_dateutil ==2.8.2 -setuptools ==65.5.1 +setuptools ==65.6.3 jsonschema ==4.17.0 requests ==2.28.1 From 3dd400127e9cca217e4188f7529068bef3d85d25 Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 10:28:57 +0100 Subject: [PATCH 06/42] [DBACLD-47417] Update OPS spec --- open-prediction-service.yaml | 79 +++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/open-prediction-service.yaml b/open-prediction-service.yaml index a6ddbfc..5c4e9bd 100644 --- a/open-prediction-service.yaml +++ b/open-prediction-service.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Open Prediction Service - version: 2.7.4-SNAPSHOT + version: 2.8.0-SNAPSHOT description: The Open Prediction Service API is an effort to provide an Open API that enables unsupported native ML Providers in Decision Designer or Decision Runtime. tags: - name: info @@ -213,6 +213,47 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + /models/{model_id}/binary: + get: + tags: + - discover + summary: Download model binary + operationId: get_binary_by_id + parameters: + - $ref: '#/components/parameters/ModelIDParam' + responses: + '200': + description: Successful Response + content: + # A binary file: + application/octet-stream: {} + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /models/{model_id}/metadata: + get: + tags: + - discover + summary: Get parsed model metadata + operationId: get_metadata_by_id + parameters: + - $ref: '#/components/parameters/ModelIDParam' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/AdditionalModelInfo' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' /endpoints: get: tags: @@ -823,7 +864,7 @@ components: unknown_file_size: type: boolean default: false - description: Whether the user can upload a file whose length is unknown at the time of the request. + description: Whether the user can upload a file whose length(Content-Length in HTTP header) is unknown at the time of the request. example: capabilities: - info @@ -907,6 +948,40 @@ components: target: - rel: endpoint href: 'http://open-prediction-service.org/endpoints/8c2af534-cdce-11ea-87d0-0242ac130003' + AdditionalModelInfo: + type: object + title: Additional Model Information + required: + - modelType + properties: + modelType: + type: string + description: the type of imported binary + example: pmml + discriminator: + propertyName: modelType + mapping: + pmml: '#/components/schemas/AdditionalPMMLModelInfo' + pickle: '#/components/schemas/AdditionalPickleModelInfo' + oneOf: + - $ref: '#/components/schemas/AdditionalPMMLModelInfo' + - $ref: '#/components/schemas/AdditionalPickleModelInfo' + AdditionalPMMLModelInfo: + allOf: + - $ref: '#/components/schemas/AdditionalModelInfo' + - properties: + modelSubType: + type: string + description: model sub type + example: ruleset + AdditionalPickleModelInfo: + allOf: + - $ref: '#/components/schemas/AdditionalModelInfo' + - properties: + pickleProtoVersion: + type: string + description: the version of pickle protocol + example: 4 parameters: ModelIDParam: in: path From b4ea755110e05c2f5c127c6aafc909e7cfa7c38f Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 10:30:35 +0100 Subject: [PATCH 07/42] [DBACLD-47417] Update OPS spec --- .../app/gen/tmp.schemas.ops.yaml | 79 ++++++++++++++++++- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml index 4e0eae8..5c4e9bd 100644 --- a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml +++ b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Open Prediction Service - version: 2.7.0-SNAPSHOT + version: 2.8.0-SNAPSHOT description: The Open Prediction Service API is an effort to provide an Open API that enables unsupported native ML Providers in Decision Designer or Decision Runtime. tags: - name: info @@ -213,6 +213,47 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + /models/{model_id}/binary: + get: + tags: + - discover + summary: Download model binary + operationId: get_binary_by_id + parameters: + - $ref: '#/components/parameters/ModelIDParam' + responses: + '200': + description: Successful Response + content: + # A binary file: + application/octet-stream: {} + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /models/{model_id}/metadata: + get: + tags: + - discover + summary: Get parsed model metadata + operationId: get_metadata_by_id + parameters: + - $ref: '#/components/parameters/ModelIDParam' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/AdditionalModelInfo' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' /endpoints: get: tags: @@ -823,7 +864,7 @@ components: unknown_file_size: type: boolean default: false - description: Whether the user can upload a file whose length is unknown at the time of the request. + description: Whether the user can upload a file whose length(Content-Length in HTTP header) is unknown at the time of the request. example: capabilities: - info @@ -907,6 +948,40 @@ components: target: - rel: endpoint href: 'http://open-prediction-service.org/endpoints/8c2af534-cdce-11ea-87d0-0242ac130003' + AdditionalModelInfo: + type: object + title: Additional Model Information + required: + - modelType + properties: + modelType: + type: string + description: the type of imported binary + example: pmml + discriminator: + propertyName: modelType + mapping: + pmml: '#/components/schemas/AdditionalPMMLModelInfo' + pickle: '#/components/schemas/AdditionalPickleModelInfo' + oneOf: + - $ref: '#/components/schemas/AdditionalPMMLModelInfo' + - $ref: '#/components/schemas/AdditionalPickleModelInfo' + AdditionalPMMLModelInfo: + allOf: + - $ref: '#/components/schemas/AdditionalModelInfo' + - properties: + modelSubType: + type: string + description: model sub type + example: ruleset + AdditionalPickleModelInfo: + allOf: + - $ref: '#/components/schemas/AdditionalModelInfo' + - properties: + pickleProtoVersion: + type: string + description: the version of pickle protocol + example: 4 parameters: ModelIDParam: in: path From 10c4e212179068b653a865bdea2fe4aaf04815bb Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 15:25:45 +0100 Subject: [PATCH 08/42] [DBACLD-47417] Update runtime.inspection for model metadata --- .../ads-ml-service/app/runtime/inspection.py | 10 + .../app/tests/predictors/pmml_sample/model.py | 4 + .../predictors/pmml_sample/scorecard.pmml | 708 ++++++++++++++++++ .../runtime/test_signature_inspection.py | 36 + 4 files changed, 758 insertions(+) create mode 100644 ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/scorecard.pmml diff --git a/ops-implementations/ads-ml-service/app/runtime/inspection.py b/ops-implementations/ads-ml-service/app/runtime/inspection.py index e53755a..59a8343 100644 --- a/ops-implementations/ads-ml-service/app/runtime/inspection.py +++ b/ops-implementations/ads-ml-service/app/runtime/inspection.py @@ -17,6 +17,7 @@ import logging import typing +import pickletools import pypmml.base as pypmml_base @@ -65,3 +66,12 @@ def inspect_pmml_model_name(model_file: bytes) -> typing.Optional[str]: return wrapper.model.modelName else: return None + + +def inspect_pmml_subtype(model_file: bytes) -> typing.Optional[str]: + wrapper = load_pmml_model(model_file).loaded_model + return wrapper.model.modelElement + + +def inspect_pickle_version(model_file: bytes) -> int: + return max(op[0].proto for op in pickletools.genops(model_file)) diff --git a/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/model.py b/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/model.py index 1473e5a..6187c34 100644 --- a/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/model.py +++ b/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/model.py @@ -22,5 +22,9 @@ def get_pmml_file() -> pathlib.Path: return pathlib.Path(__file__).resolve().parent.joinpath('model.pmml') +def get_pmml_scorecard_file() -> pathlib.Path: + return pathlib.Path(__file__).resolve().parent.joinpath('scorecard.pmml') + + def get_pmml_no_output_schema_file() -> pathlib.Path: return pathlib.Path(__file__).resolve().parent.joinpath('model-no-output-schema.pmml') diff --git a/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/scorecard.pmml b/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/scorecard.pmml new file mode 100644 index 0000000..643949c --- /dev/null +++ b/ops-implementations/ads-ml-service/app/tests/predictors/pmml_sample/scorecard.pmml @@ -0,0 +1,708 @@ + + +
+ + 2022-11-25 05:46:41.942928 +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + "F" + + + + + "M" + + + + + + + "D" + + + + + "M" + + + + + "S" + + + + + + + "N" + + + + + "Y" + + + + + + + "Auto" + + + + + "CC" + + + + + "CH" + + + + + + diff --git a/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py b/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py index 99d6c45..1ae4dfd 100644 --- a/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py +++ b/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py @@ -16,6 +16,7 @@ import pathlib +import pickle import app.runtime.inspection as app_signature_inspection import app.tests.predictors.pmml_sample.model as app_test_pmml @@ -52,3 +53,38 @@ def test_pmml_output_schema_inspection( 'probability_1': 'double', 'predicted_paymentDefault': 'integer' } + + +def test_inspect_pickle_version(): + # Arrange + p0 = pickle.dumps('test_0', protocol=0) + p1 = pickle.dumps('test_1', protocol=1) + p2 = pickle.dumps('test_2', protocol=2) + p3 = pickle.dumps(str.encode('test_3'), protocol=3) + p4 = pickle.dumps('test_4', protocol=4) + + # When + ver_0 = app_signature_inspection.inspect_pickle_version(p0) + ver_1 = app_signature_inspection.inspect_pickle_version(p1) + ver_2 = app_signature_inspection.inspect_pickle_version(p2) + ver_3 = app_signature_inspection.inspect_pickle_version(p3) + ver_4 = app_signature_inspection.inspect_pickle_version(p4) + + # Assert + assert ver_0 == 0 + assert ver_1 == 1 + assert ver_2 == 2 + assert ver_3 == 3 + assert ver_4 == 4 + + +def test_inspect_pmml_subtype(): + # When + sub_type_regression = app_signature_inspection.inspect_pmml_subtype( + str.encode(app_test_pmml.get_pmml_file().read_text())) + sub_type_scorecard = app_signature_inspection.inspect_pmml_subtype( + str.encode(app_test_pmml.get_pmml_scorecard_file().read_text())) + + # Assert + assert sub_type_regression == 'RegressionModel' + assert sub_type_scorecard == 'Scorecard' From 354cf044ac07b69971b4dbdd41f9fd3ee59ea542 Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 16:59:20 +0100 Subject: [PATCH 09/42] [DBACLD-47417] Add /models/id/metadata --- .../app/api/api_v2/endpoints/models.py | 28 +++++++++++ .../app/tests/api/api_v2/test_models.py | 48 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index 0d8cb61..20221e5 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -28,6 +28,7 @@ import app.crud as crud import app.gen.schemas.ops_schemas as ops_schemas import app.runtime.model_upload as app_model_upload +import app.runtime.inspection as app_runtime_inspection import app.schemas as schemas import app.schemas.binary_config as app_binary_config import app.schemas.impl as impl @@ -182,3 +183,30 @@ async def add_binary( model_id=model_id ) return impl.EndpointImpl.from_database(crud.endpoint.get(db, id=m)) + + +@router.get( + path='/models/{model_id}/metadata', + response_model=typing.Union[ops_schemas.AdditionalPickleModelInfo, ops_schemas.AdditionalPMMLModelInfo], + tags=['discover'] +) +def get_model_metadata( + model_id: int, + db: saorm.Session = fastapi.Depends(deps.get_db)): + LOGGER.info('Retrieving model metadata for id: %s', model_id) + model = crud.binary_ml_model.get(db=db, id=model_id) + + if model is None: + raise fastapi.HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f'Model with id {model_id} is not found') + + if model.format == app_binary_config.ModelWrapper.PICKLE: + return ops_schemas.AdditionalPickleModelInfo( + modelType='pickle', pickleProtoVersion=str(app_runtime_inspection.inspect_pickle_version(model.model_b64))) + elif model.format == app_binary_config.ModelWrapper.PMML: + return ops_schemas.AdditionalPMMLModelInfo( + modelType='pmml', modelSubType=str(app_runtime_inspection.inspect_pmml_subtype(model.model_b64))) + else: + raise fastapi.HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f'Format {model.format} is not supported for metadata') diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py index f2bd56c..d7a01e2 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py @@ -31,6 +31,8 @@ import app.schemas as schemas import app.tests.predictors.identity.model as app_tests_identity import app.tests.predictors.scikit_learn.model as app_test_skl +import app.models as models +import app.tests.predictors.pmml_sample.model as app_test_pmml def test_get_model( @@ -233,3 +235,49 @@ def test_update_binary( assert response.status_code == 201 assert response.json()['status'] == 'in_service' assert response_1.status_code == 422 + + +def test_not_supported_metadata( + client: tstc.TestClient, + xgboost_endpoint: models.Endpoint +) -> typ.NoReturn: + # When + resp = client.get(url=f'/models/{xgboost_endpoint.id}/metadata') + + # Assert + assert resp.status_code == 422 + assert resp.json()['detail'] == 'Format bst is not supported for metadata' + + +def test_pickle_metadata( + client: tstc.TestClient +) -> typ.NoReturn: + # When + model = client.post( + url='/upload', + data={'format': 'pickle'}, + files={'file': ('model.pkl', pickle.dumps(app_test_skl.get_classification_predictor()))}).json() + model_id = model['id'] + resp = client.get(url=f'/models/{model_id}/metadata') + + # Assert + assert resp.ok + assert resp.json()['modelType'] == 'pickle' + assert resp.json()['pickleProtoVersion'] == '4' + + +def test_pmml_metadata( + client: tstc.TestClient +) -> typ.NoReturn: + # When + model = client.post( + url='/upload', + data={'format': 'pmml'}, + files={'file': ('scorecard.pmml', app_test_pmml.get_pmml_scorecard_file().read_text())}).json() + model_id = model['id'] + resp = client.get(url=f'/models/{model_id}/metadata') + + # Assert + assert resp.ok + assert resp.json()['modelType'] == 'pmml' + assert resp.json()['modelSubType'] == 'Scorecard' From cf47ec6eed54195a5543f0e2633d9853c49c0701 Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 17:08:19 +0100 Subject: [PATCH 10/42] [DBACLD-47417] Fix discriminator --- .../app/api/api_v2/endpoints/models.py | 6 +++--- .../ads-ml-service/app/schemas/impl.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index 20221e5..95e6beb 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -187,7 +187,7 @@ async def add_binary( @router.get( path='/models/{model_id}/metadata', - response_model=typing.Union[ops_schemas.AdditionalPickleModelInfo, ops_schemas.AdditionalPMMLModelInfo], + response_model=impl.AdditionalModelInfo, tags=['discover'] ) def get_model_metadata( @@ -201,10 +201,10 @@ def get_model_metadata( status_code=status.HTTP_404_NOT_FOUND, detail=f'Model with id {model_id} is not found') if model.format == app_binary_config.ModelWrapper.PICKLE: - return ops_schemas.AdditionalPickleModelInfo( + return impl.AdditionalPickleModelInfo( modelType='pickle', pickleProtoVersion=str(app_runtime_inspection.inspect_pickle_version(model.model_b64))) elif model.format == app_binary_config.ModelWrapper.PMML: - return ops_schemas.AdditionalPMMLModelInfo( + return impl.AdditionalPMMLModelInfo( modelType='pmml', modelSubType=str(app_runtime_inspection.inspect_pmml_subtype(model.model_b64))) else: raise fastapi.HTTPException( diff --git a/ops-implementations/ads-ml-service/app/schemas/impl.py b/ops-implementations/ads-ml-service/app/schemas/impl.py index 7faddd9..3e84f5d 100644 --- a/ops-implementations/ads-ml-service/app/schemas/impl.py +++ b/ops-implementations/ads-ml-service/app/schemas/impl.py @@ -17,6 +17,7 @@ import enum import typing +import typing_extensions import datetime as dt import numpy @@ -158,3 +159,17 @@ class PredictionImpl(pydt.BaseModel): parameters: typing.List[typing.Union[typing.List[ParameterImpl], ParameterImpl]] = pydt.Field( ..., description='Model parameters', title='Parameters' ) + + +class AdditionalPMMLModelInfo(pydt.BaseModel): + modelType: typing.Literal['pmml'] + modelSubType: typing.Optional[str] + + +class AdditionalPickleModelInfo(pydt.BaseModel): + modelType: typing.Literal['pickle'] + pickleProtoVersion: typing.Optional[str] + + +AdditionalModelInfo = typing_extensions.Annotated[ + typing.Union[AdditionalPMMLModelInfo, AdditionalPickleModelInfo], pydt.Field(discriminator='modelType')] From 3c0ec47038f15a03890c456dd79067d2bd2ac7e0 Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 17:33:29 +0100 Subject: [PATCH 11/42] [DBACLD-47417] Add binary download --- .../app/api/api_v2/endpoints/models.py | 23 +++++++++++++++++-- .../app/tests/api/api_v2/test_models.py | 17 ++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index 95e6beb..fe16a76 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -17,6 +17,7 @@ import logging import typing +import io import fastapi import fastapi.encoders as encoders @@ -188,8 +189,7 @@ async def add_binary( @router.get( path='/models/{model_id}/metadata', response_model=impl.AdditionalModelInfo, - tags=['discover'] -) + tags=['discover']) def get_model_metadata( model_id: int, db: saorm.Session = fastapi.Depends(deps.get_db)): @@ -210,3 +210,22 @@ def get_model_metadata( raise fastapi.HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f'Format {model.format} is not supported for metadata') + + +@router.get( + path='/models/{model_id}/binary', + tags=['discover']) +def get_model_binary( + model_id: int, + db: saorm.Session = fastapi.Depends(deps.get_db)): + LOGGER.info('Retrieving model binary for id: %s', model_id) + model = crud.binary_ml_model.get(db=db, id=model_id) + + if model is None: + raise fastapi.HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f'Model binary with id {model_id} is not found') + + filelike = io.BytesIO(model.model_b64) + filelike.seek(0) + + return responses.StreamingResponse(content=filelike, media_type='application/octet-stream') diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py index d7a01e2..9b267d3 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py @@ -281,3 +281,20 @@ def test_pmml_metadata( assert resp.ok assert resp.json()['modelType'] == 'pmml' assert resp.json()['modelSubType'] == 'Scorecard' + + +def test_download_binary( + client: tstc.TestClient +) -> typ.NoReturn: + # When + model_content = app_test_pmml.get_pmml_scorecard_file().read_text() + model = client.post( + url='/upload', + data={'format': 'pmml'}, + files={'file': ('scorecard.pmml', model_content)}).json() + model_id = model['id'] + resp = client.get(url=f'/models/{model_id}/binary') + + # Assert + assert resp.ok + assert resp.content == str.encode(model_content) From eb9480dc203b698a376695c357f92e97eb0953dc Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 17:52:18 +0100 Subject: [PATCH 12/42] [DBACLD-47417] Add response_class for binary download --- .../ads-ml-service/app/api/api_v2/endpoints/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index fe16a76..c7b2ada 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -214,6 +214,7 @@ def get_model_metadata( @router.get( path='/models/{model_id}/binary', + response_class=responses.StreamingResponse, tags=['discover']) def get_model_binary( model_id: int, From 0826404fa5e3e2ba6886204f7afee1039737f0ba Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 17:54:36 +0100 Subject: [PATCH 13/42] [DBACLD-47417] Bump ver --- ops-client-sdk/pom.xml | 2 +- ops-implementations/ads-ml-service/app/version.py | 2 +- ops-implementations/ads-ml-service/pom.xml | 2 +- ops-implementations/pom.xml | 2 +- ops-implementations/sagemaker-service/openapi_server/version.py | 2 +- ops-implementations/sagemaker-service/pom.xml | 2 +- ops-implementations/wml-service/pom.xml | 2 +- ops-implementations/wml-service/swagger_server/version.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ops-client-sdk/pom.xml b/ops-client-sdk/pom.xml index 9927c82..09337fb 100644 --- a/ops-client-sdk/pom.xml +++ b/ops-client-sdk/pom.xml @@ -6,7 +6,7 @@ com.ibm.decision ops-client-sdk - 2.7.4-SNAPSHOT + 2.8.0-SNAPSHOT 2020 diff --git a/ops-implementations/ads-ml-service/app/version.py b/ops-implementations/ads-ml-service/app/version.py index 1d8e139..bb4701d 100644 --- a/ops-implementations/ads-ml-service/app/version.py +++ b/ops-implementations/ads-ml-service/app/version.py @@ -1 +1 @@ -__version__ = '2.7.4-SNAPSHOT' +__version__ = '2.8.0-SNAPSHOT' diff --git a/ops-implementations/ads-ml-service/pom.xml b/ops-implementations/ads-ml-service/pom.xml index 3347a87..e721157 100644 --- a/ops-implementations/ads-ml-service/pom.xml +++ b/ops-implementations/ads-ml-service/pom.xml @@ -5,7 +5,7 @@ com.ibm.decision.ops ml-service-implementations - 2.7.4-SNAPSHOT + 2.8.0-SNAPSHOT .. diff --git a/ops-implementations/pom.xml b/ops-implementations/pom.xml index 130c984..a6e0fc6 100644 --- a/ops-implementations/pom.xml +++ b/ops-implementations/pom.xml @@ -4,7 +4,7 @@ com.ibm.decision.ops ml-service-implementations - 2.7.4-SNAPSHOT + 2.8.0-SNAPSHOT pom diff --git a/ops-implementations/sagemaker-service/openapi_server/version.py b/ops-implementations/sagemaker-service/openapi_server/version.py index 1d8e139..bb4701d 100644 --- a/ops-implementations/sagemaker-service/openapi_server/version.py +++ b/ops-implementations/sagemaker-service/openapi_server/version.py @@ -1 +1 @@ -__version__ = '2.7.4-SNAPSHOT' +__version__ = '2.8.0-SNAPSHOT' diff --git a/ops-implementations/sagemaker-service/pom.xml b/ops-implementations/sagemaker-service/pom.xml index 1e9b29f..555e3cf 100644 --- a/ops-implementations/sagemaker-service/pom.xml +++ b/ops-implementations/sagemaker-service/pom.xml @@ -5,7 +5,7 @@ com.ibm.decision.ops ml-service-implementations - 2.7.4-SNAPSHOT + 2.8.0-SNAPSHOT .. diff --git a/ops-implementations/wml-service/pom.xml b/ops-implementations/wml-service/pom.xml index 8f3a7e4..b15d539 100644 --- a/ops-implementations/wml-service/pom.xml +++ b/ops-implementations/wml-service/pom.xml @@ -5,7 +5,7 @@ com.ibm.decision.ops ml-service-implementations - 2.7.4-SNAPSHOT + 2.8.0-SNAPSHOT .. diff --git a/ops-implementations/wml-service/swagger_server/version.py b/ops-implementations/wml-service/swagger_server/version.py index 1d8e139..bb4701d 100644 --- a/ops-implementations/wml-service/swagger_server/version.py +++ b/ops-implementations/wml-service/swagger_server/version.py @@ -1 +1 @@ -__version__ = '2.7.4-SNAPSHOT' +__version__ = '2.8.0-SNAPSHOT' From 180a9f75e4ba6bf732ee8dc45f4abfcbb7c17738 Mon Sep 17 00:00:00 2001 From: ke li Date: Mon, 28 Nov 2022 17:59:15 +0100 Subject: [PATCH 14/42] [DBACLD-47417] Add changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b674e8f..6b48e97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Open Prediction Service HUB +## 2.8.0 _2022/11/28_ + +* [DBACLD-47417] Extend OPS to support downloading of Ruleset Models + ## 2.7.1 _2022/09/15_ * [DBACLD-58320] Fix ads ml service rule import From 181420a270fbaecea56fdba7d1c162cb32a4ad7e Mon Sep 17 00:00:00 2001 From: ke li Date: Wed, 30 Nov 2022 11:40:32 +0100 Subject: [PATCH 15/42] [DBACLD-47417] Add filename support for model download --- .../app/api/api_v2/endpoints/models.py | 27 ++++++++++++++++--- .../app/tests/api/api_v2/test_models.py | 3 +++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index c7b2ada..b478887 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -220,13 +220,32 @@ def get_model_binary( model_id: int, db: saorm.Session = fastapi.Depends(deps.get_db)): LOGGER.info('Retrieving model binary for id: %s', model_id) - model = crud.binary_ml_model.get(db=db, id=model_id) + + model = crud.model.get(db=db, id=model_id) if model is None: + raise fastapi.HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f'Model with id {model_id} is not found') + + filename = model.config.configuration['name'] + + try: + binary = model.endpoint.binary + except AttributeError: raise fastapi.HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f'Model binary with id {model_id} is not found') - filelike = io.BytesIO(model.model_b64) - filelike.seek(0) + if binary.format == app_binary_config.ModelWrapper.PMML: + file_extension = 'pmml' + elif binary.format == app_binary_config.ModelWrapper.PICKLE: + file_extension = 'pickle' + elif binary.format == app_binary_config.ModelWrapper.JOBLIB: + file_extension = 'pickle' + else: + file_extension = 'bin' - return responses.StreamingResponse(content=filelike, media_type='application/octet-stream') + return responses.StreamingResponse( + content=io.BytesIO(binary.model_b64), + media_type='application/octet-stream', + headers={'Content-Disposition': 'attachment; filename="{basename}.{extension}"'.format( + basename=filename, extension=file_extension)}) diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py index 9b267d3..d878577 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py @@ -20,6 +20,7 @@ import time import typing import typing as typ +import re import pytest import fastapi.testclient as tstc @@ -294,7 +295,9 @@ def test_download_binary( files={'file': ('scorecard.pmml', model_content)}).json() model_id = model['id'] resp = client.get(url=f'/models/{model_id}/binary') + received_filename = re.findall("filename=\"(.+)\"", resp.headers['content-disposition'])[0] # Assert assert resp.ok assert resp.content == str.encode(model_content) + assert received_filename == 'scorecard.pmml' From 6d62f418c60ea70042202a782d1f5f902d5b0d2d Mon Sep 17 00:00:00 2001 From: ke li Date: Thu, 1 Dec 2022 13:42:33 +0100 Subject: [PATCH 16/42] [DBACLD-47417] Add log for model download --- .../ads-ml-service/app/api/api_v2/endpoints/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index b478887..353712e 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -244,6 +244,8 @@ def get_model_binary( else: file_extension = 'bin' + LOGGER.info('Downloading model binary with name %s and format %s', filename, file_extension) + return responses.StreamingResponse( content=io.BytesIO(binary.model_b64), media_type='application/octet-stream', From f6624a08d9485d162af596b37bfd999b0eb49b1c Mon Sep 17 00:00:00 2001 From: ke li Date: Thu, 1 Dec 2022 13:42:55 +0100 Subject: [PATCH 17/42] [DBACLD-47417] Fix model type --- .../ads-ml-service/app/api/api_v2/endpoints/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index 353712e..4b57f62 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -240,7 +240,7 @@ def get_model_binary( elif binary.format == app_binary_config.ModelWrapper.PICKLE: file_extension = 'pickle' elif binary.format == app_binary_config.ModelWrapper.JOBLIB: - file_extension = 'pickle' + file_extension = 'joblib' else: file_extension = 'bin' From 5cdd71f9f613b84ea020f8b494611c87cc8c6341 Mon Sep 17 00:00:00 2001 From: ke li Date: Thu, 1 Dec 2022 14:06:11 +0100 Subject: [PATCH 18/42] [DBACLD-47417] Update spec for cap --- open-prediction-service.yaml | 8 +++++++- .../ads-ml-service/app/gen/tmp.schemas.ops.yaml | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/open-prediction-service.yaml b/open-prediction-service.yaml index 5c4e9bd..4b3c125 100644 --- a/open-prediction-service.yaml +++ b/open-prediction-service.yaml @@ -213,7 +213,7 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' - /models/{model_id}/binary: + /models/{model_id}/download: get: tags: - discover @@ -871,6 +871,8 @@ components: - discover - manage - prediction + - download + - metadata managed_capabilities: supported_input_data_structure: - "auto" @@ -895,6 +897,10 @@ components: - discover - manage - run + # Download model binary + - download + # Inspect model specific metadata, for example for PMML model return subType (Scorecard, Ruleset) + - metadata Parameter: title: Parameter description: Parameter for ml model invocation diff --git a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml index 5c4e9bd..4b3c125 100644 --- a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml +++ b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml @@ -213,7 +213,7 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' - /models/{model_id}/binary: + /models/{model_id}/download: get: tags: - discover @@ -871,6 +871,8 @@ components: - discover - manage - prediction + - download + - metadata managed_capabilities: supported_input_data_structure: - "auto" @@ -895,6 +897,10 @@ components: - discover - manage - run + # Download model binary + - download + # Inspect model specific metadata, for example for PMML model return subType (Scorecard, Ruleset) + - metadata Parameter: title: Parameter description: Parameter for ml model invocation From 3a08e84bd8284e5a91cbefa4c9026628ed5c9462 Mon Sep 17 00:00:00 2001 From: ke li Date: Thu, 1 Dec 2022 14:18:03 +0100 Subject: [PATCH 19/42] [DBACLD-47417] Add cap --- .../ads-ml-service/app/api/api_v2/endpoints/capabilities.py | 4 +++- .../ads-ml-service/app/tests/api/api_v2/test_capabilities.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/capabilities.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/capabilities.py index b3f9f0b..6e9e92f 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/capabilities.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/capabilities.py @@ -35,7 +35,9 @@ async def server_capabilities() -> typing.Dict[typing.Text, typing.Any]: ops_model.Capability.info, ops_model.Capability.discover, ops_model.Capability.manage, - ops_model.Capability.run + ops_model.Capability.run, + ops_model.Capability.download, + ops_model.Capability.metadata ], 'managed_capabilities': { 'supported_input_data_structure': ['auto', 'DataFrame', 'ndarray', 'DMatrix', 'list'], diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_capabilities.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_capabilities.py index b9fb241..aeb6f13 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_capabilities.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_capabilities.py @@ -34,6 +34,8 @@ def test_get_server_capabilities( assert 'discover' in content['capabilities'] assert 'manage' in content['capabilities'] assert 'run' in content['capabilities'] + assert 'download' in content['capabilities'] + assert 'metadata' in content['capabilities'] def test_get_managed_capabilities( From b0b32eda033f50a3676317473bf76205249cf0ad Mon Sep 17 00:00:00 2001 From: "whitesource-ets[bot]" Date: Fri, 2 Dec 2022 09:14:52 +0000 Subject: [PATCH 20/42] chore(deps): update dependency boto3 to v1.26.21 --- ops-implementations/sagemaker-service/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/sagemaker-service/requirements.txt b/ops-implementations/sagemaker-service/requirements.txt index 67f4a52..2084528 100644 --- a/ops-implementations/sagemaker-service/requirements.txt +++ b/ops-implementations/sagemaker-service/requirements.txt @@ -8,5 +8,5 @@ openapi-schema-validator==0.3.4 protobuf==3.20.3 # --- Dependences of sagemaker --- end -boto3==1.26.3 +boto3==1.26.21 sagemaker==2.116.0 From 733a8e140b96d59e1650b70043fc71f17d5bb534 Mon Sep 17 00:00:00 2001 From: ke li Date: Fri, 2 Dec 2022 13:42:11 +0100 Subject: [PATCH 21/42] fix dep: scikit-learn==1.1.3 is needed for python3.11 --- ops-implementations/ads-ml-service/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/requirements.txt b/ops-implementations/ads-ml-service/requirements.txt index 1d31c1e..55b119b 100644 --- a/ops-implementations/ads-ml-service/requirements.txt +++ b/ops-implementations/ads-ml-service/requirements.txt @@ -28,7 +28,7 @@ PyYAML==6.0 # ML xgboost==1.2.1 -scikit-learn==1.1.2 +scikit-learn==1.1.3 pypmml==0.9.9 py4j==0.10.9.7 From 7fff52efbcb97fcf04622860880933d6e6cc860a Mon Sep 17 00:00:00 2001 From: ke li Date: Fri, 2 Dec 2022 14:01:06 +0100 Subject: [PATCH 22/42] fix install: add build-essential and c libs --- ops-implementations/ads-ml-service/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/Dockerfile b/ops-implementations/ads-ml-service/Dockerfile index 0c06f74..5b1a051 100644 --- a/ops-implementations/ads-ml-service/Dockerfile +++ b/ops-implementations/ads-ml-service/Dockerfile @@ -15,6 +15,7 @@ FROM python-base AS codegen-image COPY app/gen/. app/gen/ RUN \ + apt-get -q update && apt-get -q install --assume-yes build-essential && \ python3 -m pip install -q --no-cache-dir --upgrade pip && \ python3 -m pip install -q datamodel-code-generator~=0.11.19 && \ datamodel-codegen \ @@ -30,8 +31,9 @@ COPY requirements.txt ./ COPY logging.yaml /etc/ads-ml-service/logging/logging.yaml RUN \ python3 -m venv /usr/src/ads-ml-service/venv && \ + apt-get -q update && apt-get -q install --assume-yes build-essential libpq-dev python3-numpy && \ python3 -m pip install -q --no-cache-dir --upgrade pip && \ - python3 -m pip install -q --no-cache-dir --requirement requirements.txt && \ + python3 -m pip install -q --no-cache-dir --prefer-binary --requirement requirements.txt && \ mkdir -p /var/log/ads-ml-service && \ mkdir -p /var/lib/ads-ml-service/models From 578eb0b4de88e15c21aa1abc090e2785d8a43e10 Mon Sep 17 00:00:00 2001 From: "whitesource-ets[bot]" Date: Fri, 2 Dec 2022 13:16:33 +0000 Subject: [PATCH 23/42] chore(deps): update dependency orjson to v3.8.2 --- ops-implementations/ads-ml-service/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/requirements.txt b/ops-implementations/ads-ml-service/requirements.txt index ed62491..b818d32 100644 --- a/ops-implementations/ads-ml-service/requirements.txt +++ b/ops-implementations/ads-ml-service/requirements.txt @@ -6,7 +6,7 @@ sqlalchemy==1.4.43 alembic==1.8.1 psycopg2-binary==2.9.5 fastapi==0.86.0 -orjson==3.8.1 +orjson==3.8.2 pandas==1.5.1 ## ASGI server From 34c35e5f7848beda13e730c588908a2b50ea5221 Mon Sep 17 00:00:00 2001 From: "whitesource-ets[bot]" Date: Fri, 2 Dec 2022 13:16:58 +0000 Subject: [PATCH 24/42] chore(deps): update dependency sagemaker to v2.118.0 --- ops-implementations/sagemaker-service/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/sagemaker-service/requirements.txt b/ops-implementations/sagemaker-service/requirements.txt index 2084528..8730624 100644 --- a/ops-implementations/sagemaker-service/requirements.txt +++ b/ops-implementations/sagemaker-service/requirements.txt @@ -9,4 +9,4 @@ protobuf==3.20.3 # --- Dependences of sagemaker --- end boto3==1.26.21 -sagemaker==2.116.0 +sagemaker==2.118.0 From 2b398bcce9958724fd919cedb745c2c173b3f118 Mon Sep 17 00:00:00 2001 From: "whitesource-ets[bot]" Date: Fri, 2 Dec 2022 13:19:09 +0000 Subject: [PATCH 25/42] chore(deps): update dependency jsonschema to v4.17.3 --- ops-implementations/wml-service/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/wml-service/requirements.txt b/ops-implementations/wml-service/requirements.txt index 6a8fb3a..f3d2b2e 100755 --- a/ops-implementations/wml-service/requirements.txt +++ b/ops-implementations/wml-service/requirements.txt @@ -1,5 +1,5 @@ connexion[swagger-ui] ==2.14.1 python_dateutil ==2.8.2 setuptools ==65.6.3 -jsonschema ==4.17.0 +jsonschema ==4.17.3 requests ==2.28.1 From 5930a4ea48fbe0df2d4c8fcec63111fb52836dbc Mon Sep 17 00:00:00 2001 From: "whitesource-ets[bot]" Date: Fri, 2 Dec 2022 14:21:00 +0000 Subject: [PATCH 26/42] chore(deps): update dependency tox to v4.0.0rc1 --- ops-implementations/ads-ml-service/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/requirements.txt b/ops-implementations/ads-ml-service/requirements.txt index d37d7c7..997e63f 100644 --- a/ops-implementations/ads-ml-service/requirements.txt +++ b/ops-implementations/ads-ml-service/requirements.txt @@ -33,7 +33,7 @@ pypmml==0.9.17 py4j==0.10.9.7 ## Testes -tox==4.0.0b2 +tox==4.0.0rc1 requests==2.28.1 pytest==7.2.0 nyoka==5.4.0 From d9afcd28347522c686a5ac4ffe1286e5ec0d12ef Mon Sep 17 00:00:00 2001 From: "whitesource-ets[bot]" Date: Fri, 2 Dec 2022 14:21:51 +0000 Subject: [PATCH 27/42] chore(deps): update dependency pandas to v1.5.2 --- ops-implementations/ads-ml-service/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/requirements.txt b/ops-implementations/ads-ml-service/requirements.txt index 3c21017..6069062 100644 --- a/ops-implementations/ads-ml-service/requirements.txt +++ b/ops-implementations/ads-ml-service/requirements.txt @@ -7,7 +7,7 @@ alembic==1.8.1 psycopg2-binary==2.9.5 fastapi==0.86.0 orjson==3.8.2 -pandas==1.5.1 +pandas==1.5.2 ## ASGI server hypercorn==0.14.3 From 7fe855250b5c6c578e1e1de0b5568874932fed19 Mon Sep 17 00:00:00 2001 From: ke li Date: Fri, 2 Dec 2022 15:43:46 +0100 Subject: [PATCH 28/42] Fix CVE-2022-42969 --- ops-implementations/wml-service/test-requirements.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/ops-implementations/wml-service/test-requirements.txt b/ops-implementations/wml-service/test-requirements.txt index c4785bf..9291f70 100755 --- a/ops-implementations/wml-service/test-requirements.txt +++ b/ops-implementations/wml-service/test-requirements.txt @@ -1,6 +1,3 @@ flask_testing==0.8.1 coverage==6.5.0 nose==1.3.7 -pluggy==1.0.0 -py==1.11.0 -randomize==0.14 From 91435a790e51ec300040bec1de5457aabdd931a2 Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 13 Dec 2022 10:10:15 +0100 Subject: [PATCH 29/42] [DBACLD-47417] Update endpoint name --- .../ads-ml-service/app/api/api_v2/endpoints/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index 4b57f62..6631356 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -213,7 +213,7 @@ def get_model_metadata( @router.get( - path='/models/{model_id}/binary', + path='/models/{model_id}/download', response_class=responses.StreamingResponse, tags=['discover']) def get_model_binary( From 7cc92094bc787877aa6652b9bd5f675d1b9b2219 Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 13 Dec 2022 10:13:21 +0100 Subject: [PATCH 30/42] [DBACLD-47417] Fix upload format check --- .../ads-ml-service/app/api/api_v2/endpoints/upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/upload.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/upload.py index 60cb18a..7986364 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/upload.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/upload.py @@ -77,7 +77,7 @@ async def upload( model_binary, input_data_structure, output_data_structure, - format_, + file_format, name=model_name ) From 98afc61ccc99dc4665dc87f4aa56d9e3e5dad5a8 Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 13 Dec 2022 10:14:28 +0100 Subject: [PATCH 31/42] [DBACLD-47417] Fix db init for debug launcher --- ops-implementations/ads-ml-service/app/db/session.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/app/db/session.py b/ops-implementations/ads-ml-service/app/db/session.py index 750c265..521ffd2 100644 --- a/ops-implementations/ads-ml-service/app/db/session.py +++ b/ops-implementations/ads-ml-service/app/db/session.py @@ -43,7 +43,11 @@ def get_db_opts() -> typing.Dict[str, typing.Any]: return opt +def get_engine(): + return create_engine(get_db_url(), **get_db_opts()) + + SessionLocal = sessionmaker( autocommit=False, autoflush=False, - bind=create_engine(get_db_url(), **get_db_opts())) + bind=get_engine()) From e20507b897f07c059d24c82ee512cffe99095d67 Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 13 Dec 2022 11:11:33 +0100 Subject: [PATCH 32/42] [DBACLD-47417] Update tests for new download endpoint --- .../ads-ml-service/app/tests/api/api_v2/test_models.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py index d878577..5701869 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py @@ -291,13 +291,12 @@ def test_download_binary( model_content = app_test_pmml.get_pmml_scorecard_file().read_text() model = client.post( url='/upload', - data={'format': 'pmml'}, files={'file': ('scorecard.pmml', model_content)}).json() model_id = model['id'] - resp = client.get(url=f'/models/{model_id}/binary') - received_filename = re.findall("filename=\"(.+)\"", resp.headers['content-disposition'])[0] + resp = client.get(url=f'/models/{model_id}/download') # Assert assert resp.ok assert resp.content == str.encode(model_content) - assert received_filename == 'scorecard.pmml' + # filename + assert re.findall("filename=\"(.+)\"", resp.headers['content-disposition'])[0] == 'scorecard.pmml' From fd6b07a7dc607eb00b457d8e31d69094aa72e094 Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 13 Dec 2022 14:56:40 +0100 Subject: [PATCH 33/42] [DBACLD-47417] Fix merge issue --- .../ads-ml-service/app/api/api_v2/endpoints/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index 6631356..65faea2 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -209,7 +209,7 @@ def get_model_metadata( else: raise fastapi.HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=f'Format {model.format} is not supported for metadata') + detail=f'Format {model.format.value} is not supported for metadata') @router.get( From 9663da0a9ecb3f4a63c44f9bb101fd870cde2e59 Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 20 Dec 2022 09:39:58 +0100 Subject: [PATCH 34/42] [DBACLD-47417] Update spec for better description and required --- open-prediction-service.yaml | 8 +++++++- .../ads-ml-service/app/gen/tmp.schemas.ops.yaml | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/open-prediction-service.yaml b/open-prediction-service.yaml index 4b3c125..6e8ff84 100644 --- a/open-prediction-service.yaml +++ b/open-prediction-service.yaml @@ -973,14 +973,20 @@ components: - $ref: '#/components/schemas/AdditionalPMMLModelInfo' - $ref: '#/components/schemas/AdditionalPickleModelInfo' AdditionalPMMLModelInfo: + required: + - modelType + - modelSubType allOf: - $ref: '#/components/schemas/AdditionalModelInfo' - properties: modelSubType: type: string - description: model sub type + description: PMML MODEL-ELEMENT. Could be RuleSetModel, Scorecard... example: ruleset AdditionalPickleModelInfo: + required: + - modelType + - pickleProtoVersion allOf: - $ref: '#/components/schemas/AdditionalModelInfo' - properties: diff --git a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml index 4b3c125..6e8ff84 100644 --- a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml +++ b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml @@ -973,14 +973,20 @@ components: - $ref: '#/components/schemas/AdditionalPMMLModelInfo' - $ref: '#/components/schemas/AdditionalPickleModelInfo' AdditionalPMMLModelInfo: + required: + - modelType + - modelSubType allOf: - $ref: '#/components/schemas/AdditionalModelInfo' - properties: modelSubType: type: string - description: model sub type + description: PMML MODEL-ELEMENT. Could be RuleSetModel, Scorecard... example: ruleset AdditionalPickleModelInfo: + required: + - modelType + - pickleProtoVersion allOf: - $ref: '#/components/schemas/AdditionalModelInfo' - properties: From 8e8688eb9d2eef9862837f2bd1b8df414e126ee3 Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 20 Dec 2022 11:45:10 +0100 Subject: [PATCH 35/42] [DBACLD-47417] Update spec for AdditionalOtherModelInfo --- open-prediction-service.yaml | 3 +++ .../ads-ml-service/app/api/api_v2/endpoints/models.py | 5 ++--- .../ads-ml-service/app/gen/tmp.schemas.ops.yaml | 3 +++ ops-implementations/ads-ml-service/app/schemas/impl.py | 10 +++++++--- .../ads-ml-service/app/tests/api/api_v2/test_models.py | 4 ++-- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/open-prediction-service.yaml b/open-prediction-service.yaml index 6e8ff84..edffbd8 100644 --- a/open-prediction-service.yaml +++ b/open-prediction-service.yaml @@ -994,6 +994,9 @@ components: type: string description: the version of pickle protocol example: 4 + AdditionalOtherModelInfo: + allOf: + - $ref: '#/components/schemas/AdditionalModelInfo' parameters: ModelIDParam: in: path diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index 65faea2..bd2566e 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -207,9 +207,8 @@ def get_model_metadata( return impl.AdditionalPMMLModelInfo( modelType='pmml', modelSubType=str(app_runtime_inspection.inspect_pmml_subtype(model.model_b64))) else: - raise fastapi.HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=f'Format {model.format.value} is not supported for metadata') + return impl.AdditionalOtherModelInfo( + modelType='other') @router.get( diff --git a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml index 6e8ff84..edffbd8 100644 --- a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml +++ b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml @@ -994,6 +994,9 @@ components: type: string description: the version of pickle protocol example: 4 + AdditionalOtherModelInfo: + allOf: + - $ref: '#/components/schemas/AdditionalModelInfo' parameters: ModelIDParam: in: path diff --git a/ops-implementations/ads-ml-service/app/schemas/impl.py b/ops-implementations/ads-ml-service/app/schemas/impl.py index 3e84f5d..e56db0d 100644 --- a/ops-implementations/ads-ml-service/app/schemas/impl.py +++ b/ops-implementations/ads-ml-service/app/schemas/impl.py @@ -163,13 +163,17 @@ class PredictionImpl(pydt.BaseModel): class AdditionalPMMLModelInfo(pydt.BaseModel): modelType: typing.Literal['pmml'] - modelSubType: typing.Optional[str] + modelSubType: str class AdditionalPickleModelInfo(pydt.BaseModel): modelType: typing.Literal['pickle'] - pickleProtoVersion: typing.Optional[str] + pickleProtoVersion: str + + +class AdditionalOtherModelInfo(pydt.BaseModel): + modelType: typing.Literal['other'] AdditionalModelInfo = typing_extensions.Annotated[ - typing.Union[AdditionalPMMLModelInfo, AdditionalPickleModelInfo], pydt.Field(discriminator='modelType')] + typing.Union[AdditionalPMMLModelInfo, AdditionalPickleModelInfo, AdditionalOtherModelInfo], pydt.Field(discriminator='modelType')] diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py index 5701869..8d39c51 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py @@ -246,8 +246,8 @@ def test_not_supported_metadata( resp = client.get(url=f'/models/{xgboost_endpoint.id}/metadata') # Assert - assert resp.status_code == 422 - assert resp.json()['detail'] == 'Format bst is not supported for metadata' + assert resp.ok + assert resp.json()['modelType'] == 'other' def test_pickle_metadata( From c587ac3f7f5002ef414f02231f9b12a47304821e Mon Sep 17 00:00:00 2001 From: ke li Date: Tue, 20 Dec 2022 11:48:30 +0100 Subject: [PATCH 36/42] [DBACLD-47417] Update spec for AdditionalOtherModelInfo --- open-prediction-service.yaml | 2 ++ ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/open-prediction-service.yaml b/open-prediction-service.yaml index edffbd8..63ffbb4 100644 --- a/open-prediction-service.yaml +++ b/open-prediction-service.yaml @@ -969,9 +969,11 @@ components: mapping: pmml: '#/components/schemas/AdditionalPMMLModelInfo' pickle: '#/components/schemas/AdditionalPickleModelInfo' + other: '#/components/schemas/AdditionalOtherModelInfo' oneOf: - $ref: '#/components/schemas/AdditionalPMMLModelInfo' - $ref: '#/components/schemas/AdditionalPickleModelInfo' + - $ref: '#/components/schemas/AdditionalOtherModelInfo' AdditionalPMMLModelInfo: required: - modelType diff --git a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml index edffbd8..63ffbb4 100644 --- a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml +++ b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml @@ -969,9 +969,11 @@ components: mapping: pmml: '#/components/schemas/AdditionalPMMLModelInfo' pickle: '#/components/schemas/AdditionalPickleModelInfo' + other: '#/components/schemas/AdditionalOtherModelInfo' oneOf: - $ref: '#/components/schemas/AdditionalPMMLModelInfo' - $ref: '#/components/schemas/AdditionalPickleModelInfo' + - $ref: '#/components/schemas/AdditionalOtherModelInfo' AdditionalPMMLModelInfo: required: - modelType From 27584d3da1be29b5547d1e92b6fb7944c30977bc Mon Sep 17 00:00:00 2001 From: DominiqueBreheret Date: Thu, 5 Jan 2023 12:59:42 +0100 Subject: [PATCH 37/42] Try to ignore some files for whitesource Mend. --- .whitesource | 10 ++++++++-- whitesource.config | 8 ++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 whitesource.config diff --git a/.whitesource b/.whitesource index 7230be1..6f3269f 100644 --- a/.whitesource +++ b/.whitesource @@ -11,5 +11,11 @@ "patch" ], "automerge": true - }] -} \ No newline at end of file + }], + "scanSettings": { + "baseBranches": ["develop"], + "configMode": "LOCAL", + "configExternalURL": "", + "projectToken": "" + } +} diff --git a/whitesource.config b/whitesource.config new file mode 100644 index 0000000..23ed1a5 --- /dev/null +++ b/whitesource.config @@ -0,0 +1,8 @@ +excludes=**/*requirements.txt +npm.resolveDependencies=true +npm.ignoreSourceFiles=true +npm.includeDevDependencies=false +npm.runPreStep=true +npm.ignoreNpmLsErrors=true +npm.ignoreScripts=false +npm.yarnProject=false From 8ce1a0011fe9bbece6fc9b38dc829e9624e926a5 Mon Sep 17 00:00:00 2001 From: Mathieu Marache Date: Thu, 5 Jan 2023 15:27:10 +0100 Subject: [PATCH 38/42] don't send logs to Mend --- whitesource.config | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/whitesource.config b/whitesource.config index 23ed1a5..bf46b36 100644 --- a/whitesource.config +++ b/whitesource.config @@ -1,8 +1 @@ -excludes=**/*requirements.txt -npm.resolveDependencies=true -npm.ignoreSourceFiles=true -npm.includeDevDependencies=false -npm.runPreStep=true -npm.ignoreNpmLsErrors=true -npm.ignoreScripts=false -npm.yarnProject=false +offline=True \ No newline at end of file From e89bc2e20f186866a78f1783a8d17fb6b4d1f624 Mon Sep 17 00:00:00 2001 From: ke li Date: Wed, 18 Jan 2023 14:41:52 +0100 Subject: [PATCH 39/42] [DBACLD-47417] Update spec --- open-prediction-service.yaml | 55 ++++++------------- .../app/api/api_v2/endpoints/models.py | 48 ++++++++++------ .../app/gen/tmp.schemas.ops.yaml | 55 ++++++------------- .../ads-ml-service/app/schemas/impl.py | 18 ------ .../app/tests/api/api_v2/test_models.py | 8 +-- ops-implementations/ads-ml-service/tox.ini | 2 +- 6 files changed, 73 insertions(+), 113 deletions(-) diff --git a/open-prediction-service.yaml b/open-prediction-service.yaml index 63ffbb4..715d35a 100644 --- a/open-prediction-service.yaml +++ b/open-prediction-service.yaml @@ -958,47 +958,28 @@ components: type: object title: Additional Model Information required: + - modelPackage - modelType properties: - modelType: + modelPackage: type: string - description: the type of imported binary + description: The file format of binary model example: pmml - discriminator: - propertyName: modelType - mapping: - pmml: '#/components/schemas/AdditionalPMMLModelInfo' - pickle: '#/components/schemas/AdditionalPickleModelInfo' - other: '#/components/schemas/AdditionalOtherModelInfo' - oneOf: - - $ref: '#/components/schemas/AdditionalPMMLModelInfo' - - $ref: '#/components/schemas/AdditionalPickleModelInfo' - - $ref: '#/components/schemas/AdditionalOtherModelInfo' - AdditionalPMMLModelInfo: - required: - - modelType - - modelSubType - allOf: - - $ref: '#/components/schemas/AdditionalModelInfo' - - properties: - modelSubType: - type: string - description: PMML MODEL-ELEMENT. Could be RuleSetModel, Scorecard... - example: ruleset - AdditionalPickleModelInfo: - required: - - modelType - - pickleProtoVersion - allOf: - - $ref: '#/components/schemas/AdditionalModelInfo' - - properties: - pickleProtoVersion: - type: string - description: the version of pickle protocol - example: 4 - AdditionalOtherModelInfo: - allOf: - - $ref: '#/components/schemas/AdditionalModelInfo' + enum: + - pmml + - joblib + - pickle + - other + modelType: + type: string + description: Model type of binary model + example: RuleSetModel + enum: + # pmml ruleset + - RuleSetModel + # pmml scorecard + - Scorecard + - other parameters: ModelIDParam: in: path diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index bd2566e..f2838f8 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -34,6 +34,7 @@ import app.schemas.binary_config as app_binary_config import app.schemas.impl as impl import app.core.configuration as app_conf +import app.models as models router = fastapi.APIRouter() LOGGER = logging.getLogger(__name__) @@ -188,27 +189,42 @@ async def add_binary( @router.get( path='/models/{model_id}/metadata', - response_model=impl.AdditionalModelInfo, + response_model=ops_schemas.AdditionalModelInfo, tags=['discover']) def get_model_metadata( model_id: int, db: saorm.Session = fastapi.Depends(deps.get_db)): LOGGER.info('Retrieving model metadata for id: %s', model_id) - model = crud.binary_ml_model.get(db=db, id=model_id) - - if model is None: - raise fastapi.HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail=f'Model with id {model_id} is not found') - - if model.format == app_binary_config.ModelWrapper.PICKLE: - return impl.AdditionalPickleModelInfo( - modelType='pickle', pickleProtoVersion=str(app_runtime_inspection.inspect_pickle_version(model.model_b64))) - elif model.format == app_binary_config.ModelWrapper.PMML: - return impl.AdditionalPMMLModelInfo( - modelType='pmml', modelSubType=str(app_runtime_inspection.inspect_pmml_subtype(model.model_b64))) - else: - return impl.AdditionalOtherModelInfo( - modelType='other') + match crud.binary_ml_model.get(db=db, id=model_id): + case None: + raise fastapi.HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=f'Model with id {model_id} is not found') + case models.BinaryMlModel(format=app_binary_config.ModelWrapper.PICKLE): + return ops_schemas.AdditionalModelInfo( + modelPackage=ops_schemas.ModelPackage.pickle, + modelType=ops_schemas.ModelType.other) + case models.BinaryMlModel(format=app_binary_config.ModelWrapper.JOBLIB): + return ops_schemas.AdditionalModelInfo( + modelPackage=ops_schemas.ModelPackage.joblib, + modelType=ops_schemas.ModelType.other) + case models.BinaryMlModel(format=app_binary_config.ModelWrapper.PMML, model_b64=binary): + match app_runtime_inspection.inspect_pmml_subtype(binary): + case 'RuleSetModel': + return ops_schemas.AdditionalModelInfo( + modelPackage=ops_schemas.ModelPackage.pmml, + modelType=ops_schemas.ModelType.RuleSetModel) + case 'Scorecard': + return ops_schemas.AdditionalModelInfo( + modelPackage=ops_schemas.ModelPackage.pmml, + modelType=ops_schemas.ModelType.Scorecard) + case _: + return ops_schemas.AdditionalModelInfo( + modelPackage=ops_schemas.ModelPackage.pmml, + modelType=ops_schemas.ModelType.other) + case _: + return ops_schemas.AdditionalModelInfo( + modelPackage=ops_schemas.ModelPackage.other, + modelType=ops_schemas.ModelType.other) @router.get( diff --git a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml index 63ffbb4..715d35a 100644 --- a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml +++ b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml @@ -958,47 +958,28 @@ components: type: object title: Additional Model Information required: + - modelPackage - modelType properties: - modelType: + modelPackage: type: string - description: the type of imported binary + description: The file format of binary model example: pmml - discriminator: - propertyName: modelType - mapping: - pmml: '#/components/schemas/AdditionalPMMLModelInfo' - pickle: '#/components/schemas/AdditionalPickleModelInfo' - other: '#/components/schemas/AdditionalOtherModelInfo' - oneOf: - - $ref: '#/components/schemas/AdditionalPMMLModelInfo' - - $ref: '#/components/schemas/AdditionalPickleModelInfo' - - $ref: '#/components/schemas/AdditionalOtherModelInfo' - AdditionalPMMLModelInfo: - required: - - modelType - - modelSubType - allOf: - - $ref: '#/components/schemas/AdditionalModelInfo' - - properties: - modelSubType: - type: string - description: PMML MODEL-ELEMENT. Could be RuleSetModel, Scorecard... - example: ruleset - AdditionalPickleModelInfo: - required: - - modelType - - pickleProtoVersion - allOf: - - $ref: '#/components/schemas/AdditionalModelInfo' - - properties: - pickleProtoVersion: - type: string - description: the version of pickle protocol - example: 4 - AdditionalOtherModelInfo: - allOf: - - $ref: '#/components/schemas/AdditionalModelInfo' + enum: + - pmml + - joblib + - pickle + - other + modelType: + type: string + description: Model type of binary model + example: RuleSetModel + enum: + # pmml ruleset + - RuleSetModel + # pmml scorecard + - Scorecard + - other parameters: ModelIDParam: in: path diff --git a/ops-implementations/ads-ml-service/app/schemas/impl.py b/ops-implementations/ads-ml-service/app/schemas/impl.py index e56db0d..04512c4 100644 --- a/ops-implementations/ads-ml-service/app/schemas/impl.py +++ b/ops-implementations/ads-ml-service/app/schemas/impl.py @@ -159,21 +159,3 @@ class PredictionImpl(pydt.BaseModel): parameters: typing.List[typing.Union[typing.List[ParameterImpl], ParameterImpl]] = pydt.Field( ..., description='Model parameters', title='Parameters' ) - - -class AdditionalPMMLModelInfo(pydt.BaseModel): - modelType: typing.Literal['pmml'] - modelSubType: str - - -class AdditionalPickleModelInfo(pydt.BaseModel): - modelType: typing.Literal['pickle'] - pickleProtoVersion: str - - -class AdditionalOtherModelInfo(pydt.BaseModel): - modelType: typing.Literal['other'] - - -AdditionalModelInfo = typing_extensions.Annotated[ - typing.Union[AdditionalPMMLModelInfo, AdditionalPickleModelInfo, AdditionalOtherModelInfo], pydt.Field(discriminator='modelType')] diff --git a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py index 8d39c51..2b565ac 100644 --- a/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py +++ b/ops-implementations/ads-ml-service/app/tests/api/api_v2/test_models.py @@ -263,8 +263,8 @@ def test_pickle_metadata( # Assert assert resp.ok - assert resp.json()['modelType'] == 'pickle' - assert resp.json()['pickleProtoVersion'] == '4' + assert resp.json()['modelPackage'] == 'pickle' + assert resp.json()['modelType'] == 'other' def test_pmml_metadata( @@ -280,8 +280,8 @@ def test_pmml_metadata( # Assert assert resp.ok - assert resp.json()['modelType'] == 'pmml' - assert resp.json()['modelSubType'] == 'Scorecard' + assert resp.json()['modelPackage'] == 'pmml' + assert resp.json()['modelType'] == 'Scorecard' def test_download_binary( diff --git a/ops-implementations/ads-ml-service/tox.ini b/ops-implementations/ads-ml-service/tox.ini index d76f3d9..79716e3 100755 --- a/ops-implementations/ads-ml-service/tox.ini +++ b/ops-implementations/ads-ml-service/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38 +envlist = py311 skipsdist = True [testenv] From 24d2de415440bd275fa24141468e1bee0a544e00 Mon Sep 17 00:00:00 2001 From: ke li Date: Wed, 18 Jan 2023 15:31:42 +0100 Subject: [PATCH 40/42] [DBACLD-47417] Update spec --- open-prediction-service.yaml | 11 -------- .../app/api/api_v2/endpoints/models.py | 28 +++++++------------ .../app/gen/tmp.schemas.ops.yaml | 11 -------- 3 files changed, 10 insertions(+), 40 deletions(-) diff --git a/open-prediction-service.yaml b/open-prediction-service.yaml index 715d35a..40273f2 100644 --- a/open-prediction-service.yaml +++ b/open-prediction-service.yaml @@ -965,21 +965,10 @@ components: type: string description: The file format of binary model example: pmml - enum: - - pmml - - joblib - - pickle - - other modelType: type: string description: Model type of binary model example: RuleSetModel - enum: - # pmml ruleset - - RuleSetModel - # pmml scorecard - - Scorecard - - other parameters: ModelIDParam: in: path diff --git a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py index f2838f8..7eebb10 100644 --- a/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py +++ b/ops-implementations/ads-ml-service/app/api/api_v2/endpoints/models.py @@ -199,32 +199,24 @@ def get_model_metadata( case None: raise fastapi.HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=f'Model with id {model_id} is not found') - case models.BinaryMlModel(format=app_binary_config.ModelWrapper.PICKLE): + case models.BinaryMlModel(format=app_binary_config.ModelWrapper.PICKLE | app_binary_config.ModelWrapper.JOBLIB as format): return ops_schemas.AdditionalModelInfo( - modelPackage=ops_schemas.ModelPackage.pickle, - modelType=ops_schemas.ModelType.other) - case models.BinaryMlModel(format=app_binary_config.ModelWrapper.JOBLIB): - return ops_schemas.AdditionalModelInfo( - modelPackage=ops_schemas.ModelPackage.joblib, - modelType=ops_schemas.ModelType.other) + modelPackage=format.value, + modelType='other') case models.BinaryMlModel(format=app_binary_config.ModelWrapper.PMML, model_b64=binary): match app_runtime_inspection.inspect_pmml_subtype(binary): - case 'RuleSetModel': - return ops_schemas.AdditionalModelInfo( - modelPackage=ops_schemas.ModelPackage.pmml, - modelType=ops_schemas.ModelType.RuleSetModel) - case 'Scorecard': + case 'Scorecard' | 'RuleSetModel' as typ: return ops_schemas.AdditionalModelInfo( - modelPackage=ops_schemas.ModelPackage.pmml, - modelType=ops_schemas.ModelType.Scorecard) + modelPackage='pmml', + modelType=typ) case _: return ops_schemas.AdditionalModelInfo( - modelPackage=ops_schemas.ModelPackage.pmml, - modelType=ops_schemas.ModelType.other) + modelPackage='pmml', + modelType='other') case _: return ops_schemas.AdditionalModelInfo( - modelPackage=ops_schemas.ModelPackage.other, - modelType=ops_schemas.ModelType.other) + modelPackage='other', + modelType='other') @router.get( diff --git a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml index 715d35a..40273f2 100644 --- a/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml +++ b/ops-implementations/ads-ml-service/app/gen/tmp.schemas.ops.yaml @@ -965,21 +965,10 @@ components: type: string description: The file format of binary model example: pmml - enum: - - pmml - - joblib - - pickle - - other modelType: type: string description: Model type of binary model example: RuleSetModel - enum: - # pmml ruleset - - RuleSetModel - # pmml scorecard - - Scorecard - - other parameters: ModelIDParam: in: path From 6de1e39e852eadd778a09695fb4be351bf911446 Mon Sep 17 00:00:00 2001 From: ke li Date: Thu, 19 Jan 2023 14:35:08 +0100 Subject: [PATCH 41/42] [DBACLD-47417] Refine tox.ini --- ops-implementations/ads-ml-service/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops-implementations/ads-ml-service/tox.ini b/ops-implementations/ads-ml-service/tox.ini index 79716e3..c0cc3c9 100755 --- a/ops-implementations/ads-ml-service/tox.ini +++ b/ops-implementations/ads-ml-service/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py311 +envlist = py310, py311 skipsdist = True [testenv] From 087f8dd5f6e232acd6daa0bebf47b0f0312a9bd7 Mon Sep 17 00:00:00 2001 From: ke li Date: Thu, 19 Jan 2023 14:35:34 +0100 Subject: [PATCH 42/42] [DBACLD-47417] Remove unused inspect_pickle_version --- .../ads-ml-service/app/runtime/inspection.py | 4 ---- .../ads-ml-service/app/schemas/impl.py | 1 - .../runtime/test_signature_inspection.py | 23 ------------------- 3 files changed, 28 deletions(-) diff --git a/ops-implementations/ads-ml-service/app/runtime/inspection.py b/ops-implementations/ads-ml-service/app/runtime/inspection.py index 59a8343..6f63524 100644 --- a/ops-implementations/ads-ml-service/app/runtime/inspection.py +++ b/ops-implementations/ads-ml-service/app/runtime/inspection.py @@ -71,7 +71,3 @@ def inspect_pmml_model_name(model_file: bytes) -> typing.Optional[str]: def inspect_pmml_subtype(model_file: bytes) -> typing.Optional[str]: wrapper = load_pmml_model(model_file).loaded_model return wrapper.model.modelElement - - -def inspect_pickle_version(model_file: bytes) -> int: - return max(op[0].proto for op in pickletools.genops(model_file)) diff --git a/ops-implementations/ads-ml-service/app/schemas/impl.py b/ops-implementations/ads-ml-service/app/schemas/impl.py index 04512c4..7faddd9 100644 --- a/ops-implementations/ads-ml-service/app/schemas/impl.py +++ b/ops-implementations/ads-ml-service/app/schemas/impl.py @@ -17,7 +17,6 @@ import enum import typing -import typing_extensions import datetime as dt import numpy diff --git a/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py b/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py index 1ae4dfd..135dee0 100644 --- a/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py +++ b/ops-implementations/ads-ml-service/app/tests/runtime/test_signature_inspection.py @@ -55,29 +55,6 @@ def test_pmml_output_schema_inspection( } -def test_inspect_pickle_version(): - # Arrange - p0 = pickle.dumps('test_0', protocol=0) - p1 = pickle.dumps('test_1', protocol=1) - p2 = pickle.dumps('test_2', protocol=2) - p3 = pickle.dumps(str.encode('test_3'), protocol=3) - p4 = pickle.dumps('test_4', protocol=4) - - # When - ver_0 = app_signature_inspection.inspect_pickle_version(p0) - ver_1 = app_signature_inspection.inspect_pickle_version(p1) - ver_2 = app_signature_inspection.inspect_pickle_version(p2) - ver_3 = app_signature_inspection.inspect_pickle_version(p3) - ver_4 = app_signature_inspection.inspect_pickle_version(p4) - - # Assert - assert ver_0 == 0 - assert ver_1 == 1 - assert ver_2 == 2 - assert ver_3 == 3 - assert ver_4 == 4 - - def test_inspect_pmml_subtype(): # When sub_type_regression = app_signature_inspection.inspect_pmml_subtype(