From 72ff50e6065ecd5054fb2d09c0c5d201ce46b8a7 Mon Sep 17 00:00:00 2001 From: abhijeet-dhumal Date: Mon, 24 Jun 2024 16:34:42 +0530 Subject: [PATCH] Add raycluster-sdk oauth test case for disconnected cluster setup --- .pre-commit-config.yaml | 1 + poetry.lock | 120 ++++++++++++- pyproject.toml | 1 + tests/e2e/minio_deployment.yaml | 163 +++++++++++++++++ tests/e2e/mnist.py | 43 +++-- tests/e2e/mnist_pip_requirements.txt | 2 + .../e2e/mnist_raycluster_sdk_aw_kind_test.py | 1 + tests/e2e/mnist_raycluster_sdk_oauth_test.py | 12 ++ tests/e2e/support.py | 164 +++++++++++++++++- 9 files changed, 488 insertions(+), 19 deletions(-) create mode 100644 tests/e2e/minio_deployment.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 89e037cd6..7928084dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,6 +7,7 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml + args: [--allow-multiple-documents] - id: check-added-large-files - repo: https://github.com/psf/black rev: 23.3.0 diff --git a/poetry.lock b/poetry.lock index 1868163ed..0a45575f9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "aiohttp" @@ -124,6 +124,63 @@ files = [ [package.dependencies] frozenlist = ">=1.1.0" +[[package]] +name = "argon2-cffi" +version = "23.1.0" +description = "Argon2 for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "argon2_cffi-23.1.0-py3-none-any.whl", hash = "sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea"}, + {file = "argon2_cffi-23.1.0.tar.gz", hash = "sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08"}, +] + +[package.dependencies] +argon2-cffi-bindings = "*" + +[package.extras] +dev = ["argon2-cffi[tests,typing]", "tox (>4)"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-copybutton", "sphinx-notfound-page"] +tests = ["hypothesis", "pytest"] +typing = ["mypy"] + +[[package]] +name = "argon2-cffi-bindings" +version = "21.2.0" +description = "Low-level CFFI bindings for Argon2" +optional = false +python-versions = ">=3.6" +files = [ + {file = "argon2-cffi-bindings-21.2.0.tar.gz", hash = "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ccb949252cb2ab3a08c02024acb77cfb179492d5701c7cbdbfd776124d4d2367"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9524464572e12979364b7d600abf96181d3541da11e23ddf565a32e70bd4dc0d"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b746dba803a79238e925d9046a63aa26bf86ab2a2fe74ce6b009a1c3f5c8f2ae"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58ed19212051f49a523abb1dbe954337dc82d947fb6e5a0da60f7c8471a8476c"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:bd46088725ef7f58b5a1ef7ca06647ebaf0eb4baff7d1d0d177c6cc8744abd86"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_i686.whl", hash = "sha256:8cd69c07dd875537a824deec19f978e0f2078fdda07fd5c42ac29668dda5f40f"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f1152ac548bd5b8bcecfb0b0371f082037e47128653df2e8ba6e914d384f3c3e"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-win32.whl", hash = "sha256:603ca0aba86b1349b147cab91ae970c63118a0f30444d4bc80355937c950c082"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:b2ef1c30440dbbcba7a5dc3e319408b59676e2e039e2ae11a8775ecf482b192f"}, + {file = "argon2_cffi_bindings-21.2.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e415e3f62c8d124ee16018e491a009937f8cf7ebf5eb430ffc5de21b900dad93"}, + {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3e385d1c39c520c08b53d63300c3ecc28622f076f4c2b0e6d7e796e9f6502194"}, + {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c3e3cc67fdb7d82c4718f19b4e7a87123caf8a93fde7e23cf66ac0337d3cb3f"}, + {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a22ad9800121b71099d0fb0a65323810a15f2e292f2ba450810a7316e128ee5"}, + {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f9f8b450ed0547e3d473fdc8612083fd08dd2120d6ac8f73828df9b7d45bb351"}, + {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:93f9bf70084f97245ba10ee36575f0c3f1e7d7724d67d8e5b08e61787c320ed7"}, + {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3b9ef65804859d335dc6b31582cad2c5166f0c3e7975f324d9ffaa34ee7e6583"}, + {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4966ef5848d820776f5f562a7d45fdd70c2f330c961d0d745b784034bd9f48d"}, + {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20ef543a89dee4db46a1a6e206cd015360e5a75822f76df533845c3cbaf72670"}, + {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed2937d286e2ad0cc79a7087d3c272832865f779430e0cc2b4f3718d3159b0cb"}, + {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5e00316dabdaea0b2dd82d141cc66889ced0cdcbfa599e8b471cf22c620c329a"}, +] + +[package.dependencies] +cffi = ">=1.0.1" + +[package.extras] +dev = ["cogapp", "pre-commit", "pytest", "wheel"] +tests = ["pytest"] + [[package]] name = "asttokens" version = "2.4.1" @@ -1273,6 +1330,24 @@ docs = ["IPython", "bump2version", "furo", "sphinx", "sphinx-argparse", "towncri lint = ["black", "check-manifest", "flake8", "isort", "mypy"] test = ["Cython", "greenlet", "ipython", "pytest", "pytest-cov", "setuptools"] +[[package]] +name = "minio" +version = "7.2.7" +description = "MinIO Python SDK for Amazon S3 Compatible Cloud Storage" +optional = false +python-versions = "*" +files = [ + {file = "minio-7.2.7-py3-none-any.whl", hash = "sha256:59d1f255d852fe7104018db75b3bebbd987e538690e680f7c5de835e422de837"}, + {file = "minio-7.2.7.tar.gz", hash = "sha256:473d5d53d79f340f3cd632054d0c82d2f93177ce1af2eac34a235bea55708d98"}, +] + +[package.dependencies] +argon2-cffi = "*" +certifi = "*" +pycryptodome = "*" +typing-extensions = "*" +urllib3 = "*" + [[package]] name = "msgpack" version = "1.0.8" @@ -1903,6 +1978,47 @@ files = [ {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, ] +[[package]] +name = "pycryptodome" +version = "3.20.0" +description = "Cryptographic library for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pycryptodome-3.20.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:f0e6d631bae3f231d3634f91ae4da7a960f7ff87f2865b2d2b831af1dfb04e9a"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:baee115a9ba6c5d2709a1e88ffe62b73ecc044852a925dcb67713a288c4ec70f"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:417a276aaa9cb3be91f9014e9d18d10e840a7a9b9a9be64a42f553c5b50b4d1d"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a1250b7ea809f752b68e3e6f3fd946b5939a52eaeea18c73bdab53e9ba3c2dd"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:d5954acfe9e00bc83ed9f5cb082ed22c592fbbef86dc48b907238be64ead5c33"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-win32.whl", hash = "sha256:06d6de87c19f967f03b4cf9b34e538ef46e99a337e9a61a77dbe44b2cbcf0690"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-win_amd64.whl", hash = "sha256:ec0bb1188c1d13426039af8ffcb4dbe3aad1d7680c35a62d8eaf2a529b5d3d4f"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:5601c934c498cd267640b57569e73793cb9a83506f7c73a8ec57a516f5b0b091"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d29daa681517f4bc318cd8a23af87e1f2a7bad2fe361e8aa29c77d652a065de4"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3427d9e5310af6680678f4cce149f54e0bb4af60101c7f2c16fdf878b39ccccc"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:3cd3ef3aee1079ae44afaeee13393cf68b1058f70576b11439483e34f93cf818"}, + {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044"}, + {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a"}, + {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2"}, + {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c"}, + {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25"}, + {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128"}, + {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c"}, + {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4"}, + {file = "pycryptodome-3.20.0-cp35-abi3-win32.whl", hash = "sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72"}, + {file = "pycryptodome-3.20.0-cp35-abi3-win_amd64.whl", hash = "sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9"}, + {file = "pycryptodome-3.20.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:4401564ebf37dfde45d096974c7a159b52eeabd9969135f0426907db367a652a"}, + {file = "pycryptodome-3.20.0-pp27-pypy_73-win32.whl", hash = "sha256:ec1f93feb3bb93380ab0ebf8b859e8e5678c0f010d2d78367cf6bc30bfeb148e"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:acae12b9ede49f38eb0ef76fdec2df2e94aad85ae46ec85be3648a57f0a7db04"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f47888542a0633baff535a04726948e876bf1ed880fddb7c10a736fa99146ab3"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e0e4a987d38cfc2e71b4a1b591bae4891eeabe5fa0f56154f576e26287bfdea"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c18b381553638414b38705f07d1ef0a7cf301bc78a5f9bc17a957eb19446834b"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a60fedd2b37b4cb11ccb5d0399efe26db9e0dd149016c1cc6c8161974ceac2d6"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:405002eafad114a2f9a930f5db65feef7b53c4784495dd8758069b89baf68eab"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ab6ab0cb755154ad14e507d1df72de9897e99fd2d4922851a276ccc14f4f1a5"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:acf6e43fa75aca2d33e93409f2dafe386fe051818ee79ee8a3e21de9caa2ac9e"}, + {file = "pycryptodome-3.20.0.tar.gz", hash = "sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7"}, +] + [[package]] name = "pydantic" version = "1.10.17" @@ -2795,4 +2911,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "d656bab99c2e5a911ee1003db9e0682141328ae3ef1e1620945f8479451425bf" +content-hash = "1f7cc3cac37eb62e541aac45312dd15411c6d34756ca9279146d713314b9b79d" diff --git a/pyproject.toml b/pyproject.toml index af7dd1ca0..8a6b00e2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ cryptography = "40.0.2" executing = "1.2.0" pydantic = "< 2" ipywidgets = "8.1.2" +minio = "^7.2.7" [tool.poetry.group.docs] optional = true diff --git a/tests/e2e/minio_deployment.yaml b/tests/e2e/minio_deployment.yaml new file mode 100644 index 000000000..86d4ef01f --- /dev/null +++ b/tests/e2e/minio_deployment.yaml @@ -0,0 +1,163 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: minio-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + volumeMode: Filesystem +--- +kind: Secret +apiVersion: v1 +metadata: + name: minio-secret +stringData: + # change the username and password to your own values. + # ensure that the user is at least 3 characters long and the password at least 8 + minio_root_user: minio + minio_root_password: minio123 +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: minio +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + creationTimestamp: null + labels: + app: minio + spec: + volumes: + - name: data + persistentVolumeClaim: + claimName: minio-pvc + containers: + - resources: + limits: + cpu: 250m + memory: 1Gi + requests: + cpu: 20m + memory: 100Mi + readinessProbe: + tcpSocket: + port: 9000 + initialDelaySeconds: 5 + timeoutSeconds: 1 + periodSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + terminationMessagePath: /dev/termination-log + name: minio + livenessProbe: + tcpSocket: + port: 9000 + initialDelaySeconds: 30 + timeoutSeconds: 1 + periodSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: minio-secret + key: minio_root_user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio-secret + key: minio_root_password + ports: + - containerPort: 9000 + protocol: TCP + - containerPort: 9090 + protocol: TCP + imagePullPolicy: IfNotPresent + volumeMounts: + - name: data + mountPath: /data + subPath: minio + terminationMessagePolicy: File + image: >- + quay.io/minio/minio:RELEASE.2024-06-22T05-26-45Z + # In case of disconnected environment, use image digest instead of tag + # For example : /minio/minio@sha256:6b3abf2f59286b985bfde2b23e37230b466081eda5dccbf971524d54c8e406b5 + args: + - server + - /data + - --console-address + - :9090 + restartPolicy: Always + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirst + securityContext: {} + schedulerName: default-scheduler + strategy: + type: Recreate + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 +--- +kind: Service +apiVersion: v1 +metadata: + name: minio-service +spec: + ipFamilies: + - IPv4 + ports: + - name: api + protocol: TCP + port: 9000 + targetPort: 9000 + - name: ui + protocol: TCP + port: 9090 + targetPort: 9090 + internalTrafficPolicy: Cluster + type: ClusterIP + ipFamilyPolicy: SingleStack + sessionAffinity: None + selector: + app: minio +--- +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: minio-api +spec: + to: + kind: Service + name: minio-service + weight: 100 + port: + targetPort: api + wildcardPolicy: None + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect +--- +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: minio-ui +spec: + to: + kind: Service + name: minio-service + weight: 100 + port: + targetPort: ui + wildcardPolicy: None + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect diff --git a/tests/e2e/mnist.py b/tests/e2e/mnist.py index 2971d9c98..2e2eb880b 100644 --- a/tests/e2e/mnist.py +++ b/tests/e2e/mnist.py @@ -19,15 +19,18 @@ from pytorch_lightning.callbacks.progress import TQDMProgressBar from torch import nn from torch.nn import functional as F -from torch.utils.data import DataLoader, random_split, RandomSampler +from torch.utils.data import DataLoader, random_split from torchmetrics import Accuracy from torchvision import transforms from torchvision.datasets import MNIST +from support import download_all_from_storage_bucket PATH_DATASETS = os.environ.get("PATH_DATASETS", ".") BATCH_SIZE = 256 if torch.cuda.is_available() else 64 # %% +local_mnist_path = os.path.dirname(os.path.abspath(__file__)) + print("prior to running the trainer") print("MASTER_ADDR: is ", os.getenv("MASTER_ADDR")) print("MASTER_PORT: is ", os.getenv("MASTER_PORT")) @@ -110,28 +113,44 @@ def configure_optimizers(self): def prepare_data(self): # download - print("Downloading MNIST dataset...") - MNIST(self.data_dir, train=True, download=True) - MNIST(self.data_dir, train=False, download=True) + print("Preparing MNIST dataset...") + + if "MINIO_ENDPOINT" in os.environ: + dataset_dir = os.path.join(self.data_dir, "MNIST/raw") + endpoint = os.environ.get("MINIO_ENDPOINT") + access_key = os.environ.get("MINIO_ACCESS_KEY") + secret_key = os.environ.get("MINIO_SECRET_KEY") + bucket_name = os.environ.get("MINIO_STORAGE_BUCKET") + download_all_from_storage_bucket( + endpoint, + access_key=access_key, + secret_key=secret_key, + bucket_name=bucket_name, + dir_path=dataset_dir, + ) + download_datasets = False + else: + download_datasets = True + + MNIST(self.data_dir, train=True, download=download_datasets) + MNIST(self.data_dir, train=False, download=download_datasets) def setup(self, stage=None): # Assign train/val datasets for use in dataloaders if stage == "fit" or stage is None: - mnist_full = MNIST(self.data_dir, train=True, transform=self.transform) + mnist_full = MNIST( + self.data_dir, train=True, transform=self.transform, download=False + ) self.mnist_train, self.mnist_val = random_split(mnist_full, [55000, 5000]) # Assign test dataset for use in dataloader(s) if stage == "test" or stage is None: self.mnist_test = MNIST( - self.data_dir, train=False, transform=self.transform + self.data_dir, train=False, transform=self.transform, download=False ) def train_dataloader(self): - return DataLoader( - self.mnist_train, - batch_size=BATCH_SIZE, - sampler=RandomSampler(self.mnist_train, num_samples=1000), - ) + return DataLoader(self.mnist_train, batch_size=BATCH_SIZE) def val_dataloader(self): return DataLoader(self.mnist_val, batch_size=BATCH_SIZE) @@ -142,7 +161,7 @@ def test_dataloader(self): # Init DataLoader from MNIST Dataset -model = LitMNIST() +model = LitMNIST(data_dir=local_mnist_path) print("GROUP: ", int(os.environ.get("GROUP_WORLD_SIZE", 1))) print("LOCAL: ", int(os.environ.get("LOCAL_WORLD_SIZE", 1))) diff --git a/tests/e2e/mnist_pip_requirements.txt b/tests/e2e/mnist_pip_requirements.txt index 87edeef27..bf8726b93 100644 --- a/tests/e2e/mnist_pip_requirements.txt +++ b/tests/e2e/mnist_pip_requirements.txt @@ -1,3 +1,5 @@ pytorch_lightning==1.5.10 torchmetrics==0.9.1 torchvision==0.12.0 +kubernetes +minio diff --git a/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py b/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py index 2aa5da16d..4d97ff245 100644 --- a/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py +++ b/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py @@ -19,6 +19,7 @@ def setup_method(self): def teardown_method(self): delete_namespace(self) + delete_kueue_resources(self) def test_mnist_ray_cluster_sdk_kind(self): self.setup_method() diff --git a/tests/e2e/mnist_raycluster_sdk_oauth_test.py b/tests/e2e/mnist_raycluster_sdk_oauth_test.py index e489c39f8..0c33b04f6 100644 --- a/tests/e2e/mnist_raycluster_sdk_oauth_test.py +++ b/tests/e2e/mnist_raycluster_sdk_oauth_test.py @@ -17,10 +17,18 @@ class TestRayClusterSDKOauth: def setup_method(self): initialize_kubernetes_client(self) + # to deploy minio storage instance + # yaml_file_path= os.path.join(dir_path, "minio_deployment.yaml") + # yaml_deployment(yaml_file_path,"apply") + def teardown_method(self): delete_namespace(self) delete_kueue_resources(self) + # to delete deployment created using minio yaml file + # yaml_file_path= os.path.join(dir_path, "minio_deployment.yaml") + # yaml_deployment(yaml_file_path,"delete") + def test_mnist_ray_cluster_sdk_auth(self): self.setup_method() create_namespace(self) @@ -72,11 +80,13 @@ def run_mnist_raycluster_sdk_oauth(self): def assert_jobsubmit_withoutLogin(self, cluster): dashboard_url = cluster.cluster_dashboard_uri() + jobdata = { "entrypoint": "python mnist.py", "runtime_env": { "working_dir": "./tests/e2e/", "pip": "./tests/e2e/mnist_pip_requirements.txt", + "env_vars": get_disconnected_setup_env_variables(), }, } try: @@ -104,8 +114,10 @@ def assert_jobsubmit_withlogin(self, cluster): runtime_env={ "working_dir": "./tests/e2e/", "pip": "./tests/e2e/mnist_pip_requirements.txt", + "env_vars": get_disconnected_setup_env_variables(), }, ) + print(f"Submitted job with ID: {submission_id}") done = False time = 0 diff --git a/tests/e2e/support.py b/tests/e2e/support.py index 04c9cb421..2f1aeabcf 100644 --- a/tests/e2e/support.py +++ b/tests/e2e/support.py @@ -1,10 +1,13 @@ import os import random -import string +import string, yaml import subprocess from kubernetes import client, config -import kubernetes.client -from codeflare_sdk.utils.kube_api_helpers import _kube_api_error_handling +from kubernetes.client.exceptions import ApiException +from minio import Minio + + +dir_path = os.path.dirname(os.path.realpath(__file__)) def get_ray_image(): @@ -12,6 +15,22 @@ def get_ray_image(): return os.getenv("RAY_IMAGE", default_ray_image) +def get_disconnected_setup_env_variables(): + env_vars = dict() + # Use specified pip index url instead of default(https://pypi.org/simple) if related environment variables exists + if "PIP_INDEX_URL" in os.environ: + env_vars["PIP_INDEX_URL"] = os.environ.get("PIP_INDEX_URL") + env_vars["PIP_TRUSTED_HOST"] = os.environ.get("PIP_TRUSTED_HOST") + + # Use specified storage bucket to download datasets from + if "MINIO_ENDPOINT" in os.environ: + env_vars["MINIO_ENDPOINT"] = os.environ.get("MINIO_ENDPOINT") + env_vars["MINIO_ACCESS_KEY"] = os.environ.get("MINIO_ACCESS_KEY") + env_vars["MINIO_SECRET_KEY"] = os.environ.get("MINIO_SECRET_KEY") + env_vars["MINIO_STORAGE_BUCKET"] = os.environ.get("MINIO_STORAGE_BUCKET") + return env_vars + + def random_choice(): alphabet = string.ascii_lowercase + string.digits return "".join(random.choices(alphabet, k=5)) @@ -48,7 +67,7 @@ def create_namespace_with_name(self, namespace_name): ) self.api_instance.create_namespace(namespace_body) except Exception as e: - return _kube_api_error_handling(e) + print(f"Error creating namespace with name : {e}") def delete_namespace(self): @@ -60,7 +79,7 @@ def initialize_kubernetes_client(self): config.load_kube_config() # Initialize Kubernetes client self.api_instance = client.CoreV1Api() - self.custom_api = kubernetes.client.CustomObjectsApi(self.api_instance.api_client) + self.custom_api = client.CustomObjectsApi(self.api_instance.api_client) def run_oc_command(args): @@ -217,3 +236,138 @@ def delete_kueue_resources(self): print(f"'{self.resource_flavor}' resource-flavor deleted") except Exception as e: print(f"\nError deleting resource-flavor '{self.resource_flavor}' : {e}") + + +def download_all_from_storage_bucket( + endpoint, access_key, secret_key, bucket_name, dir_path=dir_path +): + client = Minio( + endpoint, access_key=access_key, secret_key=secret_key, cert_check=False + ) + + # for bucket in client.list_buckets(): + if not os.path.exists(dir_path): + os.makedirs(dir_path) + else: + print(f"Directory '{dir_path}' already exists") + # download all files from storage bucket recursively + for item in client.list_objects(bucket_name, recursive=True): + dataset_file_path = os.path.join(dir_path, item.object_name) + if not os.path.exists(dataset_file_path): + client.fget_object(bucket_name, item.object_name, dataset_file_path) + else: + print(f"File-path '{dataset_file_path}' already exists") + + +# For example : Can be used for minio storage instance deployment using minio-deployment.yaml file +def apply_yaml_deployment(yaml_file_path): + # Load the OpenShift/Kubernetes configuration + config.load_kube_config() + + with open(yaml_file_path, "r") as file: + yaml_data = yaml.safe_load_all(file) + + k8s_client = client.ApiClient() + for data in yaml_data: + try: + kind = data["kind"] + namespace = data["metadata"].get("namespace", "default") + + # Apply the resource based on its kind + if kind == "Deployment": + apps_v1_api = client.AppsV1Api(k8s_client) + apps_v1_api.create_namespaced_deployment( + namespace=namespace, body=data + ) + elif kind == "Service": + core_v1_api = client.CoreV1Api(k8s_client) + core_v1_api.create_namespaced_service( + namespace=namespace, body=data + ) + elif kind == "PersistentVolumeClaim": + core_v1_api = client.CoreV1Api(k8s_client) + core_v1_api.create_namespaced_persistent_volume_claim( + namespace=namespace, body=data + ) + elif kind == "Secret": + core_v1_api = client.CoreV1Api(k8s_client) + core_v1_api.create_namespaced_secret(namespace=namespace, body=data) + elif kind == "ConfigMap": + core_v1_api = client.CoreV1Api(k8s_client) + core_v1_api.create_namespaced_config_map( + namespace=namespace, body=data + ) + elif kind == "Route": + # OpenShift-specific Route creation + route_v1_api = client.CustomObjectsApi(k8s_client) + group = "route.openshift.io" + version = "v1" + plural = "routes" + route_v1_api.create_namespaced_custom_object( + group=group, + version=version, + namespace=namespace, + plural=plural, + body=data, + ) + else: + print(f"Unsupported resource kind: {kind}") + except ApiException as e: + print(f"Exception when creating resource {kind}: {e}\n") + + +def delete_yaml_deployment(yaml_file_path): + # Load the OpenShift/Kubernetes configuration + config.load_kube_config() + + with open(yaml_file_path, "r") as file: + yaml_data = yaml.safe_load_all(file) + + k8s_client = client.ApiClient() + for data in yaml_data: + try: + kind = data["kind"] + name = data["metadata"]["name"] + namespace = data["metadata"].get("namespace", "default") + + # Delete the resource based on its kind + if kind == "Deployment": + apps_v1_api = client.AppsV1Api(k8s_client) + apps_v1_api.delete_namespaced_deployment( + name=name, namespace=namespace + ) + elif kind == "Service": + core_v1_api = client.CoreV1Api(k8s_client) + core_v1_api.delete_namespaced_service( + name=name, namespace=namespace + ) + elif kind == "PersistentVolumeClaim": + core_v1_api = client.CoreV1Api(k8s_client) + core_v1_api.delete_namespaced_persistent_volume_claim( + name=name, namespace=namespace + ) + elif kind == "Secret": + core_v1_api = client.CoreV1Api(k8s_client) + core_v1_api.delete_namespaced_secret(name=name, namespace=namespace) + elif kind == "ConfigMap": + core_v1_api = client.CoreV1Api(k8s_client) + core_v1_api.delete_namespaced_config_map( + name=name, namespace=namespace + ) + elif kind == "Route": + # OpenShift-specific Route deletion + route_v1_api = client.CustomObjectsApi(k8s_client) + group = "route.openshift.io" + version = "v1" + plural = "routes" + route_v1_api.delete_namespaced_custom_object( + group=group, + version=version, + namespace=namespace, + plural=plural, + name=name, + ) + else: + print(f"Unsupported resource kind: {kind}") + except ApiException as e: + print(f"Exception when deleting resource {kind}: {e}\n")