Skip to content

Commit

Permalink
Merge pull request #12 from IBM/make-run-samples
Browse files Browse the repository at this point in the history
Make run samples
  • Loading branch information
daw3rd authored May 1, 2024
2 parents 3ff8539 + 5524ef1 commit 8a1749c
Show file tree
Hide file tree
Showing 23 changed files with 556 additions and 484 deletions.
36 changes: 18 additions & 18 deletions .github/workflows/deploy-docs.yml
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
name: deploy docs
name: deploy docs

on:
workflow_dispatch:
push:
branches:
- dev
workflow_dispatch:
push:
branches:
- dev
jobs:
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: 3.x
- run: pip install mkdocs-material mkdocstrings[python] mkdocs-badges mkdocs-same-dir
- run: |
# remove badges
cat README.md |sed '/img\.shields\.io/d' > README_.md
mv README_.md README.md
mkdocs gh-deploy --force
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: 3.x
- run: pip install mkdocs-material mkdocstrings[python] mkdocs-badges mkdocs-same-dir
- run: |
# remove badges
cat README.md |sed '/img\.shields\.io/d' > README_.md
mv README_.md README.md
mkdocs gh-deploy --force
141 changes: 70 additions & 71 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -1,74 +1,73 @@

name: Test CI

on:
workflow_dispatch:
push:
branches:
- "dev"
pull_request:
branches:
- "dev"
on:
workflow_dispatch:
push:
branches:
- "dev"
pull_request:
branches:
- "dev"
jobs:
test-lib:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Test data-processing-lib
run: |
make -C data-processing-lib DOCKER=docker venv test
test-code:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Test Code Transforms
run: |
make -C transforms/code DOCKER=docker venv test-src
test-universal:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Test Universal Transforms
run: |
make -C transforms/universal DOCKER=docker venv test-src
test-tools:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Test tools
run: |
make -C tools DOCKER=docker venv test
test-kfp-lib:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Test KFP lib
run: |
source kind/requirements.env
export PATH=$PATH:/tmp/
curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
chmod 700 /tmp/get_helm.sh
HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
chmod 777 /tmp/helm
chmod 777 /tmp/kind
curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
chmod 777 /tmp/kubectl
curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
chmod +x /tmp/mc
export DEPLOY_KUBEFLOW=0
make -C kind setup
make -C kfp/kfp_support_lib build test
test-kfp-compile:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Test KFP compile
run: |
make -C kfp/transform_workflows venv build
test-lib:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Test data-processing-lib
run: |
make -C data-processing-lib DOCKER=docker venv test
test-code:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Test Code Transforms
run: |
make -C transforms/code DOCKER=docker venv test-src
test-universal:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Test Universal Transforms
run: |
make -C transforms/universal DOCKER=docker venv test-src
test-tools:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Test tools
run: |
make -C tools DOCKER=docker venv test
test-kfp-lib:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Test KFP lib
run: |
source kind/requirements.env
export PATH=$PATH:/tmp/
curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
chmod 700 /tmp/get_helm.sh
HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
chmod 777 /tmp/helm
chmod 777 /tmp/kind
curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
chmod 777 /tmp/kubectl
curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
chmod +x /tmp/mc
export DEPLOY_KUBEFLOW=0
make -C kind setup
make -C kfp/kfp_support_lib build test
test-kfp-compile:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Test KFP compile
run: |
make -C kfp/transform_workflows venv build
79 changes: 79 additions & 0 deletions .make.defaults
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,21 @@ __check_defined = \
@# Help: Test an $(DOCKER_IMAGE) use test source inside the image.
$(DOCKER) run -t --rm $(DOCKER_IMAGE) pytest -s test

.PHONY: .defaults.test-locals
.defaults.test-locals::
@# Help: Run the *local*.py files in the src directory
@source venv/bin/activate; \
export PYTHONPATH=src; \
files=$$(find src -name '*local*.py' ); \
for i in $$files; do \
echo Executing: python $$i; \
python $$i; \
if [ $$? -ne 0 ]; then \
exit 1; \
fi; \
done


# This expects the image to already be built and so does not depending on .defaults.publish.
# This allows others to define their own image building prior to publishing.
.PHONY: .defaults.publish
Expand All @@ -219,3 +234,67 @@ __check_defined = \
pip install wheel; \
pip install -r requirements.txt; \

.PHONY: .defaults.check.installed
.defaults.check.installed::
@installed=$$(which $(CHECK_RUNNABLE)); \
if [ -z "$$installed" ]; then \
echo $(CHECK_RUNNABLE) must be installed; \
exit 1; \
fi

.PHONY: .defaults.minio.check
.defaults.minio.check::
@# Help: Check that minio and the mc CLI is installed.
$(MAKE) CHECK_RUNNABLE=minio .defaults.check.installed
$(MAKE) CHECK_RUNNABLE=mc .defaults.check.installed

MINIO_DIR=/tmp/data-prep-lab
MINIO_ALIAS=local
# These are the credentials used by samples.
MINIO_ADMIN_USER=localminioaccesskey
MINIO_ADMIN_PWD= localminiosecretkey

.PHONY: .defaults.minio.verify-running
.defaults.minio.verify-running:
@z=$$(ps -elf | grep minio | grep -v grep); \
if [ -z "$$z" ]; then \
echo ""; \
echo "Please start minio (see make help)"; \
echo ""; \
exit 1; \
fi

.PHONY: .defaults.minio.stop
.defaults.minio.stop::
$(MAKE) .defaults.minio.check
@# Help: Stop the minio server used for S3-based samples
-mc admin service stop $(MINIO_ALIAS) > /dev/null 2>&1
-mc alias rm $(MINIO_ALIAS) > /dev/null 2>&1

.PHONY: .defaults.minio.start
.defaults.minio.start::
$(MAKE) .defaults.minio.stop
@# Help: Stop, then start, the minio server used for S3-based samples
rm -rf $(MINIO_DIR)
mkdir $(MINIO_DIR)
echo Starting minio server
minio server $(MINIO_DIR) &
sleep 1
-mc alias rm $(MINIO_ALIAS) > /dev/null 2>&1
mc alias set $(MINIO_ALIAS) http://127.0.0.1:9000 minioadmin minioadmin
mc admin user svcacct add --access-key "$(MINIO_ADMIN_USER)" --secret-key "$(MINIO_ADMIN_PWD)" $(MINIO_ALIAS) minioadmin
mc alias ls $(MINIO_ALIAS)
@echo "Test minio server started"

.PHONY: .defaults.minio.load-test-data
.defaults.minio.load-test-data::
@# Help: Load the minio server with S3 sample data from $(MINIO_SRC) to $(MINIO_DEST)
@if [ -z "$(MINIO_SRC)" -o -z "$(MINIO_DEST)" ]; then \
echo MINIO_SRC and MINIO_DEST must be defined when use $? target;\
exit 1; \
fi
$(MAKE) .defaults.minio.check
@# Help: Load $(MINIO_SRC) into minio at local/test/$(MINIO_DEST)
-mc mb local/test > /dev/null 2>&1 # Igore if it already exists
mc cp --recursive $(MINIO_SRC) local/test/$(MINIO_DEST)

22 changes: 8 additions & 14 deletions tools/ingest2parquet/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,13 @@ image:: .defaults.lib-src-image

test-src:: .defaults.test-src test-local

test-image:: image .defaults.test-image-pytest

test-local::
@# Help: Run the *local*.py files in the src directory
@source venv/bin/activate; \
export PYTHONPATH=src; \
files=$$(find src -name '*local*.py' ); \
for i in $$files; do \
echo Executing: python $$i; \
python $$i; \
if [ $$? -ne 0 ]; then \
exit 1; \
fi; \
done
test-image: image .defaults.test-image-pytest

minio-load:
$(MAKE) MINIO_SRC=test-data/input MINIO_DEST=ingest2parquet/input .defaults.minio.load-test-data

minio-start: .defaults.minio.start

test-local:: .defaults.test-locals

publish:: image .defaults.publish
Loading

0 comments on commit 8a1749c

Please sign in to comment.