From d9a0fb4b174936b4c3712806fcfa95469f3a6441 Mon Sep 17 00:00:00 2001 From: wild-endeavor Date: Mon, 11 Jan 2021 13:14:25 -0800 Subject: [PATCH 1/9] wip --- flytetester/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flytetester/requirements.txt b/flytetester/requirements.txt index fd7e8c7..5a496bc 100644 --- a/flytetester/requirements.txt +++ b/flytetester/requirements.txt @@ -1,4 +1,4 @@ -flytekit[sidecar,schema]==0.16.0a0 +flytekit[sidecar,schema]==0.16.0a2 statsd opencv-python k8s-proto>=0.0.2 From 88baec7fe60b3424bae2fb1f24e5ee62a2c07957 Mon Sep 17 00:00:00 2001 From: wild-endeavor Date: Mon, 11 Jan 2021 13:29:58 -0800 Subject: [PATCH 2/9] use flyteorg --- flytetester/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flytetester/Makefile b/flytetester/Makefile index 0209d0a..e787a3f 100644 --- a/flytetester/Makefile +++ b/flytetester/Makefile @@ -20,7 +20,7 @@ register_staging: docker_push -e FLYTE_CREDENTIALS_CLIENT_SECRET=${FLYTE_CREDENTIALS_CLIENT_SECRET} \ -e FLYTE_CREDENTIALS_AUTH_MODE=basic -e FLYTE_CREDENTIALS_AUTHORIZATION_METADATA_KEY=flyte-authorization \ -e FLYTE_CREDENTIALS_SCOPE=svc -e FLYTE_PLATFORM_AUTH=True \ - docker.io/lyft/${IMAGE_NAME}:${VERSION} /usr/local/bin/flytekit_venv make register_staging_in_container + docker.io/flyteorg/${IMAGE_NAME}:${VERSION} /usr/local/bin/flytekit_venv make register_staging_in_container .PHONY: register_production_in_container register_production_in_container: @@ -32,7 +32,7 @@ register_production: docker_push -e FLYTE_CREDENTIALS_CLIENT_SECRET=${FLYTE_CREDENTIALS_CLIENT_SECRET} \ -e FLYTE_CREDENTIALS_AUTH_MODE=basic -e FLYTE_CREDENTIALS_AUTHORIZATION_METADATA_KEY=flyte-authorization \ -e FLYTE_CREDENTIALS_SCOPE=svc -e FLYTE_PLATFORM_AUTH=True \ - docker.io/lyft/${IMAGE_NAME}:${VERSION} /usr/local/bin/flytekit_venv make register_production_in_container + docker.io/flyteorg/${IMAGE_NAME}:${VERSION} /usr/local/bin/flytekit_venv make register_production_in_container .PHONY: register_production_in_container register_sandbox_in_container: @@ -40,7 +40,7 @@ register_sandbox_in_container: .PHONY: register_production register_sandbox: docker_push - docker run docker.io/lyft/${IMAGE_NAME}:${VERSION} /usr/local/bin/flytekit_venv make register_sandbox_in_container + docker run docker.io/flyteorg/${IMAGE_NAME}:${VERSION} /usr/local/bin/flytekit_venv make register_sandbox_in_container .PHONY: end2end end2end_test: @@ -52,7 +52,7 @@ docker_build: .PHONY: docker_push docker_push: - REGISTRY=docker.io/lyft scripts/docker_build.sh + REGISTRY=docker.io/flyteorg scripts/docker_build.sh .PHONY: docker_build_push docker_build_push: From 07ecb4a4bd2a6832e281c99707158b5dfe8ab9c5 Mon Sep 17 00:00:00 2001 From: wild-endeavor Date: Mon, 11 Jan 2021 14:08:31 -0800 Subject: [PATCH 3/9] move options around --- flytetester/end2end/run.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flytetester/end2end/run.sh b/flytetester/end2end/run.sh index 0dac75e..786c4b5 100755 --- a/flytetester/end2end/run.sh +++ b/flytetester/end2end/run.sh @@ -9,14 +9,14 @@ flytekit_venv flyte-cli -h flyteadmin:81 --insecure register-project -n flytetes # Currently, kill the admin pod, so that the init container sync picks up the change. # First register everything, but make sure to use local Dockernetes admin -flytekit_venv pyflyte -p flytetester -d development -c end2end/end2end.config register workflows +flytekit_venv pyflyte -c end2end/end2end.config register -p flytetester -d development workflows # Kick off workflows -flytekit_venv pyflyte -p flytetester -d development -c end2end/end2end.config lp execute app.workflows.work.WorkflowWithIO --b hello_world -flytekit_venv pyflyte -p flytetester -d development -c end2end/end2end.config lp execute app.workflows.failing_workflows.DivideByZeroWf -flytekit_venv pyflyte -p flytetester -d development -c end2end/end2end.config lp execute app.workflows.failing_workflows.RetrysWf -flytekit_venv pyflyte -p flytetester -d development -c end2end/end2end.config lp execute app.workflows.failing_workflows.FailingDynamicNodeWF -flytekit_venv pyflyte -p flytetester -d development -c end2end/end2end.config lp execute app.workflows.failing_workflows.RunToCompletionWF +flytekit_venv pyflyte -c end2end/end2end.config lp -p flytetester -d development execute app.workflows.work.WorkflowWithIO --b hello_world +flytekit_venv pyflyte -c end2end/end2end.config lp -p flytetester -d development execute app.workflows.failing_workflows.DivideByZeroWf +flytekit_venv pyflyte -c end2end/end2end.config lp -p flytetester -d development execute app.workflows.failing_workflows.RetrysWf +flytekit_venv pyflyte -c end2end/end2end.config lp -p flytetester -d development execute app.workflows.failing_workflows.FailingDynamicNodeWF +flytekit_venv pyflyte -c end2end/end2end.config lp -p flytetester -d development execute app.workflows.failing_workflows.RunToCompletionWF # Make sure workflow does everything correctly flytekit_venv python end2end/validator.py From e37ed7bdd44901a4e4c533b31f915355ab74dcf3 Mon Sep 17 00:00:00 2001 From: wild-endeavor Date: Mon, 11 Jan 2021 14:09:21 -0800 Subject: [PATCH 4/9] change image name --- flytetester/Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flytetester/Makefile b/flytetester/Makefile index e787a3f..9cc85f3 100644 --- a/flytetester/Makefile +++ b/flytetester/Makefile @@ -1,4 +1,4 @@ -export IMAGE_NAME=flytetester +export IMAGE_NAME=flytetools VERSION=$(shell ./version.sh) PROJECT=flytetester DOMAIN=development @@ -20,7 +20,7 @@ register_staging: docker_push -e FLYTE_CREDENTIALS_CLIENT_SECRET=${FLYTE_CREDENTIALS_CLIENT_SECRET} \ -e FLYTE_CREDENTIALS_AUTH_MODE=basic -e FLYTE_CREDENTIALS_AUTHORIZATION_METADATA_KEY=flyte-authorization \ -e FLYTE_CREDENTIALS_SCOPE=svc -e FLYTE_PLATFORM_AUTH=True \ - docker.io/flyteorg/${IMAGE_NAME}:${VERSION} /usr/local/bin/flytekit_venv make register_staging_in_container + ghcr.io/nuclyde-io/${IMAGE_NAME}:${VERSION} /usr/local/bin/flytekit_venv make register_staging_in_container .PHONY: register_production_in_container register_production_in_container: @@ -32,7 +32,7 @@ register_production: docker_push -e FLYTE_CREDENTIALS_CLIENT_SECRET=${FLYTE_CREDENTIALS_CLIENT_SECRET} \ -e FLYTE_CREDENTIALS_AUTH_MODE=basic -e FLYTE_CREDENTIALS_AUTHORIZATION_METADATA_KEY=flyte-authorization \ -e FLYTE_CREDENTIALS_SCOPE=svc -e FLYTE_PLATFORM_AUTH=True \ - docker.io/flyteorg/${IMAGE_NAME}:${VERSION} /usr/local/bin/flytekit_venv make register_production_in_container + ghcr.io/nuclyde-io/${IMAGE_NAME}:${VERSION} /usr/local/bin/flytekit_venv make register_production_in_container .PHONY: register_production_in_container register_sandbox_in_container: @@ -40,7 +40,7 @@ register_sandbox_in_container: .PHONY: register_production register_sandbox: docker_push - docker run docker.io/flyteorg/${IMAGE_NAME}:${VERSION} /usr/local/bin/flytekit_venv make register_sandbox_in_container + docker run ghcr.io/nuclyde-io/${IMAGE_NAME}:${VERSION} /usr/local/bin/flytekit_venv make register_sandbox_in_container .PHONY: end2end end2end_test: @@ -52,7 +52,7 @@ docker_build: .PHONY: docker_push docker_push: - REGISTRY=docker.io/flyteorg scripts/docker_build.sh + REGISTRY=ghcr.io/nuclyde-io scripts/docker_build.sh .PHONY: docker_build_push docker_build_push: From 326479be71d5341b1d6b4921d5707c45502490c6 Mon Sep 17 00:00:00 2001 From: wild-endeavor Date: Mon, 11 Jan 2021 16:22:56 -0800 Subject: [PATCH 5/9] wip --- docbuilder/Dockerfile | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/docbuilder/Dockerfile b/docbuilder/Dockerfile index c85ee55..3731a5c 100644 --- a/docbuilder/Dockerfile +++ b/docbuilder/Dockerfile @@ -1,15 +1,18 @@ -FROM ubuntu:bionic -RUN apt-get update \ - && apt-get install -y \ - python-virtualenv \ - python3-pip \ - python3-dev \ - && apt-get autoremove \ - && apt-get clean +FROM ubuntu:focal +RUN : \ + && apt-get update \ + && apt install -y software-properties-common \ + && add-apt-repository ppa:deadsnakes/ppa + +RUN : \ + && apt-get update \ + && apt-get install -y python3.8 python3-pip python3-venv make build-essential libssl-dev curl vim \ + && apt-get autoremove \ + && apt-get clean ENV VENV /venv -RUN virtualenv -p python3.6 ${VENV} \ +RUN python3.8 -m venv ${VENV} \ && ${VENV}/bin/pip install Sphinx \ && ${VENV}/bin/pip install sphinx_rtd_theme From e461362c9da2415ac5419e4b2b0f13f839bdd1fe Mon Sep 17 00:00:00 2001 From: wild-endeavor Date: Mon, 11 Jan 2021 18:29:04 -0800 Subject: [PATCH 6/9] wip --- docbuilder/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docbuilder/Dockerfile b/docbuilder/Dockerfile index 3731a5c..6755b4b 100644 --- a/docbuilder/Dockerfile +++ b/docbuilder/Dockerfile @@ -13,6 +13,7 @@ RUN : \ ENV VENV /venv RUN python3.8 -m venv ${VENV} \ + && ${VENV}/bin/pip install wheel \ && ${VENV}/bin/pip install Sphinx \ && ${VENV}/bin/pip install sphinx_rtd_theme From 810ce2918d0b0172f765873bb53f97d0bc297b56 Mon Sep 17 00:00:00 2001 From: wild-endeavor Date: Mon, 11 Jan 2021 18:59:58 -0800 Subject: [PATCH 7/9] downgrade to python3.7?? --- docbuilder/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docbuilder/Dockerfile b/docbuilder/Dockerfile index 6755b4b..ed97013 100644 --- a/docbuilder/Dockerfile +++ b/docbuilder/Dockerfile @@ -6,13 +6,13 @@ RUN : \ RUN : \ && apt-get update \ - && apt-get install -y python3.8 python3-pip python3-venv make build-essential libssl-dev curl vim \ + && apt-get install -y python3.7 python3-pip python3-venv make build-essential libssl-dev curl vim \ && apt-get autoremove \ && apt-get clean ENV VENV /venv -RUN python3.8 -m venv ${VENV} \ +RUN python3.7 -m venv ${VENV} \ && ${VENV}/bin/pip install wheel \ && ${VENV}/bin/pip install Sphinx \ && ${VENV}/bin/pip install sphinx_rtd_theme From 820ae4a2e649469707fd9f19676032cf56fa0a93 Mon Sep 17 00:00:00 2001 From: wild-endeavor Date: Mon, 11 Jan 2021 19:22:37 -0800 Subject: [PATCH 8/9] python37 --- docbuilder/Dockerfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docbuilder/Dockerfile b/docbuilder/Dockerfile index ed97013..af10eb0 100644 --- a/docbuilder/Dockerfile +++ b/docbuilder/Dockerfile @@ -6,19 +6,20 @@ RUN : \ RUN : \ && apt-get update \ - && apt-get install -y python3.7 python3-pip python3-venv make build-essential libssl-dev curl vim \ + && apt-get install -y python3.7 python3-pip python3.7-venv make build-essential libssl-dev curl vim \ && apt-get autoremove \ && apt-get clean ENV VENV /venv RUN python3.7 -m venv ${VENV} \ - && ${VENV}/bin/pip install wheel \ - && ${VENV}/bin/pip install Sphinx \ - && ${VENV}/bin/pip install sphinx_rtd_theme + && ${VENV}/bin/pip3 install wheel +RUN : \ + && ${VENV}/bin/pip3 install Sphinx \ + && ${VENV}/bin/pip3 install sphinx_rtd_theme # virtual env entrypoint COPY entrypoint.sh /opt # activate it by default -ENTRYPOINT ["/opt/entrypoint.sh"] +#ENTRYPOINT ["/opt/entrypoint.sh"] From e91a079b384be0c89c74d8fa5c8109fdd1980941 Mon Sep 17 00:00:00 2001 From: wild-endeavor Date: Mon, 11 Jan 2021 19:26:35 -0800 Subject: [PATCH 9/9] empty --- docbuilder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docbuilder/Dockerfile b/docbuilder/Dockerfile index af10eb0..fdd339a 100644 --- a/docbuilder/Dockerfile +++ b/docbuilder/Dockerfile @@ -22,4 +22,4 @@ RUN : \ COPY entrypoint.sh /opt # activate it by default -#ENTRYPOINT ["/opt/entrypoint.sh"] +ENTRYPOINT ["/opt/entrypoint.sh"]