From ffc547af353cbb47ff32e29f0e28ec503e8d3e52 Mon Sep 17 00:00:00 2001 From: Hans Keeler Date: Mon, 29 Jul 2019 12:29:02 -0400 Subject: [PATCH] Move root-required bits of drama-free-django build into Dockerfile (#5145) * Moves OS dependencies into Dockerfile This change was made to make it easier to run the build and test processes as alternate users, which is sometimes necessary to make the volumes permissions line up with the Docker host. Additionally, changes paths using `/`, which was causing permissions issues when running as non-root. * Converts dfd scripts to use docker build, then run * Revert back to centos:6 Docker image * Revert to original `static_out` path * Override PIP_NO_CACHE_DIR on pip upgrade The version of pip that comes with SCL python27 has a bug that fails to process PIP_NO_CACHE_DIR correctly. Adding --no-cache-dir overrides the envvar, preventing the error. * Removes Mac-specific `cached` volume attribute * Removes unneeded `which` package * Removes unneeded question comment * Add headings and "Notes" section to DFD README.md * Add yarn cache warning to DFD Docker README * Fix yarn warnings by setting $HOME in Dockerfile * Fix typo in drama-free-django/Dockerfile Co-Authored-By: Andy Chosak --- docker/drama-free-django/Dockerfile | 33 +++++++++++++++++++ docker/drama-free-django/README.md | 16 +++++++++ docker/drama-free-django/_build.sh | 13 -------- docker/drama-free-django/_test.sh | 11 +++---- docker/drama-free-django/build.sh | 10 +++++- docker/drama-free-django/docker-entrypoint.sh | 7 ++++ docker/drama-free-django/test.sh | 10 +++++- 7 files changed, 78 insertions(+), 22 deletions(-) create mode 100644 docker/drama-free-django/Dockerfile create mode 100755 docker/drama-free-django/docker-entrypoint.sh diff --git a/docker/drama-free-django/Dockerfile b/docker/drama-free-django/Dockerfile new file mode 100644 index 00000000000..f5933749ce2 --- /dev/null +++ b/docker/drama-free-django/Dockerfile @@ -0,0 +1,33 @@ +FROM centos:6 + +ENV SCL_PYTHON_VERSION python27 + +# Disables pip cache, which reduces build time, and suppresses warnings when run as non-root. +ENV PIP_NO_CACHE_DIR true + +ENV DFD_DIR /src/cfgov-refresh + +# Must be world writable since alternate uid:gid may be patched in at `docker run` time. +RUN mkdir -p ${DFD_DIR} && chmod 777 ${DFD_DIR} +WORKDIR ${DFD_DIR} + +# Sets a consistent $HOME no matter which user the container runs under. This prevents +# permissions issues caused by Docker's default `/` home directory. +ENV HOME /tmp/dfd-home +RUN mkdir -p ${HOME} && chmod 777 ${HOME} + +# Install dependencies +# NOTE: You MUST upgrade pip before using it further. The version packaged with SCL has issues +# with both setuptools and the PIP_NO_CACHE_DIR envvar (hence the --no-cache-dir override). +RUN yum install -y centos-release-scl && \ + curl -sL https://rpm.nodesource.com/setup_10.x | bash - && \ + curl -sL https://dl.yarnpkg.com/rpm/yarn.repo | tee /etc/yum.repos.d/yarn.repo && \ + yum install -y ${SCL_PYTHON_VERSION} gcc git nodejs yarn && \ + echo "source scl_source enable ${SCL_PYTHON_VERSION}" > /etc/profile.d/scl_python.sh && \ + source /etc/profile && \ + pip install --no-cache-dir -U pip && \ + pip install -U git+https://github.com/cfpb/drama-free-django.git + +COPY _build.sh _test.sh docker-entrypoint.sh ./ + +ENTRYPOINT ["./docker-entrypoint.sh"] diff --git a/docker/drama-free-django/README.md b/docker/drama-free-django/README.md index b17dc002cdc..b504c632e54 100644 --- a/docker/drama-free-django/README.md +++ b/docker/drama-free-django/README.md @@ -1,3 +1,7 @@ +# Docker-based drama-free-django build and test tools + +## Build + Run the `build.sh` script from the project root: ```sh @@ -6,6 +10,8 @@ docker/drama-free-django/build.sh This will run a CentOS 6 container to generate a [drama-free-django](https://github.com/cfpb/drama-free-django) release artifact in the project root named `cfgov_current_build.zip`. +## Test + To run a basic test of the artifact: ```sh @@ -14,3 +20,13 @@ docker/drama-free-django/test.sh This will run a CentOS 6 container to validate the built artifact by extracting it and running Django [`collectstatic`](https://docs.djangoproject.com/en/1.11/ref/contrib/staticfiles/#collectstatic). + +## Notes + +1. When running the container as a user that exists on the host, but not in the container, you may notice a warning similar to: + + ``` + /usr/bin/id: cannot find name for user ID 502 + ``` + + This is not anything to worry about. It simply means the uid/gid don't match any users/groups setup in the container. diff --git a/docker/drama-free-django/_build.sh b/docker/drama-free-django/_build.sh index af57fa4d088..f23305565bc 100755 --- a/docker/drama-free-django/_build.sh +++ b/docker/drama-free-django/_build.sh @@ -22,19 +22,6 @@ if [ ! -d "$cfgov_refresh_volume" ]; then exit 1 fi -# Install build requirements. -yum install -y centos-release-scl -yum install -y gcc git python27 - -source /opt/rh/python27/enable - -pip install -U pip -pip install -U git+https://github.com/cfpb/drama-free-django.git - -curl -sL https://rpm.nodesource.com/setup_10.x | bash - -curl -sL https://dl.yarnpkg.com/rpm/yarn.repo | tee /etc/yum.repos.d/yarn.repo -yum install -y nodejs yarn - # Run the frontend build. pushd "$cfgov_refresh_volume" ./frontend.sh production diff --git a/docker/drama-free-django/_test.sh b/docker/drama-free-django/_test.sh index 133d8e817f5..139bbf7f057 100755 --- a/docker/drama-free-django/_test.sh +++ b/docker/drama-free-django/_test.sh @@ -8,6 +8,7 @@ set -x artifact_filename=cfgov_current_build.zip artifact_volume=/cfgov +dfd_test_dir=/tmp/dfd-test/release # Verify that the artifact volume has been mapped. if [ ! -d "$artifact_volume" ]; then @@ -16,15 +17,11 @@ if [ ! -d "$artifact_volume" ]; then exit 1 fi -# Install runtime requirements. -yum install -y centos-release-scl -yum install -y python27 - -source /opt/rh/python27/enable # Extract the artifact in /tmp. -cp "$artifact_volume/$artifact_filename" /tmp -cd /tmp +mkdir -p $dfd_test_dir +cp "$artifact_volume/$artifact_filename" $dfd_test_dir +cd $dfd_test_dir python "./$artifact_filename" cd current diff --git a/docker/drama-free-django/build.sh b/docker/drama-free-django/build.sh index be981bd9f8b..cce3640eda7 100755 --- a/docker/drama-free-django/build.sh +++ b/docker/drama-free-django/build.sh @@ -1,3 +1,11 @@ #!/usr/bin/env bash -docker run -v `pwd`:/cfgov centos:6 /cfgov/docker/drama-free-django/_build.sh +set -e + +docker build -t cfgov-dfd-builder docker/drama-free-django + +docker run \ + --rm \ + -u $(id -u):$(id -g) \ + -v $(pwd):/cfgov \ + cfgov-dfd-builder ./_build.sh diff --git a/docker/drama-free-django/docker-entrypoint.sh b/docker/drama-free-django/docker-entrypoint.sh new file mode 100755 index 00000000000..d81332ae48d --- /dev/null +++ b/docker/drama-free-django/docker-entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/bash --login +# This entrypoint is used primarily as means of setting up a consistent +# shell environment no matter which user the process runs as. By using +# --login, it guarantees /etc/profile is always sourced, unlike the +# non-login, non-interactive shell you get by default with `docker run`. + +exec "$@" \ No newline at end of file diff --git a/docker/drama-free-django/test.sh b/docker/drama-free-django/test.sh index 4f4c8a22595..1ccf54bb4c4 100755 --- a/docker/drama-free-django/test.sh +++ b/docker/drama-free-django/test.sh @@ -1,3 +1,11 @@ #!/usr/bin/env bash -docker run -v `pwd`:/cfgov centos:6 /cfgov/docker/drama-free-django/_test.sh +set -e + +docker build -t cfgov-dfd-builder docker/drama-free-django + +docker run \ + --rm \ + -u $(id -u):$(id -g) \ + -v $(pwd):/cfgov \ + cfgov-dfd-builder ./_test.sh