Merge pull request #6827 from cfpb/python38

Update to Python 3.8

Dockerfile

@@ -5,6 +5,11 @@ ENV LANG en_US.UTF-8
 
 LABEL maintainer="[email protected]"
 
+# Specify SCL-based Python version
+# Currently used option: rh-python38
+# See: https://www.softwarecollections.org/en/scls/user/rhscl/?search=python
+ARG scl_python_version
+ENV SCL_PYTHON_VERSION ${scl_python_version}
 
 # Stops Python default buffering to stdout, improving logging to the console.
 ENV PYTHONUNBUFFERED 1
@@ -28,46 +33,29 @@ RUN yum -y install \
         git \
         mailcap \
         postgresql10 \
+        postgresql10-devel \
         which \
-        gettext && \
+        gettext \
+        xmlsec1 xmlsec1-openssl \
+        ${SCL_PYTHON_VERSION} && \
     yum -y install nodejs yarn && \
-    yum clean all && rm -rf /var/cache/yum
-
-# Build python
-WORKDIR /tmp
-ENV PYTHONVERSION=3.9.9
-RUN yum install -y epel-release
-RUN yum groupinstall -y "Development Tools"
-RUN yum install -y openssl-devel libffi-devel bzip2-devel wget
-RUN gcc --version
-RUN wget https://www.python.org/ftp/python/${PYTHONVERSION}/Python-${PYTHONVERSION}.tgz
-RUN tar xvf Python-${PYTHONVERSION}.tgz
-RUN cd Python-${PYTHONVERSION}/ && ./configure --enable-shared --enable-optimiztions && make altinstall && make bininstall
-RUN echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/lib" > /etc/profile.d/python39.sh
-RUN rm -Rf Python* *.pem
-RUN yum remove -y wget openssl-devel libffi-devel bzip2-devel
-RUN yum groupremove -y "Development Tools"
-RUN yum remove -y epel-release
-RUN yum clean all
-WORKDIR ${APP_HOME}
-
-RUN python3 -m pip install --no-cache-dir --upgrade pip setuptools wheel
+    yum clean all && rm -rf /var/cache/yum && \
+    echo "source scl_source enable ${SCL_PYTHON_VERSION}" > /etc/profile.d/enable_scl_python.sh && \
+    source /etc/profile && \
+    pip install --no-cache-dir --upgrade pip setuptools wheel
 
 # Disables pip cache. Reduces build time, and suppresses warnings when run as non-root.
 # NOTE: MUST be after pip upgrade. Build fails otherwise due to bug in old pip.
 ENV PIP_NO_CACHE_DIR true
 
 # Install python requirements
-RUN yum install -y postgresql-devel
-RUN yum install -y python3-devel.x86_64
 COPY requirements requirements
-RUN echo requirements/local.txt
-RUN python3 -m pip install -r requirements/local.txt -r requirements/deployment.txt
+RUN pip install -r requirements/local.txt -r requirements/deployment.txt
 
 EXPOSE 8000
 
 ENTRYPOINT ["./docker-entrypoint.sh"]
-CMD ["python3", "./cfgov/manage.py", "runserver", "0.0.0.0:8000"]
+CMD ["python", "./cfgov/manage.py", "runserver", "0.0.0.0:8000"]
 
 # Build Frontend Assets
 FROM cfgov-dev as cfgov-build
@@ -116,7 +104,7 @@ ENV ALLOWED_HOSTS '["*"]'
 
 # Install and enable SCL-based Apache server and mod_wsgi,
 # and converts all Docker Secrets into environment variables.
-RUN yum -y install ${SCL_HTTPD_VERSION} && \
+RUN yum -y install ${SCL_HTTPD_VERSION} ${SCL_PYTHON_VERSION}-mod_wsgi && \
     yum clean all && rm -rf /var/cache/yum && \
     echo "source scl_source enable ${SCL_HTTPD_VERSION}" > /etc/profile.d/enable_scl_httpd.sh && \
     echo '[ -d /var/run/secrets ] && cd /var/run/secrets && for s in *; do export $s=$(cat $s); done && cd -' > /etc/profile.d/secrets_env.sh
@@ -126,23 +114,16 @@ COPY --from=cfgov-build --chown=apache:apache ${CFGOV_PATH}/cfgov ${CFGOV_PATH}/
 COPY --from=cfgov-build --chown=apache:apache ${CFGOV_PATH}/docker-entrypoint.sh ${CFGOV_PATH}/refresh-data.sh ${CFGOV_PATH}/
 COPY --from=cfgov-build --chown=apache:apache ${CFGOV_PATH}/static.in ${CFGOV_PATH}/static.in
 
-# Build mod_wsgi
-WORKDIR /tmp
-RUN yum groupinstall -y "Development Tools"
-RUN yum install -y ${SCL_HTTPD_VERSION}-httpd-devel wget
-RUN wget https://github.com/GrahamDumpleton/mod_wsgi/archive/refs/tags/4.9.0.tar.gz -O mod_wsgi.tar.gz
-RUN tar xzvf mod_wsgi.tar.gz
-RUN mv mod_wsgi-* mod_wsgi
-RUN cd mod_wsgi && ./configure --with-python=/usr/local/bin/python3 && make && make install
-RUN rm -Rf mod_wsgi*
-RUN yum groupremove -y "Development Tools"
-RUN yum remove -y ${SCL_HTTPD_VERSION}-httpd-devel wget
-WORKDIR ${APP_HOME}
 
-# Clean up
 RUN yum clean all && rm -rf /var/cache/yum && \
     chown -R apache:apache ${APP_HOME} ${SCL_HTTPD_ROOT}/usr/share/httpd ${SCL_HTTPD_ROOT}/var/run
 
+ENV PATH="/opt/rh/${SCL_PYTHON_VERSION}/root/usr/bin:${PATH}"
+
+# Remove files flagged by image vulnerability scanner (doesn't seem to be needed in rh-python38)
+#RUN cd /opt/rh/rh-python38/root/usr/lib/python3.8/site-packages/ && \
+#    rm -f ndg/httpsclient/test/pki/localhost.key sslserver/certs/development.key
+
 USER apache
 
 # Build frontend, cleanup excess file, and setup filesystem

Jenkinsfile

@@ -166,7 +166,7 @@ pipeline {
                     docker.withRegistry(dockerRegistry.url, dockerRegistry.credentialsId) {
                         docker.build(
                             env.IMAGE_NAME_LOCAL,
-                            '--build-arg scl_python_version=rh-python36 --target cfgov-prod .'
+                            '--build-arg scl_python_version=rh-python38 --target cfgov-prod .'
                         )
                         if (IS_ES_IMAGE_UPDATED == 'true') {
                             echo "Building ES image"

activate-virtualenv.sh

@@ -25,9 +25,9 @@ if ! type "workon" &>/dev/null; then
   source $virtualenvwrapper_path
 fi
 
-if ! which python3.6 &>/dev/null; then
-  echo 'Error: python3.6 is not in your path.' \
-       'Please ensure python3.6 is installed and available'
+if ! which python3.8 &>/dev/null; then
+  echo 'Error: python3.8 is not in your path.' \
+       'Please ensure python3.8 is installed and available'
   return
 fi
 

cfgov/apache/conf.d/wsgi.conf.venv

@@ -0,0 +1,18 @@
+ServerName https://www.consumerfinance.gov
+
+LoadModule wsgi_module modules/mod_${SCL_PYTHON_VERSION}-wsgi.so
+
+WSGIApplicationGroup %{GLOBAL}
+WSGIDaemonProcess django home=${CFGOV_CURRENT} processes=${APACHE_PROCESS_COUNT} threads=15 display-name=%{GROUP} lang='en_US.UTF-8' locale='en_US.UTF-8' python-home=${CFGOV_CURRENT}/venv python-path=${CFGOV_CURRENT}/cfgov
+WSGIProcessGroup django
+WSGIScriptAlias / ${CFGOV_CURRENT}/cfgov/cfgov/wsgi.py
+
+<Directory ${CFGOV_PATH}>
+    Require all granted
+    Options +FollowSymLinks -Indexes
+</Directory>
+
+<Directory ${CFGOV_CURRENT}>
+    Require all granted
+    Options +FollowSymLinks
+</Directory>

cfgov/apache/conf/httpd.conf

@@ -27,7 +27,6 @@ PassEnv APACHE_PROCESS_COUNT
 PassEnv ERROR_LOG
 PassEnv ACCESS_LOG
 PassEnv SCL_PYTHON_VERSION
-PassEnv PYTHONHOME
 PassEnv APACHE_UPLOADS_F_ALIAS
 PassEnv APACHE_HTTPS_FORWARDED_HOST
 PassEnv APACHE_PORT

docker-compose.yml

@@ -35,6 +35,8 @@ services:
         build:
             context: .
             dockerfile: Dockerfile
+            args:
+                scl_python_version: rh-python38
         depends_on:
             - elasticsearch
             - postgres

docker/deployable-zipfile/Dockerfile

@@ -27,8 +27,8 @@ RUN yum -y update && \
     rpm -i https://rpm.nodesource.com/pub_16.x/el/7/x86_64/nodejs-16.13.1-1nodesource.x86_64.rpm && \
     curl -sL https://rpm.nodesource.com/setup_16.x | bash - && \
     curl -sL https://dl.yarnpkg.com/rpm/yarn.repo | tee /etc/yum.repos.d/yarn.repo && \
-    yum install -y rh-python36 gcc git nodejs yarn && \
-    echo "source scl_source enable rh-python36" > /etc/profile.d/scl_python.sh && \
+    yum install -y rh-python38 gcc git nodejs yarn && \
+    echo "source scl_source enable rh-python38" > /etc/profile.d/scl_python.sh && \
     source /etc/profile && \
     pip install --no-cache-dir -U pip setuptools wheel && \
     pip3 install --no-cache-dir -U pip setuptools wheel

docs/installation.md

@@ -81,16 +81,16 @@ outside of the Docker container, we rely on a local Python environment.
     brew install pyenv pyenv-virtualenv
     ```
 
-    Python 3.6 must then be installed once pyenv is installed:
+    Python 3.8 must then be installed once pyenv is installed:
 
     ```sh
-    pyenv install 3.6.9
+    pyenv install 3.8.12
     ```
 
 First we need to create a Python virtualenv for consumerfinance.gov:
 
 ```sh
-pyenv virtualenv 3.6.9 consumerfinance.gov
+pyenv virtualenv 3.8.12 consumerfinance.gov
 ```
 
 Then we'll need to activate it.

docs/running-docker.md

@@ -129,12 +129,12 @@ This includes:
 If you just want to build the image:
 
 ```bash
-docker build . --build-arg scl_python_version=rh-python36 -t your-desired-image-name
+docker build . --build-arg scl_python_version=rh-python38 -t your-desired-image-name
 ```
 
 **Note:** The `scl_python_version` build arg specifies which
 [Python Software Collection](https://www.softwarecollections.org/en/scls/?search=python)
-version you'd like to use. We've tested this against `rh-python36`.
+version you'd like to use. We've tested this against `rh-python38`.
 
 #### Docker Compose
 

tox.ini

@@ -30,7 +30,7 @@ allowlist_externals=echo
 changedir=
     unittest:           {[unittest-config]changedir}
 
-basepython=python3.6
+basepython=python3.8
 
 deps=
     lint:       {[lint-config]deps}