Merge pull request #296 from NERSC/sanic2

Major update to python 3 and Sanic

.github/workflows/integration.yaml

@@ -6,6 +6,7 @@ on:
     branches: 
     - main
     - sanic
+    - sanic2
 
 jobs:
   test:
@@ -15,7 +16,7 @@ jobs:
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
-          python-version: 2.7
+          python-version: 3.8
       - name: Install dependencies
         run: |
            sudo chmod 755 /var/log
@@ -26,7 +27,7 @@ jobs:
            sudo modprobe xfs
            pip install -r imagegw/requirements.txt
            # install packages for tests
-           pip install nose coverage cpp-coveralls
+           pip install nose coverage==5.5 cpp-coveralls==0.4.2
            # Not sure why this is needed
            LOC=$(dirname $(which coveralls) ); sudo mv $LOC/coveralls $LOC/cpp-coveralls
            pip install coveralls coveralls-merge

.gitignore

@@ -74,3 +74,10 @@ wlm_integration/slurm/test/test_shifterSpank_prolog
 wlm_integration/slurm/test/test_shifterSpank_cgroup
 wlm_integration/slurm/test/test_shifterSpank_util
 tokens.cfg
+imagegw/private
+*.bak
+.coverage
+rpms
+doc/html
+local/
+

.travis.yml

@@ -2,7 +2,7 @@ sudo: true
 dist: bionic
 language: python
 python:
-  - "2.7"
+  - "3.8"
 compiler:
   - clang
   - gcc
@@ -55,6 +55,7 @@ services:
   - redis
 after_failure:
   - df /tmp
+  - dmesg
   - find /tmp/|head -100
   - ls -latrh /tmp/*
   - ls -latrh /tmp/imagegw/*

Dockerfile

@@ -6,7 +6,7 @@
 #        If you are looking to build an image for the gateway.  Look at the Dockerfile
 #        in imagegw/src.
 
-FROM ubuntu:14.04
+FROM ubuntu:18.04
 MAINTAINER Shane Canon <[email protected]>
 
 # Thanks to Sven Dowideit <[email protected]>

NEWS

@@ -1,3 +1,14 @@
+22.02.1
+=======
+* Minor fixes and updates to run on CLE 6.X
+
+21.03.1
+=======
+* Update from Python 2 to Python 3 in the Gateway
+* Convert from Flask to Sanic in the Gateway 
+* Bug fix to catch Umoci failures
+* Update to Dockerfile to include tools to support use_external mode
+
 18.03.5
 ========
 * Add support for external mode (see docs/skopeo.rst)

README.md

@@ -39,4 +39,3 @@ https://www.nersc.gov/research-and-development/user-defined-images/
 # Documentation
 
 https://github.com/NERSC/shifter/tree/master/doc
-

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.62])
-AC_INIT([shifter], [18.03.5], [[email protected]])
+AC_INIT([shifter], [22.02.1], [[email protected]])
 AC_CONFIG_SRCDIR([src/shifter_core.h])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([auxdir])

extra/CI/base_udiRoot.conf

@@ -18,7 +18,7 @@ rootfsType=tmpfs
 siteFs=/home:/home
 siteEnv=SHIFTER_RUNTIME=1
 siteEnvAppend=PATH=/opt/udiImage/bin
-imageGateway=http://localhost:5000
+imageGateway=http://localhost:8000
 system=mycluster
 defaultImageType=docker
 populateEtcDynamically = 1

extra/CI/integration/test_capabilities.py

@@ -7,7 +7,7 @@
 image_name = sys.argv[1]
 
 pex = pexpect.spawnu("shifter --image=%s cat /proc/self/status | grep Cap" % image_name)
-pex.expect(u'CapInh:\s+0+')
-pex.expect(u'CapPrm:\s+0+')
-pex.expect(u'CapEff:\s+0+')
-pex.expect(u'CapBnd:\s+0+')
+pex.expect('CapInh:\s+0+')
+pex.expect('CapPrm:\s+0+')
+pex.expect('CapEff:\s+0+')
+pex.expect('CapBnd:\s+0+')

extra/CI/integration/test_etcmtab.py

@@ -7,4 +7,4 @@
 image_name = sys.argv[1]
 
 pex = pexpect.spawnu("shifter --image=%s ls -ld /etc/mtab" % image_name)
-pex.expect(u'/proc/mounts')
+pex.expect('/proc/mounts')

extra/CI/integration/test_etcpasswd.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 
-import pexpect
 import subprocess
 import sys
 
@@ -11,7 +10,15 @@
 username = stdout.strip()
 
 proc = subprocess.Popen(['getent','passwd',username], stdout=subprocess.PIPE)
-passwd, _ = proc.communicate()
+passwd = proc.communicate()[0].decode("utf-8").rstrip()
+
+cmd = ['shifter', '--image=%s' % (image_name), 'cat', '/etc/passwd']
+proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+out = proc.communicate()[0].decode("utf-8")
+
+for line in out.split('\n'):
+    if line.startswith(passwd):
+        sys.exit(0)
+
+sys.exit(1)
 
-pex = pexpect.spawnu("shifter --image=%s cat /etc/passwd" % image_name)
-pex.expect(unicode(passwd.replace('\n', '\r\n')))

extra/CI/integration/test_gpu_support.py

@@ -195,8 +195,8 @@ def _command_output_without_trailing_new_lines(self, out):
     def _assert_is_subset(self, subset, superset):
         #dirty conversion to dictionaries required here
         #(there is no assertSetContainsSubset method)
-        subset_dict = dict(zip(subset, len(subset)*[None]))
-        superset_dict = dict(zip(superset, len(superset)*[None]))
+        subset_dict = dict(list(zip(subset, len(subset)*[None])))
+        superset_dict = dict(list(zip(superset, len(superset)*[None])))
         self.assertDictContainsSubset(subset_dict, superset_dict)
 
 if __name__ == "__main__":

extra/CI/integration/test_shifterConfig_format.py

@@ -7,14 +7,14 @@
 image_name = sys.argv[1]
 
 pex = pexpect.spawnu("whoami")
-pex.expect(u'(\S+)')
+pex.expect('(\S+)')
 username = pex.match.groups()[0]
 
 pex = pexpect.spawnu("shifterimg lookup %s" % image_name)
-pex.expect(u'([0-9a-f]+)')
+pex.expect('([0-9a-f]+)')
 image_id = pex.match.groups()[0]
 
-expected_config = u'{"identifier":"%s","user":"%s","volMap":"","modules":""}' \
+expected_config = '{"identifier":"%s","user":"%s","volMap":"","modules":""}' \
         % (image_id, username)
 
 pex = pexpect.spawnu("shifter --image=%s cat /var/shifterConfig.json" % image_name)

extra/CI/integration_test.sh

@@ -9,7 +9,7 @@ fi
 
 PYDIR=
 for libpath in lib lib64; do
-    for pypath in python2.6 python2.7; do
+    for pypath in python2.6 python2.7 python3.7 python3.8; do
         for packagepath in site-packages dist-packages; do
             if [[ -e "/usr/$libpath/$pypath/$packagepath/shifter_imagegw" ]]; then
                 PYDIR="/usr/$libpath/$pypath/$packagepath"
@@ -51,12 +51,12 @@ fi
 sudo service munge start
 
 echo "Starting imagegw api"
-gunicorn -b 0.0.0.0:5000 --backlog 2048 --access-logfile=/var/log/shifter_imagegw/access.log --log-file=/var/log/shifter_imagegw/error.log shifter_imagegw.api:app &
+python -m shifter_imagegw.api &
 
 echo "setting up base config"
 sudo /bin/bash -c "cat /etc/shifter/udiRoot.conf.example | \
          sed 's|etcPath=.*|etcPath=/etc/shifter/shifter_etc_files|g' | \
-         sed 's|imageGateway=.*|imageGateway=http://localhost:5000|g' \
+         sed 's|imageGateway=.*|imageGateway=http://localhost:8000|g' \
          > /etc/shifter/udiRoot.conf"
 
 sudo mkdir -p /etc/shifter/shifter_etc_files
@@ -75,13 +75,14 @@ sudo touch /bin/nvidia-modprobe
 sudo chmod 755 /bin/nvidia-modprobe
 
 ## need to sleep a bit to let gunicorn get started
-sleep 10
+sleep 2
 
 
 echo "Pull Image"
 shifterimg pull ubuntu:16.04
 shifterimg lookup ubuntu:16.04
-ls /images
+ls -l /images
+shifter --image=ubuntu:16.04 echo test
 
 echo "var/tmp exists"
 ls -ld /var/tmp

extra/python/README.md

@@ -0,0 +1,4 @@
+# Shifter-Python
+
+This is a spec file to create an RPM with dependencies needed to
+run the image gateway.

extra/python/shifter-python.spec

@@ -0,0 +1,37 @@
+#
+# spec file for package shifter-skopeo
+#
+
+
+Name:           shifter-python
+Version:        21.12
+Release:        1
+Summary:	Python packages
+License:        Apache License 2.0
+Group:          System Environment/Base
+#Url:
+BuildRoot:      %{_tmppath}/%{name}-%{version}-build
+Requires:       skopeo
+BuildRequires:  python3-pip
+%description
+This provides python tools to handle the image convesion for Shifter.
+
+%prep
+
+%build
+
+%install
+pip3 install -t $RPM_BUILD_ROOT/opt/shifter-python-%{version} pymongo==3.6
+pip3 install -t $RPM_BUILD_ROOT/opt/shifter-python-%{version} sanic==20.12.3
+
+%post
+%postun
+
+%files
+%defattr(-,root,root)
+/opt/shifter-python-%{version}
+
+%changelog
+* Fri Dec 3 2021 Shane Canon <[email protected]> - 21.12-1
+- Initial version
+

extra/systemd/shifter_imagegw_36.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Shifter image manager
+After=munge.service
+
+[Service]
+Type=simple
+User=shifter
+Group=shifter
+PrivateTmp=true
+Environment=PYTHONPATH=/opt/shifter-python-21.12/
+PermissionsStartOnly=true
+ExecStart=/usr/bin/python3.6 -m shifter_imagegw.api
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+

imagegw/.dockerignore

@@ -2,3 +2,5 @@ test.json
 docker-compose.yml
 test/config
 *.pyc
+private
+*__pycache__*

imagegw/Dockerfile

@@ -1,9 +1,33 @@
-FROM python:2.7-slim
+FROM python:3.8-slim as tools
+
+RUN \
+    apt-get -y update && apt-get -y install curl gnupg golang git make go-md2man
+
+RUN \
+    echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list  && \
+    curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/Debian_10/Release.key | apt-key add - && \
+    apt-get -y update && \
+    apt-get -y install umoci
+
+RUN \
+    go get -d  github.com/opencontainers/image-tools/cmd/oci-image-tool && \
+    cd ~/go/src/github.com/opencontainers/image-tools/ && \
+    make && make install
+
+# Need newer version of skopeo then what is available from kubic
+FROM debian:sid as skopeo
+RUN apt-get -y update && apt-get -y install skopeo
+
+
+FROM python:3.8-slim
+
+RUN apt-get -y update && apt-get -y install squashfs-tools munge libassuan0 libgpgme11 ibdevmapper1.02.1
 
-RUN apt-get update && apt-get -y install openssh-client squashfs-tools munge
-RUN echo "   StrictHostKeyChecking no" >> /etc/ssh/ssh_config
 RUN mkdir /var/run/munge && chown munge /var/run/munge
 
+COPY --from=tools /usr/bin/umoci /usr/bin/oci-image-tool /usr/bin/
+COPY --from=skopeo /usr/bin/skopeo /usr/bin/
+COPY --from=skopeo /etc/containers/ /etc/containers/
 
 WORKDIR /usr/src/app
 
@@ -15,6 +39,8 @@ COPY . /usr/src/app
 
 RUN echo "CONFIG_PATH='/config'" >> /usr/src/app/shifter_imagegw/__init__.py
 
+ENV PYTHONPATH=/usr/src/app/
+
 ENTRYPOINT [ "./entrypoint.sh" ]
 CMD [ ]
 

imagegw/Makefile.test

@@ -13,7 +13,7 @@ testprep: test/config/ssh.key test/config/munge.key
 	docker volume inspect imagegw || docker volume create --name imagegw
 
 startdeps:
-	docker run -d --name mongot -p 27017:27017 mongo --smallfiles
+	docker run -d --name mongot -p 27017:27017 mongo
 
 starttest: testprep
 	docker build -t imagegwapi .

imagegw/docker-compose.yml

@@ -1,17 +1,20 @@
 api:
   image: imagegwapi
   ports:
-   - 5555:5000
+   - 5555:8000
   links:
    - mongo
-   - systema
-   - registry
   volumes:
-   - imagegw:/data
+   - imagegw:/images
+   - munge:/var/run/munge/
    - ./test/config:/config
-  command: munge:systema:munge munge:systemb:munge api
+  command: api
+munge:
+  image: munge
+  volumes:
+   - munge:/var/run/munge/
 mongo:
-  image: mongo:2.6
+  image: mongo:3
   volumes:
     - /data/db
   command: --smallfiles

imagegw/entrypoint.sh

@@ -13,15 +13,19 @@ fi
 for service in $@ ; do
   echo "service: $service"
   if [ "$service"  == "api" ] ; then
-    gunicorn -b 0.0.0.0:5000 --log-file /var/log/gunicorn.log \
-        --log-level $LOG_LEVEL  --backlog 2048 shifter_imagegw.api:app
+    python ./shifter_imagegw/api.py
   elif  [ $(echo $service|grep -c "munge:") -gt 0 ] ; then
     socket=$(echo $service|awk -F: '{print $2}')
     key=$(echo $service|awk -F: '{print $3}')
     cp /config/$key.key /etc/munge/$socket.key
     chown munge /etc/munge/$socket.key
     chmod 600 /etc/munge/$socket.key
     runuser -u munge -- /usr/sbin/munged  -S /var/run/munge/${socket}.socket --key-file=/etc/munge/$socket.key --force -F &
+  elif [ "$service"  == "munge" ] ; then
+    cp /config/munge.key /etc/munge/munge.key
+    chown munge /etc/munge/munge.key
+    chmod 600 /etc/munge/munge.key
+    runuser -u munge -- /usr/sbin/munged  --key-file=/etc/munge/munge.key --force -F
   else
     echo "$service not recognized"
   fi

imagegw/requirements.txt

@@ -1,4 +1,3 @@
-pymongo
-flask
-gunicorn
+pymongo==3.12.1
+sanic==20.9.1
 pylint