SUBMARINE-1387. Update Jupyter(4.0.8) and Conda(23.10.0) version

.dockerignore

@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Exclude submarine folders
+LICENSE
+LICENSE-binary
+NOTICE
+NOTICE-binary
+README.md
+bin/
+conf/
+dev-support/
+doap_Submarine.rdf
+helm-charts/
+licenses-binary
+pom.xml
+pyproject.toml
+submarine-all/
+submarine-client/
+submarine-cloud-v2/
+submarine-cloud-v3/
+submarine-commons/
+submarine-dist/
+submarine-experiment-prehandler/
+submarine-serve/
+submarine-server/
+submarine-test/
+submarine-workbench/
+submarine-workbench-v2/
+submarine.iml
+target/
+webapps/
+website/
+**/.git/
+**/.idea/
+**/.github/
+
+# Allow files in pysubmarine
+!submarine-sdk/pysubmarine/**
+# Allow files in pysubmarine
+!dev-support/docker-images/**
+
+# Ignore python Distribution / packaging
+.Python
+**/venv/
+**/env/
+**/build/
+**/develop-eggs/
+**/dist/
+**/downloads/
+**/eggs/
+**/lib/
+**/lib64/
+**/parts/
+**/sdist/
+**/var/
+**/.mypy_cache/
+**/*.egg-info/
+**/.installed.cfg
+**/*.egg

.github/workflows/jupyter_images.yml

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Building jupyter images
+
+# Trigger the workflow on master branch push
+on: [push, pull_request]
+
+jobs:
+  jupyter-images:
+    runs-on: ubuntu-latest
+    timeout-minutes: 240
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build submarine jupyter
+        run: ./dev-support/docker-images/jupyter/build.sh
+      - name: Build submarine jupyter gpu
+        run: ./dev-support/docker-images/jupyter-gpu/build.sh

dev-support/docker-images/jupyter-gpu/Dockerfile

@@ -75,26 +75,29 @@ RUN mv /tini /usr/local/bin/tini && chmod +x /usr/local/bin/tini
 # Install conda
 USER $NB_UID
 ARG PYTHON_VERSION=default
-# update conda version to 4.11.0
-ENV MINICONDA_VERSION=4.11.0 \
-    MINICONDA_MD5=7675bd23411179956bcc4692f16ef27d \
-    CONDA_VERSION=4.11.0
+# update conda version to 23.10.0-1
+ENV MINICONDA_VERSION=23.10.0-1 \
+    MINICONDA_SHA256=d0643508fa49105552c94a523529f4474f91730d3e0d1f168f1700c43ae67595 \
+    CONDA_VERSION=23.10.0
 
 WORKDIR /tmp
-RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
-    echo "${MINICONDA_MD5} *Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
-    /bin/bash Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
-    rm Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
+# repo in https://repo.anaconda.com/miniconda/
+RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py311_${MINICONDA_VERSION}-Linux-x86_64.sh && \
+    echo "${MINICONDA_SHA256} *Miniconda3-py311_${MINICONDA_VERSION}-Linux-x86_64.sh" | sha256sum -c - && \
+    /bin/bash Miniconda3-py311_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
+    rm Miniconda3-py311_${MINICONDA_VERSION}-Linux-x86_64.sh && \
     echo "conda ${CONDA_VERSION}" >> $CONDA_DIR/conda-meta/pinned && \
     conda config --system --prepend channels conda-forge && \
     conda config --system --set auto_update_conda false && \
     conda config --system --set show_channel_urls true && \
     conda config --system --set channel_priority flexible && \
     if [ ! $PYTHON_VERSION = 'default' ]; then conda install --freeze-installed --yes python=$PYTHON_VERSION; fi && \
     conda list python | grep '^python ' | tr -s ' ' | cut -d '.' -f 1,2 | sed 's/$/.*/' >> $CONDA_DIR/conda-meta/pinned && \
-    # nodejs is used for jupyter lab build
-    conda install --freeze-installed -c conda-forge nodejs==14.20.1 jupyterlab==3.5.2 jupyterlab-git==0.41.0 && \
-    pip install cvxpy==1.2.2 && \
+    conda init bash && \
+    source ~/.bashrc && conda activate && \
+    # it is used for jupyter lab build
+    conda install --freeze-installed -c conda-forge nodejs jupyterlab==4.0.8 jupyterlab-git && \
+    conda install --freeze-installed -c conda-forge cvxpy && \
     jupyter lab build && \
     # remove node_modules
     rm -rf /home/$NB_USER/.cache/yarn && \
@@ -104,25 +107,25 @@ RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA
     # clear conda to remove index cache, lock files, unused cache packages, and tarballs in /opt/conda/pkgs
     conda clean -a -y
 
-# Install latest stable qlib
-RUN pip --no-cache-dir install pyqlib==0.8.6
+### We removed the qlib dependency because the project currently does not have good support for python 3.9 and above. 
+### If you need this project, you can choose the py38 or py39 miniconda and rebuild the secondary image
+# RUN pip --no-cache-dir install pyqlib==0.9.3
+# RUN wget https://raw.githubusercontent.com/microsoft/qlib/v0.9.3/examples/workflow_by_code.ipynb -P $HOME
 
-# Install latest sumbarine python sdk and notebook
-RUN git clone --depth=1 https://github.com/apache/submarine && \
-    # replace numpy==1.19.2 to numpy>=1.20.0
-    sed -i "s/numpy==1.19.2/numpy>=1.20.0/" submarine/submarine-sdk/pysubmarine/setup.py && \
-    pip --no-cache-dir install -e submarine/submarine-sdk/pysubmarine[tf,pytorch] && \
+# Install sumbarine python sdk and notebook
+RUN mkdir -p submarine/submarine-sdk/pysubmarine
+COPY submarine-sdk/pysubmarine/ submarine/submarine-sdk/pysubmarine
+RUN pip --no-cache-dir install -e submarine/submarine-sdk/pysubmarine[tf,pytorch] && \
     # Add DeepFM example into notebook
     cp submarine/submarine-sdk/pysubmarine/example/submarine_experiment_sdk.ipynb $HOME && \
-    cp -r submarine/submarine-sdk/pysubmarine/example/{data,deepfm_example.ipynb,deepfm.json} $HOME && \
-    rm submarine -rf
-
-# Add qlib example in notebook
-RUN wget https://raw.githubusercontent.com/microsoft/qlib/v0.8.6/examples/workflow_by_code.ipynb -P $HOME
+    cp -r submarine/submarine-sdk/pysubmarine/example/{data,deepfm_example.ipynb,deepfm.json} $HOME
 
+ENV MLFLOW_S3_ENDPOINT_URL http://submarine-minio-service:9000
+ENV AWS_ACCESS_KEY_ID submarine_minio
+ENV AWS_SECRET_ACCESS_KEY submarine_minio
 
+COPY --chown=${NB_USER}:users dev-support/docker-images/jupyter/start-notebook.sh /usr/local/bin
 EXPOSE $NB_PORT
 ENTRYPOINT ["tini", "-g", "--"]
 CMD ["start-notebook.sh"]
-COPY --chown=${NB_USER}:users start-notebook.sh /usr/local/bin
 WORKDIR ${HOME}

dev-support/docker-images/jupyter-gpu/build.sh

@@ -29,4 +29,4 @@ SUBMARINE_HOME=${CURRENT_PATH}/../../..
 # build image
 echo "Start building the ${JUPYTER_IMAGE} docker image ..."
 cd ${CURRENT_PATH}
-docker build -t ${JUPYTER_IMAGE} .
+docker build -t ${JUPYTER_IMAGE} -f ${SUBMARINE_HOME}/dev-support/docker-images/jupyter-gpu/Dockerfile ${SUBMARINE_HOME}/

dev-support/docker-images/jupyter-gpu/start-notebook.sh

@@ -43,4 +43,4 @@ if [[ -n "${NB_PREFIX}" ]]; then
   NOTEBOOK_ARGS="--NotebookApp.base_url=${NB_PREFIX} ${NOTEBOOK_ARGS}"
 fi
 
-/bin/bash -c "jupyter notebook ${NOTEBOOK_ARGS}"
+/bin/bash -c "jupyter lab ${NOTEBOOK_ARGS}"

dev-support/docker-images/jupyter/Dockerfile

@@ -75,16 +75,17 @@ RUN mv /tini /usr/local/bin/tini && chmod +x /usr/local/bin/tini
 # Install conda
 USER $NB_UID
 ARG PYTHON_VERSION=default
-# update conda version to 4.11.0
-ENV MINICONDA_VERSION=4.11.0 \
-    MINICONDA_MD5=7675bd23411179956bcc4692f16ef27d \
-    CONDA_VERSION=4.11.0
+# update conda version to 23.10.0-1
+ENV MINICONDA_VERSION=23.10.0-1 \
+    MINICONDA_SHA256=d0643508fa49105552c94a523529f4474f91730d3e0d1f168f1700c43ae67595 \
+    CONDA_VERSION=23.10.0
 
 WORKDIR /tmp
-RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
-    echo "${MINICONDA_MD5} *Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
-    /bin/bash Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
-    rm Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
+# repo in https://repo.anaconda.com/miniconda/
+RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py311_${MINICONDA_VERSION}-Linux-x86_64.sh && \
+    echo "${MINICONDA_SHA256} *Miniconda3-py311_${MINICONDA_VERSION}-Linux-x86_64.sh" | sha256sum -c - && \
+    /bin/bash Miniconda3-py311_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
+    rm Miniconda3-py311_${MINICONDA_VERSION}-Linux-x86_64.sh && \
     echo "conda ${CONDA_VERSION}" >> $CONDA_DIR/conda-meta/pinned && \
     conda config --system --prepend channels conda-forge && \
     conda config --system --set auto_update_conda false && \
@@ -95,8 +96,8 @@ RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA
     conda init bash && \
     source ~/.bashrc && conda activate && \
     # it is used for jupyter lab build
-    conda install --freeze-installed -c conda-forge nodejs==14.20.1 jupyterlab==3.5.2 jupyterlab-git==0.41.0 && \
-    pip install cvxpy==1.2.2 && \
+    conda install --freeze-installed -c conda-forge nodejs jupyterlab==4.0.8 jupyterlab-git && \
+    conda install --freeze-installed -c conda-forge cvxpy && \
     jupyter lab build && \
     # remove node_modules
     rm -rf /home/$NB_USER/.cache/yarn && \
@@ -106,26 +107,25 @@ RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA
     # clear conda to remove index cache, lock files, unused cache packages, and tarballs in /opt/conda/pkgs
     conda clean -a -y
 
-RUN pip --no-cache-dir install pyqlib==0.8.6
+### We removed the qlib dependency because the project currently does not have good support for python 3.9 and above. 
+### If you need this project, you can choose the py38 or py39 miniconda and rebuild the secondary image
+# RUN pip --no-cache-dir install pyqlib==0.9.3
+# RUN wget https://raw.githubusercontent.com/microsoft/qlib/v0.9.3/examples/workflow_by_code.ipynb -P $HOME
 
-# Install latest sumbarine python sdk and notebook
-RUN git clone --depth=1 https://github.com/apache/submarine && \
-    # replace numpy==1.19.2 to numpy>=1.20.0
-    sed -i "s/numpy==1.19.2/numpy>=1.20.0/" submarine/submarine-sdk/pysubmarine/setup.py && \
-    pip --no-cache-dir install submarine/submarine-sdk/pysubmarine[tf,pytorch] && \
+# Install sumbarine python sdk and notebook
+RUN mkdir -p submarine/submarine-sdk/pysubmarine
+COPY submarine-sdk/pysubmarine/ submarine/submarine-sdk/pysubmarine
+RUN pip --no-cache-dir install -e submarine/submarine-sdk/pysubmarine[tf,pytorch] && \
+    # Add DeepFM example into notebook
     cp submarine/submarine-sdk/pysubmarine/example/submarine_experiment_sdk.ipynb $HOME && \
-    cp -r submarine/submarine-sdk/pysubmarine/example/{data,deepfm_example.ipynb,deepfm.json} $HOME && \
-    rm submarine -rf
-
-# Add qlib example in notebook
-RUN wget https://raw.githubusercontent.com/microsoft/qlib/v0.8.6/examples/workflow_by_code.ipynb -P $HOME
+    cp -r submarine/submarine-sdk/pysubmarine/example/{data,deepfm_example.ipynb,deepfm.json} $HOME
 
 ENV MLFLOW_S3_ENDPOINT_URL http://submarine-minio-service:9000
 ENV AWS_ACCESS_KEY_ID submarine_minio
 ENV AWS_SECRET_ACCESS_KEY submarine_minio
 
+COPY --chown=${NB_USER}:users dev-support/docker-images/jupyter/start-notebook.sh /usr/local/bin
 EXPOSE $NB_PORT
 ENTRYPOINT ["tini", "-g", "--"]
 CMD ["start-notebook.sh"]
-COPY --chown=${NB_USER}:users start-notebook.sh /usr/local/bin
 WORKDIR ${HOME}

dev-support/docker-images/jupyter/build.sh

@@ -29,4 +29,4 @@ SUBMARINE_HOME=${CURRENT_PATH}/../../..
 # build image
 echo "Start building the ${JUPYTER_IMAGE} docker image ..."
 cd ${CURRENT_PATH}
-docker build -t ${JUPYTER_IMAGE} .
+docker build -t ${JUPYTER_IMAGE} -f ${SUBMARINE_HOME}/dev-support/docker-images/jupyter/Dockerfile ${SUBMARINE_HOME}/

dev-support/docker-images/jupyter/start-notebook.sh

@@ -43,4 +43,4 @@ if [[ -n "${NB_PREFIX}" ]]; then
   NOTEBOOK_ARGS="--NotebookApp.base_url=${NB_PREFIX} ${NOTEBOOK_ARGS}"
 fi
 
-/bin/bash -c "jupyter notebook ${NOTEBOOK_ARGS}"
+/bin/bash -c "jupyter lab ${NOTEBOOK_ARGS}"

submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfVars.java

@@ -83,8 +83,8 @@ public enum ConfVars {
         "org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory"),
     SUBMARINE_SUBMITTER("submarine.submitter", "k8s"),
     SUBMARINE_SERVER_SERVICE_NAME("submarine.server.service.name", "submarine-server"),
-    ENVIRONMENT_CONDA_MIN_VERSION("environment.conda.min.version", "4.0.1"),
-    ENVIRONMENT_CONDA_MAX_VERSION("environment.conda.max.version", "4.11.10"),
+    ENVIRONMENT_CONDA_MIN_VERSION("environment.conda.min.version", "23.1.0"),
+    ENVIRONMENT_CONDA_MAX_VERSION("environment.conda.max.version", "23.12.10"),
 
     /* cookie setting */
     SUBMARINE_COOKIE_HTTP_ONLY("submarine.cookie.http.only", false),

submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/ExperimentSpecParserTest.java

@@ -33,7 +33,6 @@
 import io.kubernetes.client.openapi.models.V1ObjectMeta;
 import io.kubernetes.client.openapi.models.V1Volume;
 
-import org.apache.submarine.commons.utils.SubmarineConfVars;
 import org.apache.submarine.commons.utils.SubmarineConfiguration;
 import org.apache.submarine.server.api.exception.InvalidSpecException;
 import org.apache.submarine.server.api.spec.ExperimentMeta;
@@ -63,6 +62,9 @@
 import io.kubernetes.client.openapi.models.V1EmptyDirVolumeSource;
 import io.kubernetes.client.openapi.models.V1EnvVar;
 
+import static org.apache.submarine.commons.utils.SubmarineConfVars.ConfVars.ENVIRONMENT_CONDA_MAX_VERSION;
+import static org.apache.submarine.commons.utils.SubmarineConfVars.ConfVars.ENVIRONMENT_CONDA_MIN_VERSION;
+
 
 public class ExperimentSpecParserTest extends SpecBuilder {
 
@@ -359,20 +361,22 @@ public void testValidPyTorchJobSpecWithEnv()
 
     String minVersion = "minVersion=\""
         + conf.getString(
-            SubmarineConfVars.ConfVars.ENVIRONMENT_CONDA_MIN_VERSION)
+            ENVIRONMENT_CONDA_MIN_VERSION)
         + "\";";
     String maxVersion = "maxVersion=\""
         + conf.getString(
-            SubmarineConfVars.ConfVars.ENVIRONMENT_CONDA_MAX_VERSION)
+            ENVIRONMENT_CONDA_MAX_VERSION)
         + "\";";
     String currentVersion = "currentVersion=$(conda -V | cut -f2 -d' ');";
     Assert.assertEquals(
         minVersion + maxVersion + currentVersion
             + "if [ \"$(printf '%s\\n' \"$minVersion\" \"$maxVersion\" "
                + "\"$currentVersion\" | sort -V | head -n2 | tail -1 )\" "
                     + "!= \"$currentVersion\" ]; then echo \"Conda version " +
-                    "should be between minVersion=\"4.0.1\"; " +
-                    "and maxVersion=\"4.11.10\";\"; exit 1; else echo "
+                    "should be between minVersion=\""
+                    + ENVIRONMENT_CONDA_MIN_VERSION.getStringValue() + "\"; "
+                    + "and maxVersion=\"" + ENVIRONMENT_CONDA_MAX_VERSION.getStringValue()
+                    + "\";\"; exit 1; else echo "
                     + "\"Conda current version is " + currentVersion + ". "
                         + "Moving forward with env creation and activation.\"; "
                         + "fi && " +

submarine-server/server-submitter/submitter-k8s/src/test/resources/client/notebook/notebook-read-api.json

@@ -27,7 +27,7 @@
               },
               {
                 "name": "INSTALL_ENVIRONMENT_COMMAND",
-                "value": "minVersion=\"4.0.1\";maxVersion=\"4.11.10\";currentVersion=$(conda -V | cut -f2 -d' ');if [ \"$(printf '%s\\n' \"$minVersion\" \"$maxVersion\" \"$currentVersion\" | sort -V | head -n2 | tail -1 )\" != \"$currentVersion\" ]; then echo \"Conda version should be between minVersion=\"4.0.1\"; and maxVersion=\"4.11.10\";\"; exit 1; else echo \"Conda current version is currentVersion=$(conda -V | cut -f2 -d' ');. Moving forward with env creation and activation.\"; fi"
+                "value": "minVersion=\"23.1.0\";maxVersion=\"23.12.10\";currentVersion=$(conda -V | cut -f2 -d' ');if [ \"$(printf '%s\\n' \"$minVersion\" \"$maxVersion\" \"$currentVersion\" | sort -V | head -n2 | tail -1 )\" != \"$currentVersion\" ]; then echo \"Conda version should be between minVersion=\"23.1.0\"; and maxVersion=\"23.12.10\";\"; exit 1; else echo \"Conda current version is currentVersion=$(conda -V | cut -f2 -d' ');. Moving forward with env creation and activation.\"; fi"
               }
             ],
             "image": "syharbor.sypesco.com:5000/apache/submarine:jupyter-notebook-0.7.0-syzh",

submarine-test/test-k8s/src/test/java/org/apache/submarine/rest/ExperimentRestApiTest.java

@@ -65,6 +65,9 @@
 import io.kubernetes.client.util.ClientBuilder;
 import io.kubernetes.client.util.KubeConfig;
 
+import static org.apache.submarine.commons.utils.SubmarineConfVars.ConfVars.ENVIRONMENT_CONDA_MAX_VERSION;
+import static org.apache.submarine.commons.utils.SubmarineConfVars.ConfVars.ENVIRONMENT_CONDA_MIN_VERSION;
+
 @SuppressWarnings("rawtypes")
 public class ExperimentRestApiTest extends AbstractSubmarineServerTest {
   private static final Logger LOG = LoggerFactory.getLogger(ExperimentRestApiTest.class);
@@ -343,8 +346,10 @@ private void assertK8sResultEquals(Environment env, Experiment experiment) throw
             + "if [ \"$(printf '%s\\n' \"$minVersion\" \"$maxVersion\" "
             + "\"$currentVersion\" | sort -V | head -n2 | tail -1 )\" "
             + "!= \"$currentVersion\" ]; then echo \"Conda version " +
-            "should be between minVersion=\"4.0.1\"; " +
-            "and maxVersion=\"4.11.10\";\"; exit 1; else echo "
+            "should be between minVersion=\""
+            + ENVIRONMENT_CONDA_MIN_VERSION.getStringValue() + "\"; " +
+            "and maxVersion=\"" + ENVIRONMENT_CONDA_MAX_VERSION.getStringValue()
+            + "\";\"; exit 1; else echo "
             + "\"Conda current version is " + currentVersion + ". "
             + "Moving forward with env creation and activation.\"; "
             + "fi && ";