[NodeJS18 Upgrade] Add new dockerfiles based on AL2023 and fix incorr…

…ectly copied yml and entrypoint (#3962) Signed-off-by: Peter Zhu <[email protected]>
opensearch-project · Sep 6, 2023 · eff36dc · eff36dc
1 parent 811ec92
commit eff36dc
Show file tree

Hide file tree

Showing 9 changed files with 225 additions and 9 deletions.
diff --git a/docker/release/dockerfiles/opensearch-dashboards.al2.dockerfile b/docker/release/dockerfiles/opensearch-dashboards.al2.dockerfile
@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 
-# This dockerfile generates an AmazonLinux-based image containing an OpenSearch-Dashboards installation.
+# This dockerfile generates an AmazonLinux-based image containing an OpenSearch-Dashboards installation (1.x Only).
 # Dockerfile for building an OpenSearch-Dashboards image.
 # It assumes that the working directory contains four files: an OpenSearch-Dashboards tarball (opensearch-dashboards.tgz), opensearch_dashboards.yml, opensearch-dashboards-docker-entrypoint.sh, and example certs.
 # Build arguments:
@@ -17,6 +17,7 @@ FROM amazonlinux:2 AS linux_stage_0
 
 ARG UID=1000
 ARG GID=1000
+ARG VERSION
 ARG TEMP_DIR=/tmp/opensearch-dashboards
 ARG OPENSEARCH_DASHBOARDS_HOME=/usr/share/opensearch-dashboards
 

diff --git a/docker/release/dockerfiles/opensearch-dashboards.al2023.dockerfile b/docker/release/dockerfiles/opensearch-dashboards.al2023.dockerfile
@@ -0,0 +1,101 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+
+# This dockerfile generates an AmazonLinux-based image containing an OpenSearch-Dashboards installation (2.x and above since release 2.10.0).
+# Dockerfile for building an OpenSearch-Dashboards image.
+# It assumes that the working directory contains four files: an OpenSearch-Dashboards tarball (opensearch-dashboards.tgz), opensearch_dashboards.yml, opensearch-dashboards-docker-entrypoint.sh, and example certs.
+# Build arguments:
+#   VERSION: Required. Used to label the image.
+#   BUILD_DATE: Required. Used to label the image. Should be in the form 'yyyy-mm-ddThh:mm:ssZ', i.e. a date-time from https://tools.ietf.org/html/rfc3339. The timestamp must be in UTC.
+#   UID: Optional. Specify the opensearch-dashboards userid. Defaults to 1000.
+#   GID: Optional. Specify the opensearch-dashboards groupid. Defaults to 1000.
+#   OPENSEARCH_DASHBOARDS_HOME: Optional. Specify the opensearch-dashboards root directory. Defaults to /usr/share/opensearch-dashboards.
+
+########################### Stage 0 ########################
+FROM amazonlinux:2023 AS linux_stage_0
+
+ARG UID=1000
+ARG GID=1000
+ARG VERSION
+ARG TEMP_DIR=/tmp/opensearch-dashboards
+ARG OPENSEARCH_DASHBOARDS_HOME=/usr/share/opensearch-dashboards
+
+# Update packages
+# Install the tools we need: tar and gzip to unpack the OpenSearch tarball, and shadow-utils to give us `groupadd` and `useradd`.
+# Install which to allow running of securityadmin.sh
+RUN dnf update -y && dnf install -y tar gzip shadow-utils which && dnf clean all
+
+# Create an opensearch-dashboards user, group, and directory
+RUN groupadd -g $GID opensearch-dashboards && \
+    adduser -u $UID -g $GID -d $OPENSEARCH_DASHBOARDS_HOME opensearch-dashboards && \
+    mkdir $TEMP_DIR
+
+# Prepare working directory
+COPY * $TEMP_DIR/
+RUN tar -xzpf $TEMP_DIR/opensearch-dashboards-`uname -p`.tgz -C $OPENSEARCH_DASHBOARDS_HOME --strip-components=1 && \
+    MAJOR_VERSION_ENTRYPOINT=`echo $VERSION | cut -d. -f1` && \
+    MAJOR_VERSION_YML=`echo $VERSION | cut -d. -f1` && \
+    echo $MAJOR_VERSION_ENTRYPOINT && echo $MAJOR_VERSION_YML && \
+    if ! (ls $TEMP_DIR | grep -E "opensearch-dashboards-docker-entrypoint-.*.x.sh" | grep $MAJOR_VERSION_ENTRYPOINT); then MAJOR_VERSION_ENTRYPOINT="default"; fi && \
+    if ! (ls $TEMP_DIR | grep -E "opensearch_dashboards-.*.x.yml" | grep $MAJOR_VERSION_YML); then MAJOR_VERSION_YML="default"; fi && \
+    cp -v $TEMP_DIR/opensearch-dashboards-docker-entrypoint-$MAJOR_VERSION_ENTRYPOINT.x.sh $OPENSEARCH_DASHBOARDS_HOME/opensearch-dashboards-docker-entrypoint.sh && \
+    cp -v $TEMP_DIR/opensearch_dashboards-$MAJOR_VERSION_YML.x.yml $OPENSEARCH_DASHBOARDS_HOME/config/opensearch_dashboards.yml && \
+    cp -v $TEMP_DIR/opensearch.example.org.* $OPENSEARCH_DASHBOARDS_HOME/config/ && \
+    echo "server.host: '0.0.0.0'" >> $OPENSEARCH_DASHBOARDS_HOME/config/opensearch_dashboards.yml && \
+    ls -l $OPENSEARCH_DASHBOARDS_HOME && \
+    rm -rf $TEMP_DIR
+
+########################### Stage 1 ########################
+# Copy working directory to the actual release docker images
+FROM amazonlinux:2023
+
+ARG UID=1000
+ARG GID=1000
+ARG OPENSEARCH_DASHBOARDS_HOME=/usr/share/opensearch-dashboards
+
+# Update packages
+# Install the tools we need: tar and gzip to unpack the OpenSearch tarball, and shadow-utils to give us `groupadd` and `useradd`.
+# Install which to allow running of securityadmin.sh
+RUN dnf update -y && dnf install -y tar gzip shadow-utils which && dnf clean all
+
+# Install Reporting dependencies
+RUN dnf install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc fontconfig freetype && dnf clean all
+
+# Create an opensearch-dashboards user, group
+RUN groupadd -g $GID opensearch-dashboards && \
+    adduser -u $UID -g $GID -d $OPENSEARCH_DASHBOARDS_HOME opensearch-dashboards
+
+COPY --from=linux_stage_0 --chown=$UID:$GID $OPENSEARCH_DASHBOARDS_HOME $OPENSEARCH_DASHBOARDS_HOME
+
+# Setup OpenSearch-dashboards
+WORKDIR $OPENSEARCH_DASHBOARDS_HOME
+
+# Set PATH
+ENV PATH=$PATH:$OPENSEARCH_DASHBOARDS_HOME/bin
+
+# Change user
+USER $UID
+
+# Expose port
+EXPOSE 5601
+
+ARG VERSION
+ARG BUILD_DATE
+ARG NOTES
+
+# Label
+LABEL org.label-schema.schema-version="1.0" \
+  org.label-schema.name="opensearch-dashboards" \
+  org.label-schema.version="$VERSION" \
+  org.label-schema.url="https://opensearch.org" \
+  org.label-schema.vcs-url="https://github.com/opensearch-project/OpenSearch-Dashboards" \
+  org.label-schema.license="Apache-2.0" \
+  org.label-schema.vendor="OpenSearch" \
+  org.label-schema.description="$NOTES" \
+  org.label-schema.build-date="$BUILD_DATE" \
+  "DOCKERFILE"="https://github.com/opensearch-project/opensearch-build/blob/main/docker/release/dockerfiles/opensearch-dashboards.al2.dockerfile"
+
+# CMD to run
+ENTRYPOINT ["./opensearch-dashboards-docker-entrypoint.sh"]
+CMD ["opensearch-dashboards"]
diff --git a/docker/release/dockerfiles/opensearch.al2.dockerfile b/docker/release/dockerfiles/opensearch.al2.dockerfile
@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 
-# This dockerfile generates an AmazonLinux-based image containing an OpenSearch installation.
+# This dockerfile generates an AmazonLinux-based image containing an OpenSearch installation (1.x Only).
 # Dockerfile for building an OpenSearch image.
 # It assumes that the working directory contains these files: an OpenSearch tarball (opensearch.tgz), log4j2.properties, opensearch.yml, opensearch-docker-entrypoint.sh, opensearch-onetime-setup.sh.
 # Build arguments:

diff --git a/docker/release/dockerfiles/opensearch.al2023.dockerfile b/docker/release/dockerfiles/opensearch.al2023.dockerfile
@@ -0,0 +1,114 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+
+# This dockerfile generates an AmazonLinux-based image containing an OpenSearch installation (2.x and above since release 2.10.0).
+# Dockerfile for building an OpenSearch image.
+# It assumes that the working directory contains these files: an OpenSearch tarball (opensearch.tgz), log4j2.properties, opensearch.yml, opensearch-docker-entrypoint.sh, opensearch-onetime-setup.sh.
+# Build arguments:
+#   VERSION: Required. Used to label the image.
+#   BUILD_DATE: Required. Used to label the image. Should be in the form 'yyyy-mm-ddThh:mm:ssZ', i.e. a date-time from https://tools.ietf.org/html/rfc3339. The timestamp must be in UTC.
+#   UID: Optional. Specify the opensearch userid. Defaults to 1000.
+#   GID: Optional. Specify the opensearch groupid. Defaults to 1000.
+#   OPENSEARCH_HOME: Optional. Specify the opensearch root directory. Defaults to /usr/share/opensearch.
+
+
+########################### Stage 0 ########################
+FROM amazonlinux:2023 AS linux_stage_0
+
+ARG UID=1000
+ARG GID=1000
+ARG TEMP_DIR=/tmp/opensearch
+ARG OPENSEARCH_HOME=/usr/share/opensearch
+ARG OPENSEARCH_PATH_CONF=$OPENSEARCH_HOME/config
+ARG SECURITY_PLUGIN_DIR=$OPENSEARCH_HOME/plugins/opensearch-security
+ARG PERFORMANCE_ANALYZER_PLUGIN_CONFIG_DIR=$OPENSEARCH_PATH_CONF/opensearch-performance-analyzer
+
+# Update packages
+# Install the tools we need: tar and gzip to unpack the OpenSearch tarball, and shadow-utils to give us `groupadd` and `useradd`.
+# Install which to allow running of securityadmin.sh
+RUN dnf update -y && dnf install -y tar gzip shadow-utils which && dnf clean all
+
+# Create an opensearch user, group, and directory
+RUN groupadd -g $GID opensearch && \
+    adduser -u $UID -g $GID -d $OPENSEARCH_HOME opensearch && \
+    mkdir $TEMP_DIR
+
+# Prepare working directory
+# Copy artifacts and configurations to corresponding directories
+COPY * $TEMP_DIR/
+RUN ls -l $TEMP_DIR && \
+    tar -xzpf /tmp/opensearch/opensearch-`uname -p`.tgz -C $OPENSEARCH_HOME --strip-components=1 && \
+    mkdir -p $OPENSEARCH_HOME/data && chown -Rv $UID:$GID $OPENSEARCH_HOME/data && \
+    if [[ -d $SECURITY_PLUGIN_DIR ]] ; then chmod -v 750 $SECURITY_PLUGIN_DIR/tools/* ; fi && \
+    if [[ -d $PERFORMANCE_ANALYZER_PLUGIN_CONFIG_DIR ]] ; then cp -v $TEMP_DIR/performance-analyzer.properties $PERFORMANCE_ANALYZER_PLUGIN_CONFIG_DIR; fi && \
+    cp -v $TEMP_DIR/opensearch-docker-entrypoint.sh $TEMP_DIR/opensearch-onetime-setup.sh $OPENSEARCH_HOME/ && \
+    cp -v $TEMP_DIR/log4j2.properties $TEMP_DIR/opensearch.yml $OPENSEARCH_PATH_CONF/ && \
+    ls -l $OPENSEARCH_HOME && \
+    rm -rf $TEMP_DIR
+
+
+########################### Stage 1 ########################
+# Copy working directory to the actual release docker images
+FROM amazonlinux:2023
+
+ARG UID=1000
+ARG GID=1000
+ARG OPENSEARCH_HOME=/usr/share/opensearch
+
+# Update packages
+# Install the tools we need: tar and gzip to unpack the OpenSearch tarball, and shadow-utils to give us `groupadd` and `useradd`.
+# Install which to allow running of securityadmin.sh
+RUN dnf update -y && dnf install -y tar gzip shadow-utils which && dnf clean all
+
+# Create an opensearch user, group
+RUN groupadd -g $GID opensearch && \
+    adduser -u $UID -g $GID -d $OPENSEARCH_HOME opensearch
+
+# Copy from Stage0
+COPY --from=linux_stage_0 --chown=$UID:$GID $OPENSEARCH_HOME $OPENSEARCH_HOME
+WORKDIR $OPENSEARCH_HOME
+
+# Set $JAVA_HOME
+RUN echo "export JAVA_HOME=$OPENSEARCH_HOME/jdk" >> /etc/profile.d/java_home.sh && \
+    echo "export PATH=\$PATH:\$JAVA_HOME/bin" >> /etc/profile.d/java_home.sh && \
+    ls -l $OPENSEARCH_HOME
+
+ENV JAVA_HOME=$OPENSEARCH_HOME/jdk
+ENV PATH=$PATH:$JAVA_HOME/bin:$OPENSEARCH_HOME/bin
+
+# Add k-NN lib directory to library loading path variable
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$OPENSEARCH_HOME/plugins/opensearch-knn/lib"
+
+# Change user
+USER $UID
+
+# Setup OpenSearch
+# Disable security demo installation during image build, and allow user to disable during startup of the container
+# Enable security plugin during image build, and allow user to disable during startup of the container
+ARG DISABLE_INSTALL_DEMO_CONFIG=true
+ARG DISABLE_SECURITY_PLUGIN=false
+RUN ./opensearch-onetime-setup.sh
+
+# Expose ports for the opensearch service (9200 for HTTP and 9300 for internal transport) and performance analyzer (9600 for the agent and 9650 for the root cause analysis component)
+EXPOSE 9200 9300 9600 9650
+
+ARG VERSION
+ARG BUILD_DATE
+ARG NOTES
+
+# Label
+LABEL org.label-schema.schema-version="1.0" \
+  org.label-schema.name="opensearch" \
+  org.label-schema.version="$VERSION" \
+  org.label-schema.url="https://opensearch.org" \
+  org.label-schema.vcs-url="https://github.com/opensearch-project/OpenSearch" \
+  org.label-schema.license="Apache-2.0" \
+  org.label-schema.vendor="OpenSearch" \
+  org.label-schema.description="$NOTES" \
+  org.label-schema.build-date="$BUILD_DATE" \
+  "DOCKERFILE"="https://github.com/opensearch-project/opensearch-build/blob/main/docker/release/dockerfiles/opensearch.al2.dockerfile"
+
+# CMD to run
+ENTRYPOINT ["./opensearch-docker-entrypoint.sh"]
+CMD ["opensearch"]
diff --git a/jenkins/docker/docker-re-release.jenkinsfile b/jenkins/docker/docker-re-release.jenkinsfile
@@ -1,4 +1,4 @@
-lib = library(identifier: '[email protected].1', retriever: modernSCM([
+lib = library(identifier: '[email protected].2', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))

diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile
@@ -1,4 +1,4 @@
-lib = library(identifier: 'jenkins@5.6.0', retriever: modernSCM([
+lib = library(identifier: 'jenkins@5.8.0', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))

diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile
@@ -1,4 +1,4 @@
-lib = library(identifier: 'jenkins@5.6.0', retriever: modernSCM([
+lib = library(identifier: 'jenkins@5.8.0', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))

diff --git a/tests/jenkins/TestDockerReRelease.groovy b/tests/jenkins/TestDockerReRelease.groovy
@@ -25,7 +25,7 @@ class TestDockerReRelease extends BuildPipelineTest {
 
         helper.registerSharedLibrary(
             library().name('jenkins')
-                .defaultVersion('5.7.1')
+                .defaultVersion('5.8.0')
                 .allowOverride(true)
                 .implicit(true)
                 .targetPath('vars')

diff --git a/tests/jenkins/jenkinsjob-regression-files/docker/docker-re-release.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/docker/docker-re-release.jenkinsfile.txt
@@ -1,14 +1,14 @@
    docker-re-release.run()
       docker-re-release.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git})
-      docker-re-release.library({[email protected].1, retriever=null})
+      docker-re-release.library({[email protected].2, retriever=null})
       docker-re-release.pipeline(groovy.lang.Closure)
          docker-re-release.timeout({time=2, unit=HOURS})
          docker-re-release.echo(Executing on agent [label:none])
          docker-re-release.stage(Patch Docker Image, groovy.lang.Closure)
             docker-re-release.script(groovy.lang.Closure)
                docker-re-release.patchDockerImage({product=opensearch, tag=1, re_release=true})
                   patchDockerImage.legacySCM(groovy.lang.Closure)
-                  patchDockerImage.library({identifier=jenkins@5.7.1, retriever=null})
+                  patchDockerImage.library({identifier=jenkins@5.8.0, retriever=null})
                   patchDockerImage.sh(#!/bin/bash
     set -e
     set +x
@@ -23,7 +23,7 @@
                   InputManifest.asBoolean()
                   patchDockerImage.buildDockerImage({inputManifest=manifests/1.3.0/opensearch-1.3.0.yml, buildNumber=7756, buildDate=20230619, buildOption=re_release_docker_image, artifactUrlX64=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/7756/linux/x64/tar/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz, artifactUrlArm64=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/7756/linux/arm64/tar/dist/opensearch/opensearch-1.3.0-linux-arm64.tar.gz})
                      buildDockerImage.legacySCM(groovy.lang.Closure)
-                     buildDockerImage.library({identifier=jenkins@5.7.1, retriever=null})
+                     buildDockerImage.library({identifier=jenkins@5.8.0, retriever=null})
                      buildDockerImage.readYaml({file=manifests/1.3.0/opensearch-1.3.0.yml})
                      InputManifest.asBoolean()
                      buildDockerImage.echo(Triggering docker-build)