Merge pull request #252 from splunk/mergeDevelopIntoMasterRelease0.2.2

Merge develop into master release0.2.2
splunk · Feb 10, 2021 · 09d2f6e · 09d2f6e
2 parents 57ac7a1 + 89ec4b2
commit 09d2f6e
Show file tree

Hide file tree

Showing 156 changed files with 17,359 additions and 1,052 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -8,22 +8,40 @@ orbs:
 
 workflows:
   version: 2
-  build-and-push:
+  nightly-integration-test:
+    triggers:
+      - schedule:
+          cron: "0 12 * * *"
+          filters:
+            branches:
+              only:
+                - develop
     jobs:
       - build-image
       - unit-tests
       - integration-tests:
           requires:
             - build-image
             - unit-tests
+      - vulnerability-scan:
+          requires:
+            - build-image     
+  build-and-push:
+    jobs:
+      - build-image
+      - unit-tests
+      - smoke-tests:
+          requires:
+            - build-image
+            - unit-tests
       - vulnerability-scan:
           requires:
             - build-image
       - push-latest:
           requires:
             - build-image
             - unit-tests
-            - integration-tests
+            - smoke-tests
             - vulnerability-scan
           filters:
             branches:
@@ -33,7 +51,7 @@ workflows:
           requires:
             - build-image
             - unit-tests
-            - integration-tests
+            - smoke-tests
             - vulnerability-scan
           filters:
             branches:
@@ -43,7 +61,7 @@ workflows:
           requires:
             - build-image
             - unit-tests
-            - integration-tests
+            - smoke-tests
             - vulnerability-scan
           filters:
             tags:
@@ -154,6 +172,64 @@ jobs:
       - store_artifacts:
           name: Save coverage.out as artifact
           path: coverage.out
+  # Runs smoke tests against a k8s cluster
+  smoke-tests:
+    executor: ubuntu-machine
+    steps:
+      - run: 
+          name: Setup Splunk operator and enterprise image env vars
+          command: |
+            echo 'export SPLUNK_OPERATOR_IMAGE=${IMAGE_NAME}:${CIRCLE_SHA1}' >> $BASH_ENV
+            echo 'export SPLUNK_ENTERPRISE_IMAGE=${ENTERPRISE_IMAGE_NAME}' >> $BASH_ENV
+            echo 'export COMMIT_HASH=$(echo ${CIRCLE_SHA1:0:7})' >> $BASH_ENV
+      - kubernetes/install
+      - aws-cli/install
+      - aws-eks/install-eksctl
+      - run:
+          name: Install kind tool
+          command: |
+            curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.7.0/kind-$(uname)-amd64
+            chmod +x ./kind
+            sudo mv ./kind /usr/local/bin
+      - checkout
+      - attach_workspace:
+          name: Restore workspace
+          at: /tmp
+      # load the operator image to local registry in the VM
+      - load_image
+      - run:
+          name: Print out version and environment
+          command: |
+            ls -al
+            echo "GO VERSION=`go version`"
+            echo "CIRCLE_SHA1=$CIRCLE_SHA1"
+            echo "SPLUNK_OPERATOR_IMAGE=$SPLUNK_OPERATOR_IMAGE"
+            echo "SPLUNK_ENTEPRISE_IMAGE=$SPLUNK_ENTERPRISE_IMAGE"
+            echo "PRIVATE_REGISTRY=$PRIVATE_REGISTRY"
+            echo "CLUSTER_PROVIDER=$CLUSTER_PROVIDER"
+      - run:
+          # Deploys a eks or kind cluster depending of CLUSTER_PROVIDER flag. If cluster already exists,
+          # it will skip. Uses NUM_WORKERS for size of cluster
+          name: Deploy k8s cluster
+          command: |
+            make cluster-up 
+            kubectl version # log the k8s version
+          no_output_timeout: 30m
+      - run:
+          # Run the smoke tests againsts the cluster deployed above.
+          # Test againsts the SPLUNK_OPERATOR_IMAGE and SPLUNK_ENTERPRISE_IMAGE 
+          name: Run smoke tests 
+          command: |
+            make int-test
+            mkdir -p /tmp/test-results
+            find ./test -name "*junit.xml" -exec cp {} /tmp/test-results \;
+          environment: 
+            TEST_FOCUS: smoke
+      - store_test_results:
+          name: Save test results
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
   # Runs integration tests against a k8s cluster
   integration-tests:
     executor: ubuntu-machine
@@ -205,6 +281,8 @@ jobs:
             make int-test
             mkdir -p /tmp/test-results
             find ./test -name "*junit.xml" -exec cp {} /tmp/test-results \;
+          environment: 
+            TEST_FOCUS: "smoke|ingest_search|monitoring_console|deletecr|smartstore"
       - store_test_results:
           name: Save test results
           path: /tmp/test-results

diff --git a/LICENSE b/LICENSE
@@ -1,4 +1,4 @@
-Copyright 2020 Splunk Inc.
+Copyright 2021 Splunk Inc.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

diff --git a/cmd/manager/main.go b/cmd/manager/main.go
@@ -1,4 +1,4 @@
-// Copyright (c) 2018-2020 Splunk Inc. All rights reserved.
+// Copyright (c) 2018-2021 Splunk Inc. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

diff --git a/deploy/crds/enterprise.splunk.com_clustermasters_crd.yaml b/deploy/crds/enterprise.splunk.com_clustermasters_crd.yaml
@@ -659,14 +659,26 @@ spec:
               description: Full path or URL for one or more default.yml files, separated
                 by commas
               type: string
-            ephemeralStorage:
-              description: If true, ephemeral (emptyDir) storage will be used for
-                /opt/splunk/etc and /opt/splunk/var volumes
-              type: boolean
-            etcStorage:
-              description: Storage capacity to request for /opt/splunk/etc persistent
-                volume claims (default=”1Gi”)
+            defaultsUrlApps:
+              description: Full path or URL for one or more defaults.yml files specific
+                to App install, separated by commas.  The defaults listed here will
+                be installed on the CM, standalone, search head deployer or license
+                master instance.
               type: string
+            etcVolumeStorageConfig:
+              description: Storage configuration for /opt/splunk/etc volume
+              properties:
+                ephemeralStorage:
+                  description: If true, ephemeral (emptyDir) storage will be used
+                  type: boolean
+                storageCapacity:
+                  description: Storage capacity to request persistent volume claims
+                    (default=”10Gi” for etc and "100Gi" for var)
+                  type: string
+                storageClassName:
+                  description: Name of StorageClass to use for persistent volume claims
+                  type: string
+              type: object
             image:
               description: Image to use for Splunk pod containers (overrides RELATED_IMAGE_SPLUNK_ENTERPRISE
                 environment variables)
@@ -740,6 +752,11 @@ spec:
               description: Name of Scheduler to use for pod placement (defaults to
                 “default-scheduler”)
               type: string
+            serviceAccount:
+              description: ServiceAccount is the service account used by the pods
+                deployed by the CRD. If not specified uses the default serviceAccount
+                for the namespace as per https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#use-the-default-service-account-to-access-the-api-server
+              type: string
             serviceTemplate:
               description: ServiceTemplate is a template used to create Kubernetes
                 services
@@ -1087,9 +1104,6 @@ spec:
                 to setup the soft links from ../master-apps/splunk-operator/local/
                 to /mnt/splunk-operator/local/
               type: string
-            storageClassName:
-              description: Name of StorageClass to use for persistent volume claims
-              type: string
             tolerations:
               description: Pod's tolerations for Kubernetes node's taint
               items:
@@ -1130,10 +1144,20 @@ spec:
                     type: string
                 type: object
               type: array
-            varStorage:
-              description: Storage capacity to request for /opt/splunk/var persistent
-                volume claims (default=”50Gi”)
-              type: string
+            varVolumeStorageConfig:
+              description: Storage configuration for /opt/splunk/var volume
+              properties:
+                ephemeralStorage:
+                  description: If true, ephemeral (emptyDir) storage will be used
+                  type: boolean
+                storageCapacity:
+                  description: Storage capacity to request persistent volume claims
+                    (default=”10Gi” for etc and "100Gi" for var)
+                  type: string
+                storageClassName:
+                  description: Name of StorageClass to use for persistent volume claims
+                  type: string
+              type: object
             volumes:
               description: List of one or more Kubernetes volumes. These will be mounted
                 in all pod containers as as /mnt/<name>

diff --git a/deploy/crds/enterprise.splunk.com_indexerclusters_crd.yaml b/deploy/crds/enterprise.splunk.com_indexerclusters_crd.yaml
@@ -681,14 +681,26 @@ spec:
               description: Full path or URL for one or more default.yml files, separated
                 by commas
               type: string
-            ephemeralStorage:
-              description: If true, ephemeral (emptyDir) storage will be used for
-                /opt/splunk/etc and /opt/splunk/var volumes
-              type: boolean
-            etcStorage:
-              description: Storage capacity to request for /opt/splunk/etc persistent
-                volume claims (default=”1Gi”)
+            defaultsUrlApps:
+              description: Full path or URL for one or more defaults.yml files specific
+                to App install, separated by commas.  The defaults listed here will
+                be installed on the CM, standalone, search head deployer or license
+                master instance.
               type: string
+            etcVolumeStorageConfig:
+              description: Storage configuration for /opt/splunk/etc volume
+              properties:
+                ephemeralStorage:
+                  description: If true, ephemeral (emptyDir) storage will be used
+                  type: boolean
+                storageCapacity:
+                  description: Storage capacity to request persistent volume claims
+                    (default=”10Gi” for etc and "100Gi" for var)
+                  type: string
+                storageClassName:
+                  description: Name of StorageClass to use for persistent volume claims
+                  type: string
+              type: object
             image:
               description: Image to use for Splunk pod containers (overrides RELATED_IMAGE_SPLUNK_ENTERPRISE
                 environment variables)
@@ -767,6 +779,11 @@ spec:
               description: Name of Scheduler to use for pod placement (defaults to
                 “default-scheduler”)
               type: string
+            serviceAccount:
+              description: ServiceAccount is the service account used by the pods
+                deployed by the CRD. If not specified uses the default serviceAccount
+                for the namespace as per https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#use-the-default-service-account-to-access-the-api-server
+              type: string
             serviceTemplate:
               description: ServiceTemplate is a template used to create Kubernetes
                 services
@@ -1005,9 +1022,6 @@ spec:
                       type: object
                   type: object
               type: object
-            storageClassName:
-              description: Name of StorageClass to use for persistent volume claims
-              type: string
             tolerations:
               description: Pod's tolerations for Kubernetes node's taint
               items:
@@ -1048,10 +1062,20 @@ spec:
                     type: string
                 type: object
               type: array
-            varStorage:
-              description: Storage capacity to request for /opt/splunk/var persistent
-                volume claims (default=”50Gi”)
-              type: string
+            varVolumeStorageConfig:
+              description: Storage configuration for /opt/splunk/var volume
+              properties:
+                ephemeralStorage:
+                  description: If true, ephemeral (emptyDir) storage will be used
+                  type: boolean
+                storageCapacity:
+                  description: Storage capacity to request persistent volume claims
+                    (default=”10Gi” for etc and "100Gi" for var)
+                  type: string
+                storageClassName:
+                  description: Name of StorageClass to use for persistent volume claims
+                  type: string
+              type: object
             volumes:
               description: List of one or more Kubernetes volumes. These will be mounted
                 in all pod containers as as /mnt/<name>
@@ -2325,10 +2349,6 @@ spec:
               description: Indicates whether the master is ready to begin servicing,
                 based on whether it is initialized.
               type: boolean
-            skip_recheck_update:
-              description: Indicates if we need to recheck the revision update on
-                pods
-              type: boolean
           type: object
       type: object
   version: v1beta1