diff --git a/Jenkinsfile b/Jenkinsfile index 5775c95e54..59fb65e40a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,6 +5,9 @@ pipeline { environment { VERSION = readFile "${env.WORKSPACE}/version.txt" BUILD_BRANCH = readFile "${env.WORKSPACE}/build_branch.txt" + DOCKER_REGISTRY = "docker.hops.works" + CONTROLLER_IMAGE = "${DOCKER_REGISTRY}/hopsworks/rss-controller:${VERSION}" + WEBHOOK_IMAGE = "${DOCKER_REGISTRY}/hopsworks/rss-webhook:${VERSION}" } stages { stage("checkout") { @@ -26,18 +29,26 @@ pipeline { sh """ set -ex echo "Building RSS version ${VERSION} on branch ${BUILD_BRANCH}" - docker login -u ${NEXUS_CREDS_USR} -p ${NEXUS_CREDS_PSW} docker.hops.works + docker login -u ${NEXUS_CREDS_USR} -p ${NEXUS_CREDS_PSW} $DOCKER_REGISTRY ./build_distribution.sh --spark3-profile spark3 --hadoop-profile hadoop3.2 --without-dashboard cd deploy/kubernetes/docker || exit - ./build.sh --hadoop-version 3.2.0.14-EE-RC0 --registry docker.hops.works --nexus-user $NEXUS_CREDS_USR --nexus-password $NEXUS_CREDS_PSW + ./build.sh --hadoop-version 3.2.0.14-EE-RC0 --registry $DOCKER_REGISTRY --nexus-user $NEXUS_CREDS_USR --nexus-password $NEXUS_CREDS_PSW cd ../../.. mkdir -p /opt/repository/master/rss/${VERSION}/ cp client-spark/spark3-shaded/target/rss-client-spark3-shaded-${VERSION}.jar /opt/repository/master/rss/${VERSION}/ + + # build the controller and webhook images + cd deploy/kubernetes/operator || exit 1 + docker build . --progress=plain -t $CONTROLLER_IMAGE --build-arg MODULE=controller -f hack/Dockerfile + docker build . --progress=plain -t $WEBHOOK_IMAGE --build-arg MODULE=webhook -f hack/Dockerfile + # push the controller and webhook images + docker push $CONTROLLER_IMAGE + docker push $WEBHOOK_IMAGE """ } } } } -} +} \ No newline at end of file diff --git a/deploy/kubernetes/docker/build.sh b/deploy/kubernetes/docker/build.sh index 09ef3ee00b..68269f3e77 100755 --- a/deploy/kubernetes/docker/build.sh +++ b/deploy/kubernetes/docker/build.sh @@ -140,7 +140,7 @@ GIT_COMMIT=$(git describe --dirty --always --tags | sed 's/-/./g') IMAGE_VERSION=$(cat $RSS_DIR/version.txt | awk '{$1=$1; print}' | sed '/^$/d') IMAGE=$REGISTRY/rss-server:$IMAGE_VERSION echo "image version: ${IMAGE_VERSION}" -IMAGE=$REGISTRY/rss:$IMAGE_VERSION +IMAGE=$REGISTRY/hopsworks/rss:$IMAGE_VERSION echo "building image: $IMAGE" DOCKER_BUILDKIT=1 docker build --network=host -t "$IMAGE" \ diff --git a/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go b/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go index cd53c2a126..978695343a 100644 --- a/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go +++ b/deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go @@ -142,6 +142,29 @@ type ShuffleServerConfig struct { // UpgradeStrategy defines upgrade strategy of shuffle servers. UpgradeStrategy *ShuffleServerUpgradeStrategy `json:"upgradeStrategy"` + + // volumeClaimTemplates is a list of claims that pods are allowed to reference. + // The StatefulSet controller is responsible for mapping network identities to + // claims in a way that maintains the identity of a pod. Every claim in + // this list must have at least one matching (by name) volumeMount in one + // container in the template. A claim in this list takes precedence over + // any volumes in the template, with the same name. + // +optional + VolumeClaimTemplates []ShuffleServerPersistentVolumeClaimTemplate `json:"volumeClaimTemplates,omitempty" protobuf:"bytes,4,rep,name=volumeClaimTemplates"` +} + +type ShuffleServerPersistentVolumeClaimTemplate struct { + // May contain labels and annotations that will be copied into the PVC + // when creating it. No other fields are allowed and will be rejected during + // validation. + // + VolumeNameTemplate *string `json:"volumeNameTemplate"` + + // The specification for the PersistentVolumeClaim. The entire content is + // copied unchanged into the PVC that gets created from this + // template. The same fields as in a PersistentVolumeClaim + // are also valid here. + Spec corev1.PersistentVolumeClaimSpec `json:"spec" protobuf:"bytes,2,name=spec"` } // ShuffleServerUpgradeStrategy defines upgrade strategy of shuffle servers. diff --git a/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go b/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go index 8d4d40def5..0c7389f295 100644 --- a/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/kubernetes/operator/api/uniffle/v1alpha1/zz_generated.deepcopy.go @@ -443,6 +443,13 @@ func (in *ShuffleServerConfig) DeepCopyInto(out *ShuffleServerConfig) { *out = new(ShuffleServerUpgradeStrategy) (*in).DeepCopyInto(*out) } + if in.VolumeClaimTemplates != nil { + in, out := &in.VolumeClaimTemplates, &out.VolumeClaimTemplates + *out = make([]ShuffleServerPersistentVolumeClaimTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShuffleServerConfig. @@ -455,6 +462,27 @@ func (in *ShuffleServerConfig) DeepCopy() *ShuffleServerConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ShuffleServerPersistentVolumeClaimTemplate) DeepCopyInto(out *ShuffleServerPersistentVolumeClaimTemplate) { + *out = *in + if in.VolumeNameTemplate != nil { + in, out := &in.VolumeNameTemplate, &out.VolumeNameTemplate + *out = new(string) + **out = **in + } + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShuffleServerPersistentVolumeClaimTemplate. +func (in *ShuffleServerPersistentVolumeClaimTemplate) DeepCopy() *ShuffleServerPersistentVolumeClaimTemplate { + if in == nil { + return nil + } + out := new(ShuffleServerPersistentVolumeClaimTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ShuffleServerUpgradeStrategy) DeepCopyInto(out *ShuffleServerUpgradeStrategy) { *out = *in diff --git a/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml b/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml index 314304cd5d..9dca5b4221 100644 --- a/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml +++ b/deploy/kubernetes/operator/config/crd/bases/uniffle.apache.org_remoteshuffleservices.yaml @@ -8362,6 +8362,205 @@ spec: required: - type type: object + volumeClaimTemplates: + description: volumeClaimTemplates is a list of claims that pods + are allowed to reference. The StatefulSet controller is responsible + for mapping network identities to claims in a way that maintains + the identity of a pod. Every claim in this list must have at + least one matching (by name) volumeMount in one container in + the template. A claim in this list takes precedence over any + volumes in the template, with the same name. + items: + properties: + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the PVC that + gets created from this template. The same fields as in + a PersistentVolumeClaim are also valid here. + properties: + accessModes: + description: 'accessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify + either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the provisioner + or an external controller can support the specified + data source, it will create a new volume based on + the contents of the specified data source. If the + AnyVolumeDataSource feature gate is enabled, this + field will always have the same contents as the DataSourceRef + field.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'dataSourceRef specifies the object from + which to populate the volume with data, if a non-empty + volume is desired. This may be any local object from + a non-empty API group (non core object) or a PersistentVolumeClaim + object. When this field is specified, volume binding + will only succeed if the type of the specified object + matches some installed volume populator or dynamic + provisioner. This field will replace the functionality + of the DataSource field and as such if both fields + are non-empty, they must have the same value. For + backwards compatibility, both fields (DataSource and + DataSourceRef) will be set to the same value automatically + if one of them is empty and the other is non-empty. + There are two important differences between DataSource + and DataSourceRef: * While DataSource only allows + two specific types of objects, DataSourceRef allows + any non-core object, as well as PersistentVolumeClaim + objects. * While DataSource ignores disallowed values + (dropping them), DataSourceRef preserves all values, + and generates an error if a disallowed value is specified. + (Beta) Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + resources: + description: 'resources represents the minimum resources + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify resource + requirements that are lower than previous value but + must still be higher than capacity recorded in the + status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is + omitted for a container, it defaults to Limits + if that is explicitly specified, otherwise to + an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'storageClassName is the name of the StorageClass + required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem is implied + when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to + the PersistentVolume backing this claim. + type: string + type: object + volumeNameTemplate: + description: May contain labels and annotations that will + be copied into the PVC when creating it. No other fields + are allowed and will be rejected during validation. + type: string + required: + - spec + - volumeNameTemplate + type: object + type: array volumeMounts: description: VolumeMounts indicates describes mountings of volumes within shuffle servers' container. diff --git a/deploy/kubernetes/operator/examples/pvc-example/README.md b/deploy/kubernetes/operator/examples/pvc-example/README.md new file mode 100644 index 0000000000..ea18de8444 --- /dev/null +++ b/deploy/kubernetes/operator/examples/pvc-example/README.md @@ -0,0 +1,28 @@ + + +# Use PVC for Shuffle Servers + +If we want to use PVC for shuffle servers, we need to edit the rss object as follows. + ++ update `.spec.shuffleServer.volumeMounts` with the mount points of PVC. ++ update `.spec.shuffleServer.volumeClaimTemplates` with the correct volume claim templates. + +We can refer to the [example](rss-pvc-on-gce.yaml). +Of course, you should create a storage class first if it's not already created. +You may have to ask your K8S administrator to create one. +A [storage class](gce-storage-class.yml) is also provided in this example dir. \ No newline at end of file diff --git a/deploy/kubernetes/operator/examples/pvc-example/gce-storage-class.yml b/deploy/kubernetes/operator/examples/pvc-example/gce-storage-class.yml new file mode 100644 index 0000000000..7a09ae372e --- /dev/null +++ b/deploy/kubernetes/operator/examples/pvc-example/gce-storage-class.yml @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: fast +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-ssd diff --git a/deploy/kubernetes/operator/examples/pvc-example/rss-pvc-on-gce.yaml b/deploy/kubernetes/operator/examples/pvc-example/rss-pvc-on-gce.yaml new file mode 100644 index 0000000000..a72020d360 --- /dev/null +++ b/deploy/kubernetes/operator/examples/pvc-example/rss-pvc-on-gce.yaml @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +apiVersion: uniffle.apache.org/v1alpha1 +kind: RemoteShuffleService +metadata: + name: rss-pvc-full-restart-demo + namespace: kube-system +spec: + configMapName: "${rss-configuration-name}" + coordinator: + image: "${rss-shuffle-server-image}" + initContainerImage: "busybox:latest" + rpcNodePort: + - 32005 + - 32015 + httpNodePort: + - 32006 + - 32016 + xmxSize: "10G" + configDir: "/data/rssadmin/rss/conf" + replicas: 1 + excludeNodesFilePath: "/data/rssadmin/rss/coo/exclude_nodes" + securityContext: + runAsUser: 1000 + fsGroup: 1000 + shuffleServer: + sync: true + replicas: 2 + rpcPort: 20009 + rpcNodePort: 20009 + httpPort: 20019 + httpNodePort: 20019 + image: "${rss-shuffle-server-image}" + initContainerImage: "busybox:latest" + upgradeStrategy: + type: "FullRestart" + xmxSize: "10G" + configDir: "/data/rssadmin/rss/conf" + securityContext: + runAsUser: 1000 + fsGroup: 1000 + # volumeMount.name is mapping to volumeClaimTemplate.volumeNameTemplate + volumeMounts: + - name: volume + mountPath: /data1 + volumeClaimTemplates: + - volumeNameTemplate: "volume" + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: fast + resources: + requests: + storage: "${storage-size}" diff --git a/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go b/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go index a96c0eba0c..867a40386b 100644 --- a/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go +++ b/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver.go @@ -198,6 +198,17 @@ func GenerateSts(kubeClient kubernetes.Interface, rss *unifflev1alpha1.RemoteShu sts.Spec.Template.Spec.RuntimeClassName = rss.Spec.ShuffleServer.RuntimeClassName } + // add VolumeClaimTemplates, support cloud storage + sts.Spec.VolumeClaimTemplates = make([]corev1.PersistentVolumeClaim, 0, len(rss.Spec.ShuffleServer.VolumeClaimTemplates)) + for _, pvcTemplate := range rss.Spec.ShuffleServer.VolumeClaimTemplates { + sts.Spec.VolumeClaimTemplates = append(sts.Spec.VolumeClaimTemplates, corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: *pvcTemplate.VolumeNameTemplate, + }, + Spec: pvcTemplate.Spec, + }) + } + // add init containers, the main container and other containers. sts.Spec.Template.Spec.InitContainers = util.GenerateInitContainers(rss.Spec.ShuffleServer.RSSPodSpec) containers := []corev1.Container{*generateMainContainer(rss)} @@ -238,6 +249,11 @@ func generateStorageBasePath(rss *unifflev1alpha1.RemoteShuffleService) string { } paths = append(paths, strings.TrimSuffix(v, "/")+"/"+controllerconstants.RssDataDir) } + + for _, vm := range rss.Spec.ShuffleServer.VolumeMounts { + paths = append(paths, strings.TrimSuffix(vm.MountPath, "/")+"/"+controllerconstants.RssDataDir) + } + sort.Strings(paths) return strings.Join(paths, ",") } diff --git a/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver_test.go b/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver_test.go index f8c7e0a7ed..dc151c5df8 100644 --- a/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver_test.go +++ b/deploy/kubernetes/operator/pkg/controller/sync/shuffleserver/shuffleserver_test.go @@ -27,6 +27,8 @@ import ( appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/pointer" @@ -138,6 +140,26 @@ var ( Name: "default-secret", }, } + + standard = "standard" + testVolumeClaimTemplates = []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "testVolumeClaimTemplate", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + "ReadWriteOnce", + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "storage": *resource.NewQuantity(5*1000*1000*1000, resource.DecimalSI), + }, + }, + StorageClassName: &standard, + }, + }, + } ) func buildRssWithLabels() *uniffleapi.RemoteShuffleService { @@ -199,6 +221,18 @@ func withCustomImagePullSecrets(imagePullSecrets []corev1.LocalObjectReference) return rss } +func withCustomVolumeClaimTemplates(volumeClaimTemplates []corev1.PersistentVolumeClaim) *uniffleapi.RemoteShuffleService { + rss := utils.BuildRSSWithDefaultValue() + rss.Spec.ShuffleServer.VolumeClaimTemplates = make([]uniffleapi.ShuffleServerPersistentVolumeClaimTemplate, 0, len(volumeClaimTemplates)) + for _, pvcTemplate := range volumeClaimTemplates { + rss.Spec.ShuffleServer.VolumeClaimTemplates = append(rss.Spec.ShuffleServer.VolumeClaimTemplates, uniffleapi.ShuffleServerPersistentVolumeClaimTemplate{ + VolumeNameTemplate: &pvcTemplate.ObjectMeta.Name, + Spec: pvcTemplate.Spec, + }) + } + return rss +} + func buildRssWithHPA() *uniffleapi.RemoteShuffleService { rss := utils.BuildRSSWithDefaultValue() rss.Spec.ShuffleServer.Autoscaler.Enable = true @@ -486,6 +520,21 @@ func TestGenerateSts(t *testing.T) { return false, fmt.Errorf("generated sts should include imagePullSecrets: %v", testImagePullSecrets) }, }, + { + name: "set volume claim templates", + rss: withCustomVolumeClaimTemplates(testVolumeClaimTemplates), + IsValidSts: func(deploy *appsv1.StatefulSet, rss *uniffleapi.RemoteShuffleService) (bool, error) { + if deploy.Spec.VolumeClaimTemplates != nil { + for _, volumeClaimTemplate := range deploy.Spec.VolumeClaimTemplates { + equal := reflect.DeepEqual(volumeClaimTemplate, testVolumeClaimTemplates[0]) + if equal { + return true, nil + } + } + } + return false, fmt.Errorf("generated sts should include volumeClaimTemplates: %v", testImagePullSecrets) + }, + }, } { t.Run(tt.name, func(tc *testing.T) { sts := GenerateSts(nil, tt.rss)