feat: Add Azure Blob configuration to the object storage setting (#221)

* feat: Add Azure Blob configuration to the object storage setting

* chore: Setting the OSS example secret using stringData

* refactor: Change the name of the blob to azblob

* chore: change readme link

apis/v1alpha1/common.go

@@ -573,18 +573,22 @@ func (in *TLSSpec) GetSecretName() string {
 
 // ObjectStorageProviderSpec defines the object storage provider for the cluster. The data will be stored in the storage.
 type ObjectStorageProviderSpec struct {
-	// S3 is the S3 storage configuration.
+	// S3 is the AWS S3 storage configuration.
 	// +optional
 	S3 *S3Storage `json:"s3,omitempty"`
 
 	// OSS is the Aliyun OSS storage configuration.
 	// +optional
 	OSS *OSSStorage `json:"oss,omitempty"`
 
-	// GCS is the Google GCS storage configuration.
+	// GCS is the Google cloud storage configuration.
 	// +optional
 	GCS *GCSStorage `json:"gcs,omitempty"`
 
+	// AZBlob is the Azure Blob storage configuration.
+	// +optional
+	AZBlob *AZBlobStorage `json:"azblob,omitempty"`
+
 	// Cache is the cache storage configuration for object storage.
 	// +optional
 	Cache *CacheStorage `json:"cache,omitempty"`
@@ -596,6 +600,7 @@ type ObjectStorageProviderAccessor interface {
 	GetS3Storage() *S3Storage
 	GetGCSStorage() *GCSStorage
 	GetOSSStorage() *OSSStorage
+	GetAZBlobStorage() *AZBlobStorage
 	GetCacheFileStorage() *FileStorage
 }
 
@@ -629,6 +634,13 @@ func (in *ObjectStorageProviderSpec) GetOSSStorage() *OSSStorage {
 	return nil
 }
 
+func (in *ObjectStorageProviderSpec) GetAZBlobStorage() *AZBlobStorage {
+	if in != nil {
+		return in.AZBlob
+	}
+	return nil
+}
+
 func (in *ObjectStorageProviderSpec) getSetObjectStorageCount() int {
 	count := 0
 	if in.S3 != nil {
@@ -640,6 +652,9 @@ func (in *ObjectStorageProviderSpec) getSetObjectStorageCount() int {
 	if in.GCS != nil {
 		count++
 	}
+	if in.AZBlob != nil {
+		count++
+	}
 	return count
 }
 
@@ -782,6 +797,41 @@ func (in *GCSStorage) GetRoot() string {
 	return ""
 }
 
+// AZBlobStorage defines the Azure Blob storage specification.
+type AZBlobStorage struct {
+	// The data will be stored in the container.
+	// +required
+	Container string `json:"container"`
+
+	// The secret of storing the credentials of account name and account key.
+	// The secret should contain keys named `account-name` and `account-key`.
+	// The secret must be the same namespace with the GreptimeDBCluster resource.
+	// +optional
+	SecretName string `json:"secretName,omitempty"`
+
+	// The Blob directory path.
+	// +required
+	Root string `json:"root"`
+
+	// The Blob Storage endpoint.
+	// +optional
+	Endpoint string `json:"endpoint,omitempty"`
+}
+
+func (in *AZBlobStorage) GetSecretName() string {
+	if in != nil {
+		return in.SecretName
+	}
+	return ""
+}
+
+func (in *AZBlobStorage) GetRoot() string {
+	if in != nil {
+		return in.Root
+	}
+	return ""
+}
+
 // PrometheusMonitorSpec defines the PodMonitor configuration.
 type PrometheusMonitorSpec struct {
 	// Enabled indicates whether the PodMonitor is enabled.

apis/v1alpha1/constants.go

@@ -93,4 +93,10 @@ const (
 
 	// ServiceAccountKey is the key for the service account in the secret.
 	ServiceAccountKey = "service-account-key"
+
+	// AccountName is the name for the account in the secret.
+	AccountName = "account-name"
+
+	// AccountKey is the key for the account in the secret.
+	AccountKey = "account-key"
 )

apis/v1alpha1/validate.go

@@ -98,6 +98,12 @@ func (in *GreptimeDBCluster) Check(ctx context.Context, client client.Client) er
 		}
 	}
 
+	if secretName := in.GetObjectStorageProvider().GetAZBlobStorage().GetSecretName(); secretName != "" {
+		if err := checkAZBlobCredentialsSecret(ctx, client, in.GetNamespace(), secretName); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -195,6 +201,12 @@ func (in *GreptimeDBStandalone) Check(ctx context.Context, client client.Client)
 		}
 	}
 
+	if secretName := in.GetObjectStorageProvider().GetAZBlobStorage().GetSecretName(); secretName != "" {
+		if err := checkAZBlobCredentialsSecret(ctx, client, in.GetNamespace(), secretName); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -282,6 +294,10 @@ func checkS3CredentialsSecret(ctx context.Context, client client.Client, namespa
 	return checkSecretData(ctx, client, namespace, name, []string{AccessKeyIDSecretKey, SecretAccessKeySecretKey})
 }
 
+func checkAZBlobCredentialsSecret(ctx context.Context, client client.Client, namespace, name string) error {
+	return checkSecretData(ctx, client, namespace, name, []string{AccountName, AccountKey})
+}
+
 // checkPodMonitorExists checks if the PodMonitor CRD exists.
 func checkPodMonitorExists(ctx context.Context, client client.Client) error {
 	const (

config/crd/resources/greptime.io_greptimedbclusters.yaml

@@ -18318,6 +18318,20 @@ spec:
                         type: integer
                       objectStorage:
                         properties:
+                          azblob:
+                            properties:
+                              container:
+                                type: string
+                              endpoint:
+                                type: string
+                              root:
+                                type: string
+                              secretName:
+                                type: string
+                            required:
+                            - container
+                            - root
+                            type: object
                           cache:
                             properties:
                               cacheCapacity:
@@ -18528,6 +18542,20 @@ spec:
                 type: integer
               objectStorage:
                 properties:
+                  azblob:
+                    properties:
+                      container:
+                        type: string
+                      endpoint:
+                        type: string
+                      root:
+                        type: string
+                      secretName:
+                        type: string
+                    required:
+                    - container
+                    - root
+                    type: object
                   cache:
                     properties:
                       cacheCapacity:

config/crd/resources/greptime.io_greptimedbstandalones.yaml

@@ -3082,6 +3082,20 @@ spec:
                 type: integer
               objectStorage:
                 properties:
+                  azblob:
+                    properties:
+                      container:
+                        type: string
+                      endpoint:
+                        type: string
+                      root:
+                        type: string
+                      secretName:
+                        type: string
+                    required:
+                    - container
+                    - root
+                    type: object
                   cache:
                     properties:
                       cacheCapacity:

docs/api-references/docs.md

@@ -15,6 +15,25 @@
 
 
 
+#### AZBlobStorage
+
+
+
+AZBlobStorage defines the Azure Blob storage specification.
+
+
+
+_Appears in:_
+- [ObjectStorageProviderSpec](#objectstorageproviderspec)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `container` _string_ | The data will be stored in the container. |  |  |
+| `secretName` _string_ | The secret of storing the credentials of account name and account key.<br />The secret should contain keys named `account-name` and `account-key`.<br />The secret must be the same namespace with the GreptimeDBCluster resource. |  |  |
+| `root` _string_ | The Blob directory path. |  |  |
+| `endpoint` _string_ | The Blob Storage endpoint. |  |  |
+
+
 #### CacheStorage
 
 
@@ -691,9 +710,10 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `s3` _[S3Storage](#s3storage)_ | S3 is the S3 storage configuration. |  |  |
+| `s3` _[S3Storage](#s3storage)_ | S3 is the AWS S3 storage configuration. |  |  |
 | `oss` _[OSSStorage](#ossstorage)_ | OSS is the Aliyun OSS storage configuration. |  |  |
-| `gcs` _[GCSStorage](#gcsstorage)_ | GCS is the Google GCS storage configuration. |  |  |
+| `gcs` _[GCSStorage](#gcsstorage)_ | GCS is the Google cloud storage configuration. |  |  |
+| `azblob` _[AZBlobStorage](#azblobstorage)_ | AZBlob is the Azure Blob storage configuration. |  |  |
 | `cache` _[CacheStorage](#cachestorage)_ | Cache is the cache storage configuration for object storage. |  |  |
 
 

examples/README.md

@@ -8,6 +8,7 @@ The following examples suppose that you have installed the etcd cluster in the `
 - [S3](./cluster/s3/cluster.yaml): Create a GreptimeDB cluster with S3 storage.
 - [GCS](./cluster/gcs/cluster.yaml): Create a GreptimeDB cluster with Google GCS storage.
 - [OSS](./cluster/oss/cluster.yaml): Create a GreptimeDB cluster with Aliyun OSS storage.
+- [AZBlob](./cluster/azblob/cluster.yaml): Create a GreptimeDB cluster with Azure Blob storage.
 - [Flownode](./cluster/flownode/cluster.yaml): Create a GreptimeDB cluster with `flownode` enabled. By adding the `flownode` configuration, you can use [continuous aggregation](https://docs.greptime.com/user-guide/continuous-aggregation/overview) in the GreptimeDB cluster.
 - [TLS Service](./cluster/tls-service/cluster.yaml): Create a GreptimeDB cluster with TLS service.
 - [Prometheus Monitoring](./cluster/prometheus-monitor/cluster.yaml): Create a GreptimeDB cluster with Prometheus monitoring. Please ensure you have already installed prometheus-operator and created a Prometheus instance with the label `release=prometheus`.
@@ -24,3 +25,4 @@ The following examples suppose that you have installed the etcd cluster in the `
 - [S3](./standalone/s3/standalone.yaml): Create a GreptimeDB standalone with S3 storage.
 - [GCS](./standalone/gcs/standalone.yaml): Create a GreptimeDB standalone with Google GCS storage.
 - [OSS](./standalone/oss/standalone.yaml): Create a GreptimeDB standalone with Aliyun OSS storage.
+- [AZBlob](./standalone/azblob/standalone.yaml): Create a GreptimeDB standalone with Azure Blob storage.

examples/cluster/azblob/azblob-credentials.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: azblob-credentials
+stringData:
+  account-name: "your-blob-account-name"
+  account-key: "your-blob-account-key"

examples/cluster/azblob/cluster.yaml

@@ -0,0 +1,22 @@
+apiVersion: greptime.io/v1alpha1
+kind: GreptimeDBCluster
+metadata:
+  name: cluster-with-blob
+spec:
+  base:
+    main:
+      image: greptime/greptimedb:latest
+  frontend:
+    replicas: 1
+  meta:
+    replicas: 1
+    etcdEndpoints:
+      - "etcd.etcd-cluster.svc.cluster.local:2379"
+  datanode:
+    replicas: 1
+  objectStorage:
+    azblob:
+      container: "greptimedb"
+      secretName: "azblob-credentials"
+      endpoint: "https://<storage-account>.blob.core.windows.net"
+      root: "cluster-with-blob-data"

examples/cluster/oss/oss-credentials.yaml

@@ -3,6 +3,6 @@ kind: Secret
 type: Opaque
 metadata:
   name: oss-credentials
-data:
+stringData:
   access-key-id: "your-access-key-id"
   secret-access-key: "your-secret-access-key"

examples/cluster/s3/s3-credentials.yaml

@@ -3,6 +3,6 @@ kind: Secret
 type: Opaque
 metadata:
   name: s3-credentials
-data:
-  access-key-id: "your-access-key-id-in-base64"
-  secret-access-key: "your-secret-access-key-in-base64"
+stringData:
+  access-key-id: "your-access-key-id"
+  secret-access-key: "your-secret-access-key"

examples/standalone/azblob/azblob-credentials.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: azblob-credentials
+stringData:
+  account-name: "your-blob-account-name"
+  account-key: "your-blob-account-key"

examples/standalone/azblob/standalone.yaml

@@ -0,0 +1,14 @@
+apiVersion: greptime.io/v1alpha1
+kind: GreptimeDBStandalone
+metadata:
+  name: standalone-with-blob
+spec:
+  base:
+    main:
+      image: greptime/greptimedb:latest
+  objectStorage:
+    azblob:
+      container: "greptimedb"
+      secretName: "azblob-credentials"
+      endpoint: "https://<storage-account>.blob.core.windows.net"
+      root: "standalone-with-blob"

examples/standalone/oss/oss-credentials.yaml

@@ -3,6 +3,6 @@ kind: Secret
 type: Opaque
 metadata:
   name: oss-credentials
-data:
+stringData:
   access-key-id: "your-access-key-id-in-base64"
   secret-access-key: "your-secret-access-key-in-base64"

examples/standalone/s3/s3-credentials.yaml

@@ -3,6 +3,6 @@ kind: Secret
 type: Opaque
 metadata:
   name: s3-credentials
-data:
-  access-key-id: "your-access-key-id-in-base64"
-  secret-access-key: "your-secret-access-key-in-base64"
+stringData:
+  access-key-id: "your-access-key-id"
+  secret-access-key: "your-secret-access-key"