From aa34a066f98b1e8492d641dcf7e85f4cd218e0ba Mon Sep 17 00:00:00 2001 From: MdSahil-oss Date: Tue, 31 Jan 2023 19:04:23 +0530 Subject: [PATCH 01/14] Added postgres-operator Signed-off-by: MdSahil-oss --- .../addons/postgres-operator/README.md | 3 ++ .../addons/postgres-operator/metadata.yaml | 10 ++++ .../addons/postgres-operator/parameter.cue | 11 ++++ .../addons/postgres-operator/template.cue | 52 +++++++++++++++++++ 4 files changed, 76 insertions(+) create mode 100644 experimental/addons/postgres-operator/README.md create mode 100644 experimental/addons/postgres-operator/metadata.yaml create mode 100644 experimental/addons/postgres-operator/parameter.cue create mode 100644 experimental/addons/postgres-operator/template.cue diff --git a/experimental/addons/postgres-operator/README.md b/experimental/addons/postgres-operator/README.md new file mode 100644 index 00000000..5eedb52f --- /dev/null +++ b/experimental/addons/postgres-operator/README.md @@ -0,0 +1,3 @@ +# postgres-operator + +This is an addon template. Check how to build your own addon: https://kubevela.net/docs/platform-engineers/addon/intro diff --git a/experimental/addons/postgres-operator/metadata.yaml b/experimental/addons/postgres-operator/metadata.yaml new file mode 100644 index 00000000..f90114ae --- /dev/null +++ b/experimental/addons/postgres-operator/metadata.yaml @@ -0,0 +1,10 @@ +name: postgres-operator +version: 1.0.0 +description: This addon adds postgres-operator to the kubevela to be used by across the application installed in kubevela. +icon: "" +url: "https://www.postgresql.org/" + +tags: +- postgres-operator + +invisible: false diff --git a/experimental/addons/postgres-operator/parameter.cue b/experimental/addons/postgres-operator/parameter.cue new file mode 100644 index 00000000..9ff20518 --- /dev/null +++ b/experimental/addons/postgres-operator/parameter.cue @@ -0,0 +1,11 @@ +// parameter.cue is used to store addon parameters. +// +// You can use these parameters in template.cue or in resources/ by 'parameter.myparam' +// +// For example, you can use parameters to allow the user to customize +// container images, ports, and etc. +parameter: { + // +usage=Custom parameter description + namespace: *"postgres-operator" | string + clusters?: [...string] +} diff --git a/experimental/addons/postgres-operator/template.cue b/experimental/addons/postgres-operator/template.cue new file mode 100644 index 00000000..491f0a0f --- /dev/null +++ b/experimental/addons/postgres-operator/template.cue @@ -0,0 +1,52 @@ +package main + +output: { + apiVersion: "core.oam.dev/v1beta1" + kind: "Application" + spec: { + components: [ + { + name: "ns-postgres-operator" + type: "k8s-objects" + properties: objects: [{ + kind: "Namespace" + apiVersion: "v1" + metadata: + name: "postgres-operator" + }] + }, + { + name: "postgres-operator" + type: "helm" + properties: { + repoType: "helm" + url: "https://opensource.zalando.com/postgres-operator/charts/postgres-operator" + chart: "postgres-operator-charts/postgres-operator" + version: "1.8.2" + } + }, + ] + policies: [ + { + type: "shared-resource" + name: "postgres-operator-ns" + properties: rules: [{ + selector: resourceTypes: ["Namespace"] + }] + }, + { + type: "topology" + name: "deploy-postgres-operator" + properties: { + namespace: parameter.namespace + if parameter.clusters != _|_ { + clusters: parameter.clusters + } + if parameter.clusters == _|_ { + clusterLabelSelector: {} + } + } + }, + ] + } +} From a25efc538885bfb7a24258b5b396dc7958451f0c Mon Sep 17 00:00:00 2001 From: MdSahil-oss Date: Fri, 24 Feb 2023 20:07:11 +0530 Subject: [PATCH 02/14] Removed unwanted changes Signed-off-by: MdSahil-oss --- experimental/addons/postgres-operator/parameter.cue | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 experimental/addons/postgres-operator/parameter.cue diff --git a/experimental/addons/postgres-operator/parameter.cue b/experimental/addons/postgres-operator/parameter.cue new file mode 100644 index 00000000..9ff20518 --- /dev/null +++ b/experimental/addons/postgres-operator/parameter.cue @@ -0,0 +1,11 @@ +// parameter.cue is used to store addon parameters. +// +// You can use these parameters in template.cue or in resources/ by 'parameter.myparam' +// +// For example, you can use parameters to allow the user to customize +// container images, ports, and etc. +parameter: { + // +usage=Custom parameter description + namespace: *"postgres-operator" | string + clusters?: [...string] +} From a34505e9682095a89536edd8818025b424945b45 Mon Sep 17 00:00:00 2001 From: MdSahil-oss Date: Tue, 14 Mar 2023 17:58:50 +0530 Subject: [PATCH 03/14] Fixed unwanted mistake Signed-off-by: MdSahil-oss --- experimental/addons/postgres-operator/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experimental/addons/postgres-operator/metadata.yaml b/experimental/addons/postgres-operator/metadata.yaml index ad87f6fd..5cac4c1f 100644 --- a/experimental/addons/postgres-operator/metadata.yaml +++ b/experimental/addons/postgres-operator/metadata.yaml @@ -10,4 +10,4 @@ tags: invisible: false dependencies: - - name: fluxcd \ No newline at end of file + - name: fluxcd From 7bcf3e434450674440b6c5a99c79001624ab6399 Mon Sep 17 00:00:00 2001 From: MdSahil-oss Date: Tue, 14 Mar 2023 20:16:16 +0530 Subject: [PATCH 04/14] Added hive-operator Signed-off-by: MdSahil-oss --- experimental/addons/hive-operator/README.md | 3 + .../addons/hive-operator/metadata.yaml | 13 +++ .../addons/hive-operator/parameter.cue | 11 +++ .../addons/hive-operator/template.cue | 93 +++++++++++++++++++ 4 files changed, 120 insertions(+) create mode 100644 experimental/addons/hive-operator/README.md create mode 100644 experimental/addons/hive-operator/metadata.yaml create mode 100644 experimental/addons/hive-operator/parameter.cue create mode 100644 experimental/addons/hive-operator/template.cue diff --git a/experimental/addons/hive-operator/README.md b/experimental/addons/hive-operator/README.md new file mode 100644 index 00000000..cd5fabf1 --- /dev/null +++ b/experimental/addons/hive-operator/README.md @@ -0,0 +1,3 @@ +# hive-operator + +This is an addon template. Check how to build your own addon: https://kubevela.net/docs/platform-engineers/addon/intro diff --git a/experimental/addons/hive-operator/metadata.yaml b/experimental/addons/hive-operator/metadata.yaml new file mode 100644 index 00000000..98e9fd63 --- /dev/null +++ b/experimental/addons/hive-operator/metadata.yaml @@ -0,0 +1,13 @@ +name: hive-operator +version: 1.0.0 +description: This addon adds hive-operator to the kubevela to be used across the application installed in kubevela. +icon: "https://github.com/apache/hive/blob/master/docs/images/hive-logo.jpg" +url: "https://hive.apache.org/" + +tags: +- hive-operator + +invisible: false + +dependencies: + - name: fluxcd diff --git a/experimental/addons/hive-operator/parameter.cue b/experimental/addons/hive-operator/parameter.cue new file mode 100644 index 00000000..c3b8dbef --- /dev/null +++ b/experimental/addons/hive-operator/parameter.cue @@ -0,0 +1,11 @@ +// parameter.cue is used to store addon parameters. +// +// You can use these parameters in template.cue or in resources/ by 'parameter.myparam' +// +// For example, you can use parameters to allow the user to customize +// container images, ports, and etc. +parameter: { + // +usage=Custom parameter description + namespace: *"hive-operator" | string + clusters?: [...string] +} diff --git a/experimental/addons/hive-operator/template.cue b/experimental/addons/hive-operator/template.cue new file mode 100644 index 00000000..e4a54f86 --- /dev/null +++ b/experimental/addons/hive-operator/template.cue @@ -0,0 +1,93 @@ +package main +output: { + apiVersion: "core.oam.dev/v1beta1" + kind: "Application" + spec: { + components: [ + { + name: "ns-hive-operator" + type: "k8s-objects" + properties: objects: [{ + kind: "Namespace" + apiVersion: "v1" + metadata: + name: parameter.namespace + }] + }, + { + name: "minio" + type: "helm" + properties: { + repoType: "helm" + url: "https://charts.min.io/" + chart: "minio" + version: "4.0.2" + targetNamespace: "prod" + } + }, + { + name: "postgresql" + type: "helm" + properties: { + repoType: "helm" + url: "https://charts.bitnami.com/bitnami" + chart: "postgresql" + version: "12.1.5" + targetNamespace: "prod" + } + }, + { + name: "commons-operator" + type: "helm" + properties: { + repoType: "helm" + url: "https://repo.stackable.tech/repository/helm-stable/" + chart: "commons-operator" + version: "23.1.0" + } + }, + { + name: "secret-operator" + type: "helm" + properties: { + repoType: "helm" + url: "https://repo.stackable.tech/repository/helm-stable/" + chart: "secret-operator" + version: "23.1.0" + } + }, + { + name: "hive-operator" + type: "helm" + properties: { + repoType: "helm" + url: "https://repo.stackable.tech/repository/helm-stable/" + chart: "hive-operator" + version: "23.1.0" + } + }, + ] + policies: [ + { + type: "shared-resource" + name: "hive-operator-ns" + properties: rules: [{ + selector: resourceTypes: ["Namespace"] + }] + }, + { + type: "topology" + name: "deploy-hive-operator" + properties: { + namespace: parameter.namespace + if parameter.clusters != _|_ { + clusters: parameter.clusters + } + if parameter.clusters == _|_ { + clusterLabelSelector: {} + } + } + }, + ] + } +} From 03d9e9b1365d0c6437d470c9b848179bbeeccf27 Mon Sep 17 00:00:00 2001 From: MdSahil-oss Date: Wed, 15 Mar 2023 19:53:04 +0530 Subject: [PATCH 05/14] Removed dependencies installation at addon enable time Signed-off-by: MdSahil-oss --- .../addons/hive-operator/template.cue | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/experimental/addons/hive-operator/template.cue b/experimental/addons/hive-operator/template.cue index e4a54f86..4f3b5a83 100644 --- a/experimental/addons/hive-operator/template.cue +++ b/experimental/addons/hive-operator/template.cue @@ -14,28 +14,6 @@ output: { name: parameter.namespace }] }, - { - name: "minio" - type: "helm" - properties: { - repoType: "helm" - url: "https://charts.min.io/" - chart: "minio" - version: "4.0.2" - targetNamespace: "prod" - } - }, - { - name: "postgresql" - type: "helm" - properties: { - repoType: "helm" - url: "https://charts.bitnami.com/bitnami" - chart: "postgresql" - version: "12.1.5" - targetNamespace: "prod" - } - }, { name: "commons-operator" type: "helm" From 6d85bc532259d299dd4bd557f3bb5f9d28872f93 Mon Sep 17 00:00:00 2001 From: MdSahil-oss Date: Thu, 16 Mar 2023 18:01:08 +0530 Subject: [PATCH 06/14] Added definitions Signed-off-by: MdSahil-oss --- .../definitions/hive-cluster.cue | 80 +++++++++++++++++++ .../hive-operator/definitions/s3-bucket.cue | 33 ++++++++ .../definitions/s3-connection.cue | 37 +++++++++ 3 files changed, 150 insertions(+) create mode 100644 experimental/addons/hive-operator/definitions/hive-cluster.cue create mode 100644 experimental/addons/hive-operator/definitions/s3-bucket.cue create mode 100644 experimental/addons/hive-operator/definitions/s3-connection.cue diff --git a/experimental/addons/hive-operator/definitions/hive-cluster.cue b/experimental/addons/hive-operator/definitions/hive-cluster.cue new file mode 100644 index 00000000..d5ad2089 --- /dev/null +++ b/experimental/addons/hive-operator/definitions/hive-cluster.cue @@ -0,0 +1,80 @@ +"hive-cluster": { + alias: "" + annotations: {} + attributes: workload: type: "autodetects.core.oam.dev" + description: "s3 bucket component" + labels: {} + type: "component" +} + +template: { + output: { + kind: "HiveCluster" + apiVersion: "hive.stackable.tech/v1alpha1" + metadata: { + name: context.name + } + spec: { + image: parameter.image + clusterConfig: parameter.clusterConfig + metastore: parameter.metastore + } + } + parameter: { + //+usage=The Hive metastore image to use. + image: { + //+usage=Overwrite the docker image. Specify the full docker image name, e.g. `docker.stackable.tech/stackable/superset:1.4.1-stackable2.1.0 + custom: *null | string + //+usage=Version of the product, e.g. `1.4.1`. + productVersion: *null | string + //+usage=Pull policy used when pulling the Images. + pullPolicy: *"IfNotPresent" | string + //+usage=Image pull secrets to pull images from a private registry. + pullSecrets: *null | [...] + //+usage=Name of the docker repo, e.g. `docker.stackable.tech/stackable. + repo: *null | string + //+usage=Stackable version of the product, e.g. 2.1.0. + stackableVersion: *null | string + } + //+usage=General Hive metastore cluster settings. + clusterConfig: { + //+usage=Database connection specification. + database: { + connString: *null | string + dbType: *null | string + password: *null | string + user: *null | string + } + //+usage=HDFS connection specification. + hdfs: { + //+usage=Name of the discovery-configmap providing information about the HDFS cluster. + configMap: *null | string + } + //+usage=S3 connection specification. + s3: { + //+usage=S3 connection definition as CRD. + inline: *null | {...} + reference: *null | string + } + //+usage=Specify the type of the created kubernetes service. This attribute will be removed in a future release when listener-operator is finished. Use with caution. + serviceType: *null | string + //+usage=Name of the Vector aggregator discovery ConfigMap. It must contain the key `ADDRESS` with the address of the Vector aggregator. + vectorAggregatorConfigMapName: *null | string + } + //+usage=Configure metastore. + metastore: { + //+usage=Name of the discovery-configmap providing information about the HDFS cluster. + cliOverrides: *{} | {...} + //+usage=Name of the discovery-configmap providing information about the HDFS cluster. + config: *{} | {...} + //+usage=Name of the discovery-configmap providing information about the HDFS cluster. + configOverrides: *{} | {...} + //+usage=Name of the discovery-configmap providing information about the HDFS cluster. + envOverrides: *{} | {...} + //+usage=Name of the discovery-configmap providing information about the HDFS cluster. + roleGroups: *{} | {...} + } + //+usage=Emergency stop button, if `true` then all pods are stopped without affecting configuration (as setting `replicas` to `0` would. + stopped: *null | bool + } +} diff --git a/experimental/addons/hive-operator/definitions/s3-bucket.cue b/experimental/addons/hive-operator/definitions/s3-bucket.cue new file mode 100644 index 00000000..cb1d3c4d --- /dev/null +++ b/experimental/addons/hive-operator/definitions/s3-bucket.cue @@ -0,0 +1,33 @@ +"s3-bucket": { + alias: "" + annotations: {} + attributes: workload: type: "autodetects.core.oam.dev" + description: "s3 bucket component" + labels: {} + type: "component" +} + +template: { + output: { + kind: "S3Bucket" + apiVersion: "s3.stackable.tech/v1alpha1" + metadata: { + name: context.name + } + spec: { + bucketName: parameter.bucketName + connection: parameter.connection + } + } + parameter: { + //+usage=the name of the Bucket. + bucketName: *null | string + //+usage=can either be inline or reference. + connection: { + //+usage=the name of the Bucket. + inline: *null | {...} + //+usage=the name of the referenced S3Connection resource, which must be in the same namespace as the S3Bucket resource. + reference: *null | string + } + } +} diff --git a/experimental/addons/hive-operator/definitions/s3-connection.cue b/experimental/addons/hive-operator/definitions/s3-connection.cue new file mode 100644 index 00000000..b5a8a339 --- /dev/null +++ b/experimental/addons/hive-operator/definitions/s3-connection.cue @@ -0,0 +1,37 @@ +"s3-connection": { + alias: "" + annotations: {} + attributes: workload: type: "autodetects.core.oam.dev" + description: "s3 connection component" + labels: {} + type: "component" +} + +template: { + output: { + kind: "S3Connection" + apiVersion: "s3.stackable.tech/v1alpha1" + metadata: { + name: context.name + } + spec: { + host: parameter.host + port: parameter.port + accessStyle: parameter.accessStyle + credentials: parameter.credentials + } + } + parameter: { + //+usage=the domain name of the host of the object store, such as s3.west.provider.com. + host: *null | string + //+usage=a port such as 80 or 4242. + port: *null | int + //+usage=Optional. Can be either "VirtualHosted" (default) or "Path". + accessStyle: *"VirtualHosted" | string + //+usage=contains a secretClass. + credentials: { + //+usage=a reference to a SecretClass resource + secretClass: *null | string + } + } +} From 7d6b85e59829f42e43a22f63c2a340234b49539f Mon Sep 17 00:00:00 2001 From: MdSahil-oss Date: Fri, 17 Mar 2023 13:28:12 +0530 Subject: [PATCH 07/14] corrected some errors Signed-off-by: MdSahil-oss --- .../addons/hive-operator/definitions/hive-cluster.cue | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/experimental/addons/hive-operator/definitions/hive-cluster.cue b/experimental/addons/hive-operator/definitions/hive-cluster.cue index d5ad2089..cc55632c 100644 --- a/experimental/addons/hive-operator/definitions/hive-cluster.cue +++ b/experimental/addons/hive-operator/definitions/hive-cluster.cue @@ -46,10 +46,7 @@ template: { user: *null | string } //+usage=HDFS connection specification. - hdfs: { - //+usage=Name of the discovery-configmap providing information about the HDFS cluster. - configMap: *null | string - } + hdfs: *null | {...} //+usage=S3 connection specification. s3: { //+usage=S3 connection definition as CRD. @@ -57,7 +54,7 @@ template: { reference: *null | string } //+usage=Specify the type of the created kubernetes service. This attribute will be removed in a future release when listener-operator is finished. Use with caution. - serviceType: *null | string + serviceType: *"ClusterIP" | string //+usage=Name of the Vector aggregator discovery ConfigMap. It must contain the key `ADDRESS` with the address of the Vector aggregator. vectorAggregatorConfigMapName: *null | string } From adb4b66c8b4e8251f936683f14aeece2e11572da Mon Sep 17 00:00:00 2001 From: MdSahil-oss Date: Fri, 17 Mar 2023 17:00:48 +0530 Subject: [PATCH 08/14] Added more definitions Signed-off-by: MdSahil-oss --- .../definitions/secret-class.cue | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 experimental/addons/hive-operator/definitions/secret-class.cue diff --git a/experimental/addons/hive-operator/definitions/secret-class.cue b/experimental/addons/hive-operator/definitions/secret-class.cue new file mode 100644 index 00000000..fb672be6 --- /dev/null +++ b/experimental/addons/hive-operator/definitions/secret-class.cue @@ -0,0 +1,30 @@ +"secret-class": { + alias: "" + annotations: {} + attributes: workload: type: "autodetects.core.oam.dev" + description: "secret class component" + labels: {} + type: "component" +} + +template: { + output: { + kind: "SecretClass" + apiVersion: "secrets.stackable.tech/v1alpha1" + metadata: { + name: context.name + } + spec: { + backend: parameter.backend + } + } + parameter: { + //+usage=Configure backend. + backend: { + //+usage=Configure k8sSearch. + k8sSearch: *null | {...} + //+usage=Configure autoTls. + autoTls: *null | {...} + } + } +} From 91e4d4c67be4cddb39c51f0ef43d8a0e5bc5e80f Mon Sep 17 00:00:00 2001 From: MdSahil-oss Date: Sat, 18 Mar 2023 14:02:47 +0530 Subject: [PATCH 09/14] Added Readme Signed-off-by: MdSahil-oss --- experimental/addons/hive-operator/README.md | 167 +++++++++++++++++++- 1 file changed, 166 insertions(+), 1 deletion(-) diff --git a/experimental/addons/hive-operator/README.md b/experimental/addons/hive-operator/README.md index cd5fabf1..f59f7082 100644 --- a/experimental/addons/hive-operator/README.md +++ b/experimental/addons/hive-operator/README.md @@ -1,3 +1,168 @@ # hive-operator -This is an addon template. Check how to build your own addon: https://kubevela.net/docs/platform-engineers/addon/intro +Apache Hive is a distributed, fault-tolerant data warehouse system that enables analytics at a massive scale. Hive Metastore(HMS) provides a central repository of metadata that can easily be analyzed to make informed, data driven decisions, and therefore it is a critical component of many data lake architectures. Hive is built on top of Apache Hadoop and supports storage on S3, adls, gs etc though hdfs. Hive allows users to read, write, and manage petabytes of data using SQL. + +This is an operator for Kubernetes that can manage Apache Hive. Currently, it only supports the Hive Metastore! + +## Install Operator + +Add experimental registry +``` +vela addon registry add experimental --type=helm --endpoint=https://addons.kubevela.net/experimental/ +``` + +Enable this addon +``` +vela addon enable hive-operator +``` + +```shell +$ vela ls -A | grep hive +vela-system addon-hive-operator ns-hive-operator k8s-objects running healthy +vela-system └─ hive-operator helm running healthy Fetch repository successfully, Create helm release +``` + +Disable this addon +``` +vela addon disable hive-operator +``` + +## Install Dependencies + +In order to install the MinIO and PostgreSQL dependencies via Helm, you have to deploy two charts. + +**Minio** + +```shell +helm install minio \ +--namespace prod \ +--version 4.0.2 \ +--set mode=standalone \ +--set replicas=1 \ +--set persistence.enabled=false \ +--set buckets[0].name=hive,buckets[0].policy=none \ +--set users[0].accessKey=hive,users[0].secretKey=hivehive,users[0].policy=readwrite \ +--set resources.requests.memory=1Gi \ +--set service.type=NodePort,service.nodePort=null \ +--set consoleService.type=NodePort,consoleService.nodePort=null \ +--repo https://charts.min.io/ minio +``` + +**PostgresSQL** + +```shell +helm install postgresql \ +--version=12.1.5 \ +--namespace prod \ +--set postgresqlUsername=hive \ +--set postgresqlPassword=hive \ +--set postgresqlDatabase=hive \ +--repo https://charts.bitnami.com/bitnami postgresql +``` + +## Use + +After going through the Installation section and having installed all the dependencies, you will now deploy a Hive metastore cluster and it’s dependencies. Afterwards you can verify that it works. + +In order to connect Hive to MinIO we need to create several dependent components like S3Connection, Secret and a SecretClass + +**s3-connection** + +An S3Connection to connect to MinIO: + +```yaml +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: s3-connection-sample +spec: + components: + - type: s3-connection + name: minio + properties: + host: minio + port: 9000 + accessStyle: Path + credentials: + secretClass: hive-s3-secret-class +``` + +**secret** + +Credentials for the S3Connection to log into MinIO: + +```yaml +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: hive-secret-sample +spec: + components: + - type: k8s-objects + name: k8s-demo-secret + properties: + objects: + - apiVersion: v1 + kind: Secret + metadata: + name: hive-s3-secret + labels: + secrets.stackable.tech/class: hive-s3-secret-class + stringData: + accessKey: hive + secretKey: hivehive +``` + +**secret-class** + +A SecretClass for the credentials to the Minio. The credentials were defined in the installation of Minio via helm: + +```yaml +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: hive-s3-secret-class +spec: + backend: + k8sSearch: + searchNamespace: + pod: {} +``` + +**hive-cluster** + +And lastly the actual Apache Hive cluster definition: + +```yaml +--- +apiVersion: hive.stackable.tech/v1alpha1 +kind: HiveCluster +metadata: + name: hive-postgres-s3 +spec: + image: + productVersion: 3.1.3 + stackableVersion: 23.1.0 + clusterConfig: + database: + connString: jdbc:postgresql://postgresql:5432/hive + user: hive + password: hive + dbType: postgres + s3: + reference: minio + metastore: + roleGroups: + default: + replicas: 1 +``` + +Verify that it works + +```shell +$ kubectl get statefulset -n prod +NAME READY AGE +hive-postgres-cluster-metastore-default 1/1 76s +``` + +For more visit on the website https://docs.stackable.tech/home/stable/hive/index.html. From 86e0ecd8f384b31db94d5532b6846139696c330c Mon Sep 17 00:00:00 2001 From: MdSahil-oss Date: Sat, 18 Mar 2023 14:07:57 +0530 Subject: [PATCH 10/14] Corrected mistake Signed-off-by: MdSahil-oss --- experimental/addons/hive-operator/README.md | 60 ++++++++++++--------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/experimental/addons/hive-operator/README.md b/experimental/addons/hive-operator/README.md index f59f7082..2039def4 100644 --- a/experimental/addons/hive-operator/README.md +++ b/experimental/addons/hive-operator/README.md @@ -118,15 +118,19 @@ spec: A SecretClass for the credentials to the Minio. The credentials were defined in the installation of Minio via helm: ```yaml -apiVersion: secrets.stackable.tech/v1alpha1 -kind: SecretClass +apiVersion: core.oam.dev/v1beta1 +kind: Application metadata: - name: hive-s3-secret-class + name: secret-class-sample spec: - backend: - k8sSearch: - searchNamespace: - pod: {} + components: + - type: secret-class + name: hive-s3-secret-class + properties: + backend: + k8sSearch: + searchNamespace: + pod: {} ``` **hive-cluster** @@ -134,27 +138,31 @@ spec: And lastly the actual Apache Hive cluster definition: ```yaml ---- -apiVersion: hive.stackable.tech/v1alpha1 -kind: HiveCluster +apiVersion: core.oam.dev/v1beta1 +kind: Application metadata: - name: hive-postgres-s3 + name: hive-postgres-cluster spec: - image: - productVersion: 3.1.3 - stackableVersion: 23.1.0 - clusterConfig: - database: - connString: jdbc:postgresql://postgresql:5432/hive - user: hive - password: hive - dbType: postgres - s3: - reference: minio - metastore: - roleGroups: - default: - replicas: 1 + components: + - type: hive-cluster + name: hive-postgres-cluster + properties: + image: + productVersion: 3.1.3 + stackableVersion: 23.1.0 + clusterConfig: + database: + connString: jdbc:postgresql://postgresql:5432/hive + user: hive + password: hive + dbType: postgres + s3: + reference: minio + metastore: + roleGroups: + default: + replicas: 1 + ``` Verify that it works From 0b3aa6cf9e42a215bb94f18a237a21efc2d4666f Mon Sep 17 00:00:00 2001 From: MdSahil-oss Date: Sat, 18 Mar 2023 14:18:56 +0530 Subject: [PATCH 11/14] Updated Readme Signed-off-by: MdSahil-oss --- experimental/addons/hive-operator/README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/experimental/addons/hive-operator/README.md b/experimental/addons/hive-operator/README.md index 2039def4..e467215b 100644 --- a/experimental/addons/hive-operator/README.md +++ b/experimental/addons/hive-operator/README.md @@ -68,7 +68,7 @@ In order to connect Hive to MinIO we need to create several dependent components **s3-connection** -An S3Connection to connect to MinIO: +An S3Connection to connect to MinIO, Apply below YAML: ```yaml apiVersion: core.oam.dev/v1beta1 @@ -89,7 +89,7 @@ spec: **secret** -Credentials for the S3Connection to log into MinIO: +Credentials for the S3Connection to log into MinIO, Apply below YAML: ```yaml apiVersion: core.oam.dev/v1beta1 @@ -115,7 +115,7 @@ spec: **secret-class** -A SecretClass for the credentials to the Minio. The credentials were defined in the installation of Minio via helm: +A SecretClass for the credentials to the Minio. The credentials were defined in the installation of Minio via helm, Apply below YAML: ```yaml apiVersion: core.oam.dev/v1beta1 @@ -135,7 +135,7 @@ spec: **hive-cluster** -And lastly the actual Apache Hive cluster definition: +And lastly the actual Apache Hive cluster definition, Apply below YAML: ```yaml apiVersion: core.oam.dev/v1beta1 @@ -162,7 +162,6 @@ spec: roleGroups: default: replicas: 1 - ``` Verify that it works From 1a2ab347488ba08be442a36bc554cf66569433d8 Mon Sep 17 00:00:00 2001 From: MdSahil-oss Date: Sun, 19 Mar 2023 00:13:17 +0530 Subject: [PATCH 12/14] Added examples Signed-off-by: MdSahil-oss --- examples/hive-operator/hive-cluster.yaml | 24 +++++++++++++++++++++++ examples/hive-operator/s3-connection.yaml | 14 +++++++++++++ examples/hive-operator/secret-class.yaml | 13 ++++++++++++ 3 files changed, 51 insertions(+) create mode 100644 examples/hive-operator/hive-cluster.yaml create mode 100644 examples/hive-operator/s3-connection.yaml create mode 100644 examples/hive-operator/secret-class.yaml diff --git a/examples/hive-operator/hive-cluster.yaml b/examples/hive-operator/hive-cluster.yaml new file mode 100644 index 00000000..be09d650 --- /dev/null +++ b/examples/hive-operator/hive-cluster.yaml @@ -0,0 +1,24 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: hive-postgres-cluster +spec: + components: + - type: hive-cluster + name: hive-postgres-cluster + properties: + image: + productVersion: 3.1.3 + stackableVersion: 23.1.0 + clusterConfig: + database: + connString: jdbc:postgresql://postgresql:5432/hive + user: hive + password: hive + dbType: postgres + s3: + reference: minio + metastore: + roleGroups: + default: + replicas: 1 diff --git a/examples/hive-operator/s3-connection.yaml b/examples/hive-operator/s3-connection.yaml new file mode 100644 index 00000000..0940d4fd --- /dev/null +++ b/examples/hive-operator/s3-connection.yaml @@ -0,0 +1,14 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: s3-connection-sample +spec: + components: + - type: s3-connection + name: minio + properties: + host: minio + port: 9000 + accessStyle: Path + credentials: + secretClass: hive-s3-secret-class diff --git a/examples/hive-operator/secret-class.yaml b/examples/hive-operator/secret-class.yaml new file mode 100644 index 00000000..d9522a8c --- /dev/null +++ b/examples/hive-operator/secret-class.yaml @@ -0,0 +1,13 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: secret-class-sample +spec: + components: + - type: secret-class + name: hive-s3-secret-class + properties: + backend: + k8sSearch: + searchNamespace: + pod: {} From fbd4ae583e213c472b3a0a5245a3b53b160d7334 Mon Sep 17 00:00:00 2001 From: MdSahil-oss Date: Mon, 20 Mar 2023 13:38:01 +0530 Subject: [PATCH 13/14] Added dependencies to be installed at addon installation time. Signed-off-by: MdSahil-oss --- .../addons/hive-operator/template.cue | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/experimental/addons/hive-operator/template.cue b/experimental/addons/hive-operator/template.cue index 4f3b5a83..f2a25402 100644 --- a/experimental/addons/hive-operator/template.cue +++ b/experimental/addons/hive-operator/template.cue @@ -14,6 +14,66 @@ output: { name: parameter.namespace }] }, + { + name: "minio" + type: "helm" + properties: { + repoType: "helm" + url: "https://charts.min.io/" + chart: "minio" + version: "4.0.2" + targetNamespace: "prod" + values: { + mode: "standalone", + replicas: 1, + persistence: { + enabled: false + } + buckets: [ + { + name: "hive", + policy: "none" + } + ] + users: [ + { + accessKey: "hive", + secretKey: "hivehive", + policy: "readwrite" + } + ] + resources: { + requests: { + memory: "1Gi" + } + } + service: { + type: "NodePort", + nodePort: null + } + consoleService: { + type: "NodePort", + nodePort: null + } + } + } + }, + { + name: "postgresql" + type: "helm" + properties: { + repoType: "helm" + url: "https://charts.bitnami.com/bitnami" + chart: "postgresql" + version: "12.1.5" + targetNamespace: "prod" + values: { + postgresqlUsername: "hive", + postgresqlPassword: "hive", + postgresqlDatabase: "hive" + } + } + }, { name: "commons-operator" type: "helm" From b85ac8e057c6c7017fcbd36828d52e0b4a7885c7 Mon Sep 17 00:00:00 2001 From: MdSahil-oss Date: Mon, 20 Mar 2023 13:51:27 +0530 Subject: [PATCH 14/14] Updated Readme Signed-off-by: MdSahil-oss --- experimental/addons/hive-operator/README.md | 33 --------------------- 1 file changed, 33 deletions(-) diff --git a/experimental/addons/hive-operator/README.md b/experimental/addons/hive-operator/README.md index e467215b..46100b7d 100644 --- a/experimental/addons/hive-operator/README.md +++ b/experimental/addons/hive-operator/README.md @@ -27,39 +27,6 @@ Disable this addon vela addon disable hive-operator ``` -## Install Dependencies - -In order to install the MinIO and PostgreSQL dependencies via Helm, you have to deploy two charts. - -**Minio** - -```shell -helm install minio \ ---namespace prod \ ---version 4.0.2 \ ---set mode=standalone \ ---set replicas=1 \ ---set persistence.enabled=false \ ---set buckets[0].name=hive,buckets[0].policy=none \ ---set users[0].accessKey=hive,users[0].secretKey=hivehive,users[0].policy=readwrite \ ---set resources.requests.memory=1Gi \ ---set service.type=NodePort,service.nodePort=null \ ---set consoleService.type=NodePort,consoleService.nodePort=null \ ---repo https://charts.min.io/ minio -``` - -**PostgresSQL** - -```shell -helm install postgresql \ ---version=12.1.5 \ ---namespace prod \ ---set postgresqlUsername=hive \ ---set postgresqlPassword=hive \ ---set postgresqlDatabase=hive \ ---repo https://charts.bitnami.com/bitnami postgresql -``` - ## Use After going through the Installation section and having installed all the dependencies, you will now deploy a Hive metastore cluster and it’s dependencies. Afterwards you can verify that it works.