Merge pull request #7 from lalithkota/develop-G2P-2355

Reorganized Reporting Framework Helm charts, docker. Connectors, etc.
OpenG2P · Jun 19, 2024 · bfe16ab · bfe16ab
2 parents 1bd6afe + e4e65b3
commit bfe16ab
Show file tree

Hide file tree

Showing 114 changed files with 5,755 additions and 960 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -7,5 +7,5 @@ indent_style = space
 insert_final_newline = true
 trim_trailing_whitespace = true
 
-[*.{json,yml,yaml,rst,md}]
+[*.{yml,yaml,rst,md}]
 indent_size = 2
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
@@ -0,0 +1,78 @@
+name: Build Docker and Push
+
+on:
+  push:
+    branches:
+      - "**"
+    tags-ignore:
+      - '**'
+  workflow_dispatch:
+
+jobs:
+  docker-build:
+    name: Docker Build and Push
+    runs-on: ubuntu-latest
+    env:
+      NAMESPACE: ${{ secrets.docker_hub_organisation || 'openg2p' }}
+      OS_KAFKA_CONNECTOR_SERVICE_NAME: reporting-opensearch-kafka-connector
+      REPORTING_INIT_SERVICE_NAME: reporting-init-shell-tools
+    steps:
+      - uses: actions/checkout@v3
+      - name: Docker build OS Kafka Connector
+        run: |
+          BRANCH_NAME=$(echo ${{ github.ref }} | sed -e 's,.*/\(.*\),\1,')
+          OS_KAFKA_CONNECTOR_IMAGE_ID=$NAMESPACE/$OS_KAFKA_CONNECTOR_SERVICE_NAME
+
+          # Change all uppercase to lowercase
+          OS_KAFKA_CONNECTOR_IMAGE_ID=$(echo $OS_KAFKA_CONNECTOR_IMAGE_ID | tr '[A-Z]' '[a-z]')
+          OS_KAFKA_CONNECTOR_VERSION=$BRANCH_NAME
+          if [[ $BRANCH_NAME == master || $BRANCH_NAME == main ]]; then
+            OS_KAFKA_CONNECTOR_VERSION=develop
+          fi
+          echo OS_KAFKA_CONNECTOR_IMAGE_ID=$OS_KAFKA_CONNECTOR_IMAGE_ID
+          echo OS_KAFKA_CONNECTOR_VERSION=$OS_KAFKA_CONNECTOR_VERSION
+          echo OS_KAFKA_CONNECTOR_IMAGE_ID=$OS_KAFKA_CONNECTOR_IMAGE_ID >> $GITHUB_ENV
+          echo OS_KAFKA_CONNECTOR_VERSION=$OS_KAFKA_CONNECTOR_VERSION >> $GITHUB_ENV
+
+          cd opensearch-kafka-connector
+          docker build . \
+            --file Dockerfile \
+            --tag $OS_KAFKA_CONNECTOR_IMAGE_ID:$OS_KAFKA_CONNECTOR_VERSION
+          if [[ '${{ secrets.docker_hub_token }}' != '' && '${{ secrets.docker_hub_actor }}' != '' ]]; then
+            export OS_KAFKA_CONNECTOR_DOCKER_PUSH="true"
+            echo OS_KAFKA_CONNECTOR_DOCKER_PUSH=$OS_KAFKA_CONNECTOR_DOCKER_PUSH >> $GITHUB_ENV
+          fi
+      - name: Docker push OS Kafka Connector
+        if: env.OS_KAFKA_CONNECTOR_DOCKER_PUSH == 'true'
+        run: |
+          echo "${{ secrets.docker_hub_token }}" | docker login -u ${{ secrets.docker_hub_actor }} --password-stdin
+          docker push ${{ env.OS_KAFKA_CONNECTOR_IMAGE_ID }}:${{ env.OS_KAFKA_CONNECTOR_VERSION }}
+      - name: Docker build Reporting Init
+        run: |
+          BRANCH_NAME=$(echo ${{ github.ref }} | sed -e 's,.*/\(.*\),\1,')
+          REPORTING_INIT_IMAGE_ID=$NAMESPACE/$REPORTING_INIT_SERVICE_NAME
+
+          # Change all uppercase to lowercase
+          REPORTING_INIT_IMAGE_ID=$(echo $REPORTING_INIT_IMAGE_ID | tr '[A-Z]' '[a-z]')
+          REPORTING_INIT_VERSION=$BRANCH_NAME
+          if [[ $BRANCH_NAME == master || $BRANCH_NAME == main ]]; then
+            REPORTING_INIT_VERSION=develop
+          fi
+          echo REPORTING_INIT_IMAGE_ID=$REPORTING_INIT_IMAGE_ID
+          echo REPORTING_INIT_VERSION=$REPORTING_INIT_VERSION
+          echo REPORTING_INIT_IMAGE_ID=$REPORTING_INIT_IMAGE_ID >> $GITHUB_ENV
+          echo REPORTING_INIT_VERSION=$REPORTING_INIT_VERSION >> $GITHUB_ENV
+
+          cd reporting-init-docker
+          docker build . \
+            --file Dockerfile \
+            --tag $REPORTING_INIT_IMAGE_ID:$REPORTING_INIT_VERSION
+          if [[ '${{ secrets.docker_hub_token }}' != '' && '${{ secrets.docker_hub_actor }}' != '' ]]; then
+            export REPORTING_INIT_DOCKER_PUSH="true"
+            echo REPORTING_INIT_DOCKER_PUSH=$REPORTING_INIT_DOCKER_PUSH >> $GITHUB_ENV
+          fi
+      - name: Docker push Reporting Init
+        if: env.REPORTING_INIT_DOCKER_PUSH == 'true'
+        run: |
+          echo "${{ secrets.docker_hub_token }}" | docker login -u ${{ secrets.docker_hub_actor }} --password-stdin
+          docker push ${{ env.REPORTING_INIT_IMAGE_ID }}:${{ env.REPORTING_INIT_VERSION }}
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
@@ -1,28 +1,23 @@
 name: pre-commit
 
-on: push
+on:
+  pull_request:
+  push:
+    branches:
+      - "**"
+    tags-ignore:
+      - '**'
+  workflow_dispatch:
 
 jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - name: Get python version
-        run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
-      - uses: actions/cache@v1
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+      - uses: actions/setup-node@v4
         with:
-          path: ~/.cache/pre-commit
-          key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
-      - name: Install pre-commit
-        run: pip install pre-commit
-      - name: Run pre-commit
-        run: pre-commit run --all-files --show-diff-on-failure --color=always
-      - name: Check that all files generated by pre-commit are in git
-        run: |
-          newfiles="$(git ls-files --others --exclude-from=.gitignore)"
-          if [ "$newfiles" != "" ] ; then
-              echo "Please check-in the following files:"
-              echo "$newfiles"
-              exit 1
-          fi
+          node-version: 18
+      - uses: pre-commit/[email protected]
+        with:
+          extra_args: --all-files --show-diff-on-failure
diff --git a/.github/workflows/push_trigger.yml b/.github/workflows/push_trigger.yml
@@ -0,0 +1,110 @@
+name: Publish OpenG2P Helm charts on
+
+on:
+  push:
+    branches:
+      - 1.*
+      - develop
+      - main
+
+jobs:
+  generate-charts:
+    runs-on: ubuntu-latest
+    env:
+      SKIP: 'FALSE'
+      RANCHER_CHART_FILTER: "openg2p.org/add-to-rancher"
+    defaults:
+      run:
+        shell: bash
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v3
+
+    - id: files
+      uses: jitterbit/get-changed-files@v1
+
+    - name: save helm/charts to tmp.txt file
+      run: |
+        touch charts-list.txt
+        for changed_file in ${{ steps.files.outputs.all }}; do
+          if [[ ${changed_file} =~ ^[charts] ]]; then
+            chart_name=$(echo "${changed_file}" | awk -F/ '/^[charts]/{print $2}')
+            echo $chart_name >> charts-list.txt;
+            echo "Saved $chart_name chart to charts-list.txt"
+          fi
+        done
+        cat charts-list.txt | sort | uniq > charts-list-unique.txt
+        mv charts-list-unique.txt charts-list.txt
+        echo "List of charts to be published";
+        cat charts-list.txt
+
+    - name: Generate tar files
+      run: |
+        if [[ ! -s charts-list.txt ]]; then
+          echo "::warning::No Charts to publish";
+          echo "SKIP=TRUE" >> $GITHUB_ENV
+        else
+          RANCHER_CHARTS=()
+          while IFS= read -r chartpath; do
+            echo "chartpath: $chartpath"
+            chartname=$(basename "$chartpath")
+            echo "Chartname: $chartname"
+            helm dep up charts/$chartpath
+            helm package charts/$chartpath
+            is_rancher_chart=$(grep "$RANCHER_CHART_FILTER" charts/${chartpath%*/}/Chart.yaml || true)
+            if [ -n "$is_rancher_chart" ]; then
+              RANCHER_CHARTS+=("$chartname")
+            fi
+          done < charts-list.txt
+          echo "RANCHER_CHARTS=${RANCHER_CHARTS[@]}" >> $GITHUB_ENV
+          rm charts-list.txt
+        fi
+
+        shopt -s nocasematch
+        if [[ '${{ github.repository_owner }}' != 'OpenG2P' ]]; then
+          echo "SKIP=TRUE" >> $GITHUB_ENV
+        fi
+    - name: Upload tar as Artifact
+      uses: actions/upload-artifact@v2
+      with:
+          name: charts
+          path: ./*.tgz
+      if: env.SKIP != 'TRUE'
+
+    - name: Checkout branch for publishing
+      uses: actions/checkout@v3
+      with:
+          repository: 'openg2p/openg2p-helm'
+          ref: gh-pages
+          token: ${{ secrets.OPENG2P_BOT_GITHUB_PAT }}
+      if: env.SKIP != 'TRUE'
+
+    - name: Download tar from Artifacts
+      uses: actions/download-artifact@v2
+      with:
+          name: charts
+          path: ./
+      if: env.SKIP != 'TRUE'
+
+    - name: Update index.yaml
+      run: |
+        helm repo index --url https://openg2p.github.io/openg2p-helm/ .
+        for chartname in $RANCHER_CHARTS; do
+          cp ${chartname}*.tgz rancher/
+        done
+        helm repo index --url https://openg2p.github.io/openg2p-helm/ --merge rancher/index.yaml rancher
+        for chartname in $RANCHER_CHARTS; do
+          rm rancher/${chartname}*.tgz
+        done
+      if: env.SKIP != 'TRUE'
+
+    - name: Commit Changes to repository
+      uses: EndBug/add-and-commit@v7
+      with:
+        branch: gh-pages
+        author_name: openg2pbot
+        author_email: [email protected]
+        default_author: user_info
+        message: 'added social registry helm charts for publish openg2p/openg2p-reporting@${{ github.sha }}'
+        add: './*.tgz ./index.yaml rancher/index.yaml'
+      if: env.SKIP != 'TRUE'
diff --git a/.gitignore b/.gitignore
@@ -72,3 +72,9 @@ docs/_build/
 
 # OCA rules
 !static/lib/
+
+# JAVA
+target
+
+# IDEA
+.idea
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,15 +1,4 @@
 repos:
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.4.1
-    hooks:
-      - id: prettier
-        name: prettier (with plugin-xml)
-        additional_dependencies:
-          - "[email protected]"
-          - "@prettier/[email protected]"
-        args:
-          - --plugin=@prettier/plugin-xml
-        files: \.(css|htm|html|js|json|jsx|less|md|scss|toml|ts|xml|yaml|yml)$
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
     hooks:

diff --git a/.prettierrc.yml b/.prettierrc.yml
diff --git a/README.md b/README.md
@@ -1,6 +1,5 @@
 # OpenG2P Reporting
-This repository contains components for OpenG2P Reporting system. Refer to [OpenG2P Docs](https://docs.openg2p.org/v/1.1) for details of the reporting framework and installation of the same.
+This repository contains components for OpenG2P Reporting system. Refer to [OpenG2P Docs](https://docs.openg2p.org) for details of the reporting framework and installation of the same.
 
 # License
 This repository is licensed under [MPL-2.0](LICENSE).
-
diff --git a/charts/debezium/.gitignore b/charts/debezium/.gitignore
@@ -0,0 +1,2 @@
+charts/
+Chart.lock
diff --git a/charts/debezium/.helmignore b/charts/debezium/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/charts/debezium/Chart.yaml b/charts/debezium/Chart.yaml
@@ -0,0 +1,15 @@
+apiVersion: v2
+name: debezium
+description: Debezium Helm chart
+type: application
+version: 2.6.0
+dependencies:
+- name: common
+  repository: oci://registry-1.docker.io/bitnamicharts
+  version: 2.x.x
+maintainers:
+  - email: [email protected]
+    name: OpenG2P
+icon: https://openg2p.github.io/openg2p-helm/openg2p-logo.png
+annotations:
+  catalog.cattle.io/display-name: "Debezium"
diff --git a/charts/debezium/README.md b/charts/debezium/README.md
@@ -0,0 +1,3 @@
+# Debezium
+
+Helm chart for installing Debezium.
diff --git a/charts/debezium/app-readme.md b/charts/debezium/app-readme.md
diff --git a/charts/debezium/questions.yaml b/charts/debezium/questions.yaml
@@ -0,0 +1 @@
+questions: []
diff --git a/charts/debezium/templates/_helpers.tpl b/charts/debezium/templates/_helpers.tpl
@@ -0,0 +1,33 @@
+{{/*
+Return podAnnotations
+*/}}
+{{- define "debezium.podAnnotations" -}}
+{{- if .Values.podAnnotations }}
+{{ include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) }}
+{{- end }}
+{{- if and .Values.metrics.enabled .Values.metrics.podAnnotations }}
+{{ include "common.tplvalues.render" (dict "value" .Values.metrics.podAnnotations "context" $) }}
+{{- end }}
+{{- end -}}
+
+{{/*
+Render Env values section
+*/}}
+{{- define "debezium.baseEnvVars" -}}
+{{- $context := .context -}}
+{{- range $k, $v := .envVars }}
+- name: {{ $k }}
+{{- if or (kindIs "int64" $v) (kindIs "float64" $v) (kindIs "bool" $v) }}
+  value: {{ $v | quote }}
+{{- else if kindIs "string" $v }}
+  value: {{ include "common.tplvalues.render" ( dict "value" $v "context" $context ) | squote }}
+{{- else }}
+  valueFrom: {{- include "common.tplvalues.render" ( dict "value" $v "context" $context ) | nindent 4}}
+{{- end }}
+{{- end }}
+{{- end -}}
+
+{{- define "debezium.envVars" -}}
+{{- $envVars := merge (deepCopy .Values.envVars) (deepCopy .Values.envVarsFrom) (.Values.metrics.enabled | ternary .Values.metrics.envVars dict) -}}
+{{- include "debezium.baseEnvVars" (dict "envVars" $envVars "context" $) }}
+{{- end -}}
diff --git a/charts/debezium/templates/clusterrolebinding.yaml b/charts/debezium/templates/clusterrolebinding.yaml
@@ -0,0 +1,19 @@
+kind: ClusterRoleBinding
+apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }}
+metadata:
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  name: {{ template "common.names.fullname" . }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ template "common.names.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ default (include "common.names.fullname" .) .Values.serviceAccount.name }}
+    namespace: {{ .Release.Namespace }}
-Original file line number
+Diff line change
@@ Expand Up / @@ -72,3 +72,9 @@ docs/_build/ @@
     # OCA rules
     !static/lib/
+    # JAVA
+    target
+    # IDEA
+    .idea
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# Debezium

		Helm chart for installing Debezium.