diff --git a/Makefile b/Makefile index 4462536cd4..b07151a280 100644 --- a/Makefile +++ b/Makefile @@ -244,7 +244,7 @@ install: HELM_OPTS=--atomic --wait-for-jobs --timeout 2400s --namespace $(NAMESP --values $(CHART_DIR)/values-$(ENVIRONMENT).yaml install: @set -euo pipefail; \ - dagConfig=$$(echo '{"org": "bcgov", "repo": "cas-cif", "ref": "$(GIT_SHA1)", "path": "dags/cas_cif_dags.py"}' | base64 -w0); \ + dagConfig=$$(echo '{"org": "bcgov", "repo": "cas-cif", "ref": "$(GIT_SHA1)", "path": "dags/cas_cif_dags.py"}' | base64); \ helm dep up $(CHART_DIR); \ if ! helm status --namespace $(NAMESPACE) $(CHART_INSTANCE); then \ echo 'Installing the application and issuing SSL certificate'; \ diff --git a/chart/cas-cif/Chart.lock b/chart/cas-cif/Chart.lock index b82534374e..8d63dedbf1 100644 --- a/chart/cas-cif/Chart.lock +++ b/chart/cas-cif/Chart.lock @@ -11,5 +11,5 @@ dependencies: - name: certbot repository: https://bcdevops.github.io/certbot version: 0.1.3 -digest: sha256:22a3c8a861907a570a2415e8c4b5c0a5c7f6ae164207507f093b0bfaa752c8cf -generated: "2024-02-23T14:03:27.259123-08:00" +digest: sha256:d2da5fb2e1380ca061fdadb0e4cbe54137f1e3d43f90afe245339ddb320a771b +generated: "2024-07-17T18:07:23.81134-07:00" diff --git a/chart/cas-cif/templates/app-deployment.yaml b/chart/cas-cif/templates/app-deployment.yaml index 5f6a7cb917..f3bf44325a 100644 --- a/chart/cas-cif/templates/app-deployment.yaml +++ b/chart/cas-cif/templates/app-deployment.yaml @@ -53,6 +53,7 @@ spec: sleep 10; done; imagePullSecrets: {{ include "cas-cif.imagePullSecrets" . | nindent 8 }} + serviceAccountName: pod-logger containers: - name: {{ template "cas-cif.fullname" . }} imagePullPolicy: {{ default .Values.defaultImagePullPolicy .Values.image.app.pullPolicy }} @@ -139,6 +140,51 @@ spec: - mountPath: "/attachments-credentials" name: gcs-documents-credentials readOnly: true + - name: oc-logs-container + image: openshift/origin-cli:latest + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + command: + - "/bin/sh" + - "-c" + - | + echo 'Starting log capture'; + oc logs -f $POD_NAME -c {{ template "cas-cif.fullname" . }} --pod-running-timeout=20s >> /var/log/oc-cif-test.log; + volumeMounts: + - name: shared-logs + mountPath: /var/log + - name: logrotate-container + image: skymatic/logrotate:latest + command: + - "/bin/sh" + - "-c" + - "while true; do logrotate -s /var/log/logrotate.status -f /etc/logrotate.conf; sleep 5; done" + volumeMounts: + - name: shared-logs + mountPath: /var/log + - name: logrotate-config + mountPath: /etc/logrotate.conf + subPath: logrotate.conf + - name: fluent-bit + image: fluent/fluent-bit:latest + env: + - name: FLUENT_ELASTICSEARCH_HOST + value: elasticsearch.9212c9-tools.svc.cluster.local + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" + command: [ "/fluent-bit/bin/fluent-bit", "-c", "/var/log/fluent-bit.conf" ] + volumeMounts: + - name: shared-logs + mountPath: /var/log + - name: fluent-bit-config + mountPath: /var/log/fluent-bit.conf + subPath: fluent-bit.conf + - name: parsers-config + mountPath: /var/log/parsers.conf + subPath: parsers.conf volumes: - name: gcs-documents-credentials secret: @@ -146,4 +192,15 @@ spec: items: - key: credentials.json path: attachments-credentials.json + - name: shared-logs + emptyDir: { } + - name: logrotate-config + configMap: + name: logrotate-configmap + - name: fluent-bit-config + configMap: + name: fluent-bit-config + - name: parsers-config + configMap: + name: fluent-bit-config restartPolicy: Always diff --git a/chart/cas-cif/templates/fluent-bit-configmap.yaml b/chart/cas-cif/templates/fluent-bit-configmap.yaml new file mode 100644 index 0000000000..24a41adc9f --- /dev/null +++ b/chart/cas-cif/templates/fluent-bit-configmap.yaml @@ -0,0 +1,48 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluent-bit-config + namespace: c53ff1-dev +data: + fluent-bit.conf: | + [SERVICE] + Flush 5 + Daemon Off + Parsers_File parsers.conf + Log_Level info + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + + [INPUT] + Name tail + Path /var/log/oc-cif-test.log + Tag oc-cif-test + Mem_Buf_Limit 5MB + DB /var/log/flb_kube.db + Refresh_Interval 10 + Rotate_Wait 5 + Ignore_Older 24h + + [OUTPUT] + Name es + Match * + Host elasticsearch.9212c9-tools.svc.cluster.local + Port 9200 + Index ciif-logs + Logstash_Prefix ciif-logs + Logstash_DateFormat %Y.%m.%d + Logstash_Format On + Retry_Limit False + Suppress_Type_Name On + Type _doc + Time_Key @timestamp + Time_Key_Format iso8601 + + parsers.conf: | + [PARSER] + Name json + Format json + Time_Key timestamp + Decode_Field_as escaped_utf8 log do_next + Decode_Field_as json log diff --git a/chart/cas-cif/templates/logrotate-configmap.yaml b/chart/cas-cif/templates/logrotate-configmap.yaml new file mode 100644 index 0000000000..ee4cb7789c --- /dev/null +++ b/chart/cas-cif/templates/logrotate-configmap.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: logrotate-configmap +data: + logrotate.conf: | + /var/log/oc-cif-test.log { + size 100M + missingok + notifempty + copytruncate + rotate 5 + compress + delaycompress + dateext + dateformat -%Y%m%d%H%M%S + } diff --git a/chart/cas-cif/templates/pod-logger-role.yaml b/chart/cas-cif/templates/pod-logger-role.yaml new file mode 100644 index 0000000000..b26c70f187 --- /dev/null +++ b/chart/cas-cif/templates/pod-logger-role.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: pod-logger-role + namespace: c53ff1-dev +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get", "list","watch"] diff --git a/chart/cas-cif/templates/pod-logger-rolebinding.yaml b/chart/cas-cif/templates/pod-logger-rolebinding.yaml new file mode 100644 index 0000000000..8cf57d241f --- /dev/null +++ b/chart/cas-cif/templates/pod-logger-rolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pod-logger-rolebinding +subjects: + - kind: ServiceAccount + name: pod-logger + namespace: c53ff1-dev +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pod-logger-role diff --git a/chart/cas-cif/templates/service-account.yaml b/chart/cas-cif/templates/service-account.yaml new file mode 100644 index 0000000000..d822b75650 --- /dev/null +++ b/chart/cas-cif/templates/service-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pod-logger # Name of the service account + namespace: c53ff1-dev diff --git a/package.json b/package.json index 77df333d38..2049894d3c 100644 --- a/package.json +++ b/package.json @@ -26,5 +26,6 @@ "before:bump": "sqitch --chdir schema tag ${version} -m 'release v${version}'", "before:git:beforeRelease": "./.bin/pre-commit-format.sh CHANGELOG.md" } - } + }, + "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e" }