diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..d0fa86a2 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +.PHONY: ci-validate +ci-validate: check-reference + +# Basic lint checking +lintCheck: + # The configuration is done piece-wise in order to skip the + # kube-compare reference tree. Those yamls are augmented with + # golang templating and are not expected to be legal yaml. + yamllint -c .yamllint.yaml telco-core/configuration/*yaml + yamllint -c .yamllint.yaml telco-core/configuration/reference-crs + yamllint -c .yamllint.yaml telco-core/configuration/template-values + yamllint -c .yamllint.yaml telco-core/install/ + yamllint -c .yamllint.yaml telco-hub/ + +.PHONY: check-reference +check-reference: + $(MAKE) -C ./telco-core/configuration check diff --git a/telco-core/configuration/Makefile b/telco-core/configuration/Makefile new file mode 100644 index 00000000..94e95f9f --- /dev/null +++ b/telco-core/configuration/Makefile @@ -0,0 +1,18 @@ +.PHONY: check +check: compare_crs missing_crs + + +.PHONY: compare_crs +compare_crs: + @CLUSTER_COMPARE=$$(command -v kubectl-cluster_compare); \ + if [[ -z $${CLUSTER_COMPARE} ]]; then \ + echo "kubectl-cluster_compare tool isn't installed; please download it from https://github.com/openshift/kube-compare"; \ + exit 1; \ + fi; \ + $${CLUSTER_COMPARE} -r ./reference-crs-kube-compare/metadata.yaml -f ./reference-crs -R -p ./reference-crs-kube-compare/comparison-overrides.yaml + + +.PHONY: missing_crs +missing_crs: + ./compare.sh reference-crs reference-crs-kube-compare reference-crs-kube-compare/compare_ignore + diff --git a/telco-core/configuration/README.md b/telco-core/configuration/README.md new file mode 100644 index 00000000..68f253b2 --- /dev/null +++ b/telco-core/configuration/README.md @@ -0,0 +1,65 @@ +# Reference configuration + +## Structure +This directory contains four key components of the reference configuration + - The `reference-crs` tree contains the baseline configuration CRs which make + up the Core reference configuration. These are further separated into + optional vs required configuration. + - The yaml files in this top level support application and ongoing management + of the reference configuration using Advanced Cluster Management (ACM) + Policy. These yaml serve as manifests which define how CRs from the + reference-crs tree are grouped into policies and apply certain use case + specific patches to the policy wrapped CRs. + - The `template-values` directory holds ConfigMaps which provide values used in + the ACM Policies. See the "Templating" section below for more details. + - The `reference-crs-kube-compare` tree contains the template copy of the + baseline configuration for use by the + [cluster-compare tool](https://github.com/openshift/kube-compare). + +## Reference CRs + +## Policy generation CRs +### Policy Generators +There are three reference PolicyGenerator CRs. + - `core-baseline` contains fixed required content + - `core-overlay` contains content where updates/patches are expected. This + reference also contains the optional components + - `core-upgrade` contains policies which can be used to upgrade a cluster from + the prior release to the current release. + +Other custom content can be added through additional PolicyGenerator CRs. + +### Templating +These PolicyGenerator CRs create Policies which include ACM hub side +templates. These templates will pull values from 3 configmaps: + +`template-values/hw-types` -- Hardware dependent data. + - Current set of keys are fixed valued based on hardware profiles (mcp names) + as defined in core-overlay. + +`template-values/regional` -- Values which may depend on the region/zone where a +cluster is deployed. + - keyed by a "region" label on the ManagedCluster + - eg %s-log-url -- a cluster labeled 'region: abcd' would use abcd-log-url + from regional configmap + +`` -- Values which are cluster specific. One ConfigMap per cluster +is needed. The ConfigMap name is the cluster name eg cluster-1234 + - Current set of keys are fixed values + +# Contributing + +Given that the `reference-crs` and `reference-crs-kube-compare` versions of the +baseline configuration must be kept in sync, there is a github CI check than +enforces this. Running `make check` in this directory locally is equivalent to +the CI. + +If `make check` detects differences, you should take one of the following actions: + +- Edit the `reference-crs` CRs or `reference-crs-kube-compare` templates so the + templates match the corresponding CRs. +- For missing files, add the missing file to either the `reference-crs` + directory, or the `reference-crs-kube-compare` directory and metadata.yaml + - Alternatively, add the filename to the + `reference-crs-kube-compare/compare_ignore`, but only if the CR in + `reference-crs` should not be checked by the cluster-compare tool. diff --git a/telco-core/configuration/compare.sh b/telco-core/configuration/compare.sh new file mode 100755 index 00000000..a34cf274 --- /dev/null +++ b/telco-core/configuration/compare.sh @@ -0,0 +1,157 @@ +#! /bin/bash + +trap cleanup EXIT + +function cleanup() { + rm -rf source_file rendered_file same_file +} + +function read_dir() { + local dir=$1 + local file + + for file in "$dir"/*; do + if [ -d "$file" ]; then + read_dir "$file" + else + echo "$file" + fi + done +} + +function compare_cr { + local rendered_dir=$1 + local source_dir=$2 + local exclusionfile=$3 + local status=0 + + read_dir "$rendered_dir" |grep yaml > rendered_file + read_dir "$source_dir" |grep yaml > source_file + + local source_cr rendered + while IFS= read -r source_cr; do + while IFS= read -r rendered; do + if [ "${source_cr##*/}" = "${rendered##*/}" ]; then + echo "$source_cr" >> same_file + fi + done < rendered_file + done < source_file + + # Filter out files with a source-cr/reference match from the full list of potentiol source-crs/reference files + while IFS= read -r file; do + [[ ${file::1} != "#" ]] || continue # Skip any comment lines in the exclusionfile + [[ -n ${file} ]] || continue # Skip empty lines + sed -i "/${file##*/}/d" source_file + sed -i "/${file##*/}/d" rendered_file + done < <(cat same_file "$exclusionfile") + + if [[ -s source_file || -s rendered_file ]]; then + [ -s source_file ] && printf "\n\nThe following files exist in source-crs only, but not found in reference:\n" && cat source_file + [ -s rendered_file ] && printf "\nThe following files exist in reference only, but not found in source-crs:\n" && cat rendered_file + status=1 + fi + + return $status +} + +sync_cr() { + local rendered_dir=$1 + local source_dir=$2 + local exclusionfile=$3 + local status=0 + + local -a renderedFiles + readarray -t renderedFiles < <(read_dir "$rendered_dir" | grep yaml) + + local -a sourceFiles + readarray -t sourceFiles < <(read_dir "$source_dir" | grep yaml) + + local -a excludedFiles + readarray -t excludedFiles < <(grep -v '^#' "$exclusionfile" | grep -v '^$') + + local source rendered excluded found + for rendered in "${renderedFiles[@]}"; do + found=0 + for source in "${sourceFiles[@]}"; do + if [ "${source##*/}" = "${rendered##*/}" ]; then + # Match found! + found=1 + break + fi + done + if [[ $found == 0 ]]; then + source="$source_dir/${rendered##*/}" + fi + + # Replace the CR with the rendered copy (minus the helm-rendered heading) + tail -n +3 "$rendered" >"$source" + git add "$source" + done + + for source in "${sourceFiles[@]}"; do + found=0 + for rendered in "${renderedFiles[@]}"; do + if [ "${source##*/}" = "${rendered##*/}" ]; then + # Match found! + found=1 + break + fi + done + for excluded in "${excludedFiles[@]}"; do + if [ "${source##*/}" = "${excluded##*/}" ]; then + # Match found! + found=1 + break + fi + done + if [[ $found == 0 ]]; then + git rm -f "$source" + fi + done + + git diff --cached --stat --exit-code +} + +usage() { + echo "$(basename "$0") [--sync] sourceDir renderDir" + echo + echo "Compares the rendered reference-based CRs to the CRs in the compare directory" +} + +DOSYNC=0 +for arg in "$@"; do + case "$arg" in + -h | --help) + usage + exit 0 + ;; + --sync) + DOSYNC=1 + shift + ;; + esac +done +SOURCEDIR=$1 +if [[ ! -d $SOURCEDIR ]]; then + echo "No such source directory $SOURCEDIR" + usage + exit 1 +fi +RENDERDIR=$2 +if [[ ! -d $RENDERDIR ]]; then + echo "No such source directory $RENDERDIR" + usage + exit 1 +fi +IGNORE=$3 +if [[ ! -f $IGNORE ]]; then + echo "No such ignorefile $IGNORE" + usage + exit 1 +fi + +if [[ $DOSYNC == 1 ]]; then + sync_cr "$RENDERDIR" "$SOURCEDIR" "$IGNORE" +else + compare_cr "$RENDERDIR" "$SOURCEDIR" "$IGNORE" +fi diff --git a/telco-core/configuration/reference-crs-kube-compare/ReferenceVersionCheck.yaml b/telco-core/configuration/reference-crs-kube-compare/ReferenceVersionCheck.yaml new file mode 100644 index 00000000..273cf9ea --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/ReferenceVersionCheck.yaml @@ -0,0 +1,7 @@ +apiVersion: config.openshift.io/v1 +kind: ClusterVersion +metadata: + name: version +status: + desired: + version: {{ template "versionMatch" (list .status.desired.version "4.16") }} diff --git a/telco-core/configuration/reference-crs-kube-compare/compare_ignore b/telco-core/configuration/reference-crs-kube-compare/compare_ignore new file mode 100644 index 00000000..b50c5329 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/compare_ignore @@ -0,0 +1,20 @@ +# Internal files for cluster-compare, not real CRs +metadata.yaml +comparison-overrides.yaml + +# Used in the reference only for version compliance checks +ReferenceVersionCheck.yaml + +# Utility objects used to migrate from CLO5->CLO6 +ClusterLogging5Cleanup.yaml +ClusterLogOperatorStatus.yaml +ClusterLogForwarderDeleted.yaml +ClusterLogging.yaml + +# Utility objects to wait for and acknowledge cluster ugprades +ClusterVersion.yaml +upgrade-ack.yaml + +# Deprecated CRs, no longer in the reference-crs examples: +optional/networking/sriov/SriovNetworkPoolConfig.yaml +required/networking/metallb/service.yaml diff --git a/telco-core/configuration/reference-crs-kube-compare/comparison-overrides.yaml b/telco-core/configuration/reference-crs-kube-compare/comparison-overrides.yaml new file mode 100644 index 00000000..613d3c3b --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/comparison-overrides.yaml @@ -0,0 +1,7 @@ +- apiVersion: config.openshift.io/v1 + kind: ClusterVersion + name: version + patch: '{"status":null}' + reason: The ClusterVersion in reference-crs should not be corellated to ReferenceVersionCheck + templatePath: ReferenceVersionCheck.yaml + type: mergepatch diff --git a/telco-core/configuration/reference-crs-kube-compare/metadata.yaml b/telco-core/configuration/reference-crs-kube-compare/metadata.yaml new file mode 100644 index 00000000..2d58e4c9 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/metadata.yaml @@ -0,0 +1,202 @@ +apiVersion: v2 +parts: + - name: version-check + description: |- + A mismatch here means you may be using the wrong reference. + This reference was designed for OpenShift 4.16. + components: + - name: version-check + allOf: + - path: ReferenceVersionCheck.yaml + config: + ignore-unspecified-fields: true + fieldsToOmitRefs: + - allowStatusCheck + - name: networking + description: |- + https://docs.openshift.com/container-platform/4.16/scalability_and_performance/telco_ref_design_specs/core/telco-core-ref-design-components.html#telco-core-networking_core-ref-design-components + components: + - name: networking-root + allOf: + - path: required/networking/Network.yaml + config: + ignore-unspecified-fields: true + - path: required/networking/nodeNetworkConfigurationPolicy.yaml + - name: networking-nmsate + description: |- + https://docs.openshift.com/container-platform/4.16/scalability_and_performance/telco_ref_design_specs/core/telco-core-ref-design-components.html#telco-core-nmstate-operator_core-ref-design-components + allOrNoneOf: + - path: required/networking/NMState.yaml + - path: required/networking/NMStateNS.yaml + - path: required/networking/NMStateOperGroup.yaml + config: + ignore-unspecified-fields: true + - path: required/networking/NMStateSubscription.yaml + - name: networking-metallb + description: |- + https://docs.openshift.com/container-platform/4.16/scalability_and_performance/telco_ref_design_specs/core/telco-core-ref-design-components.html#telco-core-load-balancer_core-ref-design-components + allOf: + - path: required/networking/metallb/addr-pool.yaml + - path: required/networking/metallb/bfd-profile.yaml + - path: required/networking/metallb/bgp-advr.yaml + - path: required/networking/metallb/bgp-peer.yaml + - path: required/networking/metallb/community.yaml + - path: required/networking/metallb/metallb.yaml + - path: required/networking/metallb/metallbNS.yaml + - path: required/networking/metallb/metallbOperGroup.yaml + config: + ignore-unspecified-fields: true + - path: required/networking/metallb/metallbSubscription.yaml + - name: networking-multinetworkpolicy + description: |- + https://docs.openshift.com/container-platform/4.16/scalability_and_performance/telco_ref_design_specs/core/telco-core-ref-design-components.html#telco-core-sriov_core-ref-design-components + allOf: + - path: required/networking/multinetworkpolicy/multiNetworkPolicyAllowPortProtocol.yaml + - name: networking-multinetworkpolicy-denyall + description: |- + https://docs.openshift.com/container-platform/4.16/scalability_and_performance/telco_ref_design_specs/core/telco-core-ref-design-components.html#telco-core-sriov_core-ref-design-components + anyOf: + - path: required/networking/multinetworkpolicy/multiNetworkPolicyDenyAll.yaml + - name: networking-sriov + description: |- + https://docs.openshift.com/container-platform/4.16/scalability_and_performance/telco_ref_design_specs/core/telco-core-ref-design-components.html#telco-core-sriov_core-ref-design-components + allOf: + - path: required/networking/sriov/sriovNetwork.yaml + - path: required/networking/sriov/sriovNetworkNodePolicy.yaml + - path: required/networking/sriov/SriovOperatorConfig.yaml + config: + ignore-unspecified-fields: true + - path: required/networking/sriov/SriovSubscription.yaml + - path: required/networking/sriov/SriovSubscriptionNS.yaml + - path: required/networking/sriov/SriovSubscriptionOperGroup.yaml + config: + ignore-unspecified-fields: true + - name: optional-networking + anyOf: + - path: optional/networking/networkAttachmentDefinition.yaml + - path: optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml + - name: required-other + components: + - name: disconnected-registry + description: |- + https://docs.openshift.com/container-platform/4.16/scalability_and_performance/telco_ref_design_specs/core/telco-core-ref-design-components.html#telco-core-disconnected-environment_core-ref-design-components + allOf: + - path: required/other/catalog-source.yaml + - path: required/other/icsp.yaml + - path: required/other/operator-hub.yaml + - name: required-performance + description: |- + https://docs.openshift.com/container-platform/4.16/scalability_and_performance/telco_ref_design_specs/core/telco-core-ref-design-components.html#telco-core-cpu-partitioning-performance-tune_core-ref-design-components + components: + - name: performance + allOf: + - path: required/performance/PerformanceProfile.yaml + - name: required-scheduling + components: + - name: scheduling + description: |- + https://docs.openshift.com/container-platform/4.16/scalability_and_performance/telco_ref_design_specs/core/telco-core-ref-design-components.html#telco-core-scheduling_core-ref-design-components + allOf: + - path: required/scheduling/nrop.yaml + - path: required/scheduling/NROPSubscription.yaml + - path: required/scheduling/NROPSubscriptionNS.yaml + - path: required/scheduling/NROPSubscriptionOperGroup.yaml + config: + ignore-unspecified-fields: true + - path: required/scheduling/sched.yaml + - path: required/scheduling/Scheduler.yaml + - name: required-storage + components: + - name: storage-odf + description: |- + https://docs.openshift.com/container-platform/4.16/scalability_and_performance/telco_ref_design_specs/core/telco-core-ref-design-components.html#telco-core-rh-storage_core-ref-design-components + allOf: + - path: required/storage/odf-external/01-rook-ceph-external-cluster-details.secret.yaml + - path: required/storage/odf-external/02-ocs-external-storagecluster.yaml + config: + ignore-unspecified-fields: true + - path: required/storage/odf-external/odfNS.yaml + - path: required/storage/odf-external/odfOperGroup.yaml + config: + ignore-unspecified-fields: true + - path: required/storage/odf-external/odfSubscription.yaml + - name: other + description: |- + https://docs.openshift.com/container-platform/4.16/scalability_and_performance/telco_ref_design_specs/core/telco-core-ref-crs.html#node-configuration-crs_ran-core-ref-design-crs + components: + - name: other + anyOf: + - path: optional/other/control-plane-load-kernel-modules.yaml + - path: optional/other/monitoring-config-cm.yaml + - path: optional/other/worker-load-kernel-modules.yaml + - path: optional/other/sctp_module_mc.yaml + - name: logging + description: |- + https://docs.openshift.com/container-platform/4.16/scalability_and_performance/telco_ref_design_specs/core/telco-core-ref-crs.html#other-crs_ran-core-ref-design-crs + components: + - name: logging + allOrNoneOf: + - path: optional/logging/ClusterLogForwarder.yaml + - path: optional/logging/ClusterLogNS.yaml + - path: optional/logging/ClusterLogOperGroup.yaml + config: + ignore-unspecified-fields: true + - path: optional/logging/ClusterLogSubscription.yaml + - path: optional/logging/ClusterLogServiceAccountAuditBinding.yaml + - path: optional/logging/ClusterLogServiceAccountInfrastructureBinding.yaml + - name: tuning + description: |- + https://docs.openshift.com/container-platform/4.16/scalability_and_performance/telco_ref_design_specs/core/telco-core-ref-design-components.html#telco-core-cpu-partitioning-performance-tune_core-ref-design-components + components: + - name: other + anyOf: + - path: optional/tuning/control-plane-system-reserved.yaml + +templateFunctionFiles: + - version_match.tmpl + +fieldsToOmit: + defaultOmitRef: all + items: + defaults: + - pathToKey: metadata.annotations."kubernetes.io/metadata.name" + - pathToKey: metadata.annotations."openshift.io/sa.scc.uid-range" + - pathToKey: metadata.annotations."kubernetes.io/metadata.name" + - pathToKey: metadata.annotations."openshift.io/sa.scc.mcs" + - pathToKey: metadata.annotations."openshift.io/sa.scc.supplemental-groups" + - pathToKey: metadata.annotations."machineconfiguration.openshift.io/mc-name-suffix" + - pathToKey: metadata.annotations."kubectl.kubernetes.io/last-applied-configuration" + - pathToKey: metadata.annotations."nmstate.io/webhook-mutating-timestamp" + - pathToKey: metadata.labels."kubernetes.io/metadata.name" + - pathToKey: metadata.labels."pod-security.kubernetes.io" + isPrefix: true + - pathToKey: metadata.labels."operators.coreos.com/" + isPrefix: true + - pathToKey: metadata.labels."security.openshift.io/scc.podSecurityLabelSync" + - pathToKey: metadata.resourceVersion + - pathToKey: metadata.uid + - pathToKey: spec.finalizers + - pathToKey: metadata.creationTimestamp + - pathToKey: metadata.generation + - pathToKey: metadata.finalizers + - pathToKey: metadata.annotations."ran.openshift.io/ztp-gitops-generated" + - pathToKey: spec.ownerReferences + - pathToKey: metadata.ownerReferences + - pathToKey: metadata.annotations."include.release.openshift.io/ibm-cloud-managed" + - pathToKey: metadata.annotations."include.release.openshift.io/self-managed-high-availability" + - pathToKey: metadata.annotations."include.release.openshift.io/single-node-developer" + - pathToKey: metadata.annotations."release.openshift.io/create-only" + - pathToKey: metadata.labels."lca.openshift.io/target-ocp-version" + - pathToKey: metadata.annotations."capability.openshift.io/name" + - pathToKey: metadata.annotations."olm.providedAPIs" + - pathToKey: metadata.labels."olm.operatorgroup.uid" + isPrefix: true + - pathToKey: metadata.annotations."nmstate.io/webhook-mutating-timestamp" + - pathToKey: metadata.annotations."operator.sriovnetwork.openshift.io/last-network-namespace" + - pathToKey: metadata.annotations."workload.openshift.io/allowed" + - pathToKey: spec.installPlanApproval + allowStatusCheck: + - include: defaults + all: + - include: defaults + - pathToKey: status diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogForwarder.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogForwarder.yaml new file mode 100644 index 00000000..5871ace2 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogForwarder.yaml @@ -0,0 +1,26 @@ +apiVersion: observability.openshift.io/v1 +kind: ClusterLogForwarder +metadata: + name: instance + namespace: openshift-logging +spec: + {{- if .spec.outputs }} + outputs: + - name: {{ (index .spec.outputs 0).name }} + type: kafka + kafka: + url: {{ (index .spec.outputs 0).kafka.url }} + {{- end }} + {{- if .spec.filters }} + filters: + {{- .spec.filters | toYaml | nindent 2 }} + {{- end }} + {{- if .spec.pipelines }} + pipelines: + {{- .spec.pipelines | toYaml | nindent 2 }} + {{- end }} + serviceAccount: + name: {{ .spec.serviceAccount.name }} + {{- if .spec.managementState }} + managementState: {{ .spec.managementState }} + {{- end }} diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogNS.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogNS.yaml new file mode 100644 index 00000000..47d3eee0 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogNS.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-logging diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogOperGroup.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogOperGroup.yaml new file mode 100644 index 00000000..2e26e674 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogOperGroup.yaml @@ -0,0 +1,11 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: cluster-logging + namespace: openshift-logging +spec: + targetNamespaces: + - openshift-logging + {{- if .spec.upgradeStrategy }} + upgradeStrategy: Default + {{- end }} diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogServiceAccount.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogServiceAccount.yaml new file mode 100644 index 00000000..70b1d60c --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogServiceAccount.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: collector + namespace: openshift-logging diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogServiceAccountAuditBinding.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogServiceAccountAuditBinding.yaml new file mode 100644 index 00000000..9dba7e56 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogServiceAccountAuditBinding.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: logcollector-audit-logs-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: collect-audit-logs +subjects: +- kind: ServiceAccount + name: logcollector + namespace: openshift-logging diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogServiceAccountInfrastructureBinding.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogServiceAccountInfrastructureBinding.yaml new file mode 100644 index 00000000..36682d83 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogServiceAccountInfrastructureBinding.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: logcollector-infrastructure-logs-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: collect-infrastructure-logs +subjects: +- kind: ServiceAccount + name: logcollector + namespace: openshift-logging diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogSubscription.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogSubscription.yaml new file mode 100644 index 00000000..749585c3 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/logging/ClusterLogSubscription.yaml @@ -0,0 +1,13 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: cluster-logging + namespace: openshift-logging +spec: + channel: "stable-6.0" + name: cluster-logging + source: {{ .spec.source }} + sourceNamespace: openshift-marketplace + installPlanApproval: Manual +status: + state: AtLatestKnown diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml new file mode 100644 index 00000000..b08bb52b --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml @@ -0,0 +1,26 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: worker + name: 99-worker-setsebool +spec: + config: + ignition: + version: 3.2.0 + systemd: + units: + - contents: | + [Unit] + Description=Set SELinux boolean for tap cni plugin + Before=kubelet.service + + [Service] + Type=oneshot + ExecStart=/sbin/setsebool container_use_devices=on + RemainAfterExit=true + + [Install] + WantedBy=multi-user.target graphical.target + enabled: true + name: setsebool.service diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/networking/networkAttachmentDefinition.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/networking/networkAttachmentDefinition.yaml new file mode 100644 index 00000000..689db0a1 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/networking/networkAttachmentDefinition.yaml @@ -0,0 +1,13 @@ +# optional +# copies: 0-N +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: {{ .metadata.name }} + namespace: {{ .metadata.namespace }} +spec: + {{ if .spec.nodeSelector }} + nodeSelector: + {{ .spec.nodeSelector | toYaml | indent 4 }} + {{ end }} + config: {{ .spec.config | toJson }} diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/other/control-plane-load-kernel-modules.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/other/control-plane-load-kernel-modules.yaml new file mode 100644 index 00000000..8e1af63e --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/other/control-plane-load-kernel-modules.yaml @@ -0,0 +1,25 @@ +# optional +# count: 1 +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: master + name: 40-load-kernel-modules-control-plane +spec: + config: + # Release info found in https://github.com/coreos/butane/releases + ignition: + version: 3.2.0 + storage: + files: + - contents: + source: data:, + mode: 420 + overwrite: true + path: /etc/modprobe.d/kernel-blacklist.conf + - contents: + source: data:text/plain;charset=utf-8;base64,aXBfZ3JlCmlwNl90YWJsZXMKaXA2dF9SRUpFQ1QKaXA2dGFibGVfZmlsdGVyCmlwNnRhYmxlX21hbmdsZQppcHRhYmxlX2ZpbHRlcgppcHRhYmxlX21hbmdsZQppcHRhYmxlX25hdAp4dF9tdWx0aXBvcnQKeHRfb3duZXIKeHRfUkVESVJFQ1QKeHRfc3RhdGlzdGljCnh0X1RDUE1TUwo= + mode: 420 + overwrite: true + path: /etc/modules-load.d/kernel-load.conf diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/other/monitoring-config-cm.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/other/monitoring-config-cm.yaml new file mode 100644 index 00000000..73d52504 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/other/monitoring-config-cm.yaml @@ -0,0 +1,25 @@ +# optional +# count: 1 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-monitoring-config + namespace: openshift-monitoring +data: + config.yaml: | + prometheusK8s: + retention: 15d + volumeClaimTemplate: + spec: + storageClassName: ocs-external-storagecluster-ceph-rbd + resources: + requests: + storage: 100Gi + alertmanagerMain: + volumeClaimTemplate: + spec: + storageClassName: ocs-external-storagecluster-ceph-rbd + resources: + requests: + storage: 20Gi diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/other/sctp_module_mc.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/other/sctp_module_mc.yaml new file mode 100644 index 00000000..9298a860 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/other/sctp_module_mc.yaml @@ -0,0 +1,32 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + name: load-sctp-module + labels: + machineconfiguration.openshift.io/role: {{ index (or .metadata.labels dict) "machineconfiguration.openshift.io/role" }} +spec: + config: + ignition: + version: 2.2.0 + storage: + files: + - contents: + source: data:, + verification: {} + filesystem: root + mode: 420 + path: /etc/modprobe.d/sctp-blacklist.conf + - contents: + {{- $sctpData := "" }} + {{- range $file := (or .spec.config.storage.files list) }} + {{- if eq $file.path "/etc/modules-load.d/sctp-load.conf" }} + {{- $sctpData = $file.contents.source }} + {{- if not (eq $sctpData "data:,sctp" "data:text/plain;charset=utf-8,sctp" "data:text/plain;charset=utf-8;base64,c2N0cA==") }} + {{- $sctpData = "data:,sctp (or equivalent encoded values)" }} + {{- end }} + {{- end }} + {{- end }} + source: {{ $sctpData }} + filesystem: root + mode: 420 + path: /etc/modules-load.d/sctp-load.conf diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/other/worker-load-kernel-modules.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/other/worker-load-kernel-modules.yaml new file mode 100644 index 00000000..8e940d71 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/other/worker-load-kernel-modules.yaml @@ -0,0 +1,25 @@ +# optional +# count: 1 +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: worker + name: 40-load-kernel-modules-worker +spec: + config: + # Release info found in https://github.com/coreos/butane/releases + ignition: + version: 3.2.0 + storage: + files: + - contents: + source: data:, + mode: 420 + overwrite: true + path: /etc/modprobe.d/kernel-blacklist.conf + - contents: + source: data:text/plain;charset=utf-8;base64,aXBfZ3JlCmlwNl90YWJsZXMKaXA2dF9SRUpFQ1QKaXA2dGFibGVfZmlsdGVyCmlwNnRhYmxlX21hbmdsZQppcHRhYmxlX2ZpbHRlcgppcHRhYmxlX21hbmdsZQppcHRhYmxlX25hdAp4dF9tdWx0aXBvcnQKeHRfb3duZXIKeHRfUkVESVJFQ1QKeHRfc3RhdGlzdGljCnh0X1RDUE1TUwo= + mode: 420 + overwrite: true + path: /etc/modules-load.d/kernel-load.conf diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/tuning/control-plane-system-reserved.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/tuning/control-plane-system-reserved.yaml new file mode 100644 index 00000000..49ba247c --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/optional/tuning/control-plane-system-reserved.yaml @@ -0,0 +1,11 @@ +# optional +# count: 1 +apiVersion: machineconfiguration.openshift.io/v1 +kind: KubeletConfig +metadata: + name: autosizing-master +spec: + autoSizingReserved: true + machineConfigPoolSelector: + matchLabels: + pools.operator.machineconfiguration.openshift.io/master: "" diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/NMState.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/NMState.yaml new file mode 100644 index 00000000..7f9178ae --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/NMState.yaml @@ -0,0 +1,11 @@ +apiVersion: nmstate.io/v1 +kind: NMState +metadata: + name: nmstate + +{{- if eq (len (or .spec dict )) 0 }} +spec: {} +{{- else -}} +spec: +{{ .spec | toYaml | indent 2 }} +{{ end }} diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/NMStateNS.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/NMStateNS.yaml new file mode 100644 index 00000000..1270b635 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/NMStateNS.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-nmstate diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/NMStateOperGroup.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/NMStateOperGroup.yaml new file mode 100644 index 00000000..f8e6315c --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/NMStateOperGroup.yaml @@ -0,0 +1,8 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: openshift-nmstate + namespace: openshift-nmstate +spec: + targetNamespaces: + - openshift-nmstate diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/NMStateSubscription.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/NMStateSubscription.yaml new file mode 100644 index 00000000..307729fa --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/NMStateSubscription.yaml @@ -0,0 +1,13 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: kubernetes-nmstate-operator + namespace: openshift-nmstate +spec: + channel: "stable" + name: kubernetes-nmstate-operator + source: {{ .spec.source }} + sourceNamespace: openshift-marketplace + installPlanApproval: Manual +status: + state: AtLatestKnown diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/Network.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/Network.yaml new file mode 100644 index 00000000..418b1c43 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/Network.yaml @@ -0,0 +1,12 @@ +apiVersion: operator.openshift.io/v1 +kind: Network +metadata: + name: cluster +spec: + defaultNetwork: + ovnKubernetesConfig: + gatewayConfig: + routingViaHost: true + {{ if hasKey .spec "useMultiNetworkPolicy" }} + useMultiNetworkPolicy: {{ .spec.useMultiNetworkPolicy }} + {{ end }} diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/addr-pool.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/addr-pool.yaml new file mode 100644 index 00000000..7b8b6360 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/addr-pool.yaml @@ -0,0 +1,19 @@ +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: {{ .metadata.name }} # eg addresspool3 + namespace: metallb-system + {{- if and (.metadata.annotations) (ne (index .metadata.annotations "metallb.universe.tf/address-pool") nil) }} + annotations: + metallb.universe.tf/address-pool: {{ .metadata.name }} # eg addresspool3 + {{- end }} +spec: + ############## + # Expected variation in this configuration +{{ if .spec.addresses }} + addresses: +{{ .spec.addresses | toYaml | indent 2}} +{{ end }} + #- 3.3.3.0/24 + autoAssign: true + ############## diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/bfd-profile.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/bfd-profile.yaml new file mode 100644 index 00000000..6da88d67 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/bfd-profile.yaml @@ -0,0 +1,19 @@ +apiVersion: metallb.io/v1beta1 +kind: BFDProfile +metadata: + name: {{ .metadata.name }} + namespace: metallb-system +spec: + ################ + # These values may vary. Recommended values are included as default + receiveInterval: {{ if .spec.receiveInterval }} {{ .spec.receiveInterval }} {{ else }} "300ms" {{ end }} # default 300ms + transmitInterval: {{ if .spec.transmitInterval }} {{ .spec.transmitInterval }} {{ else }} "300ms" {{ end }} # default 300ms + {{ if .spec.echoInterval }} + echoInterval: {{ if .spec.echoInterval }} {{ .spec.echoInterval }} {{ else }} "50ms" {{ end }} # default 50ms + {{ end }} + detectMultiplier: {{ if .spec.detectMultiplier }} {{ .spec.detectMultiplier }} {{ else }} "3" {{ end }} # default 3 + echoMode: true + passiveMode: true + minimumTtl: {{ if .spec.minimumTtl }} {{ .spec.minimumTtl }} {{ else }} "254" {{ end }} # default 254 + # + ################ diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/bgp-advr.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/bgp-advr.yaml new file mode 100644 index 00000000..7ab5ab6b --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/bgp-advr.yaml @@ -0,0 +1,25 @@ +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: {{ .metadata.name }} # eg bgpadvertisement-1 + namespace: metallb-system +spec: +{{ if .spec.ipAddressPools }} + ipAddressPools: +{{ .spec.ipAddressPools | toYaml | indent 2}} +{{ end }} +{{ if .spec.peers }} + peers: +{{ .spec.peers | toYaml | indent 2}} +{{ end }} +{{ if .spec.communities }} + communities: +{{ .spec.communities | toYaml | indent 2 }} +{{ end }} + #communities: [ $communities ] + # Note correlation with address pool. + # eg: +# - 65535:65282 + aggregationLength: 32 + aggregationLengthV6: 128 + localPref: 100 diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/bgp-peer.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/bgp-peer.yaml new file mode 100644 index 00000000..96caae7f --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/bgp-peer.yaml @@ -0,0 +1,12 @@ +apiVersion: metallb.io/v1beta2 +kind: BGPPeer +metadata: + name: {{ .metadata.name }} + namespace: metallb-system +spec: + peerAddress: {{ .spec.peerAddress }} + peerASN: {{ .spec.peerASN }} + myASN: {{ .spec.myASN }} + routerID: {{ .spec.routerID }} + bfdProfile: {{ .spec.bfdProfile }} + passwordSecret: {{ .spec.passwordSecret | toJson }} diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/community.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/community.yaml new file mode 100644 index 00000000..d21c1af2 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/community.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: metallb.io/v1beta1 +kind: Community +metadata: + name: {{ .metadata.name }} + namespace: metallb-system +spec: + communities: +{{ .spec.communities | toYaml | indent 4 }} diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/metallb.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/metallb.yaml new file mode 100644 index 00000000..e864758e --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/metallb.yaml @@ -0,0 +1,14 @@ +apiVersion: metallb.io/v1beta1 +kind: MetalLB +metadata: + name: metallb + namespace: metallb-system +{{- if .spec }} +spec: + {{- if .spec.nodeSelector }} + nodeSelector: + {{- .spec.nodeSelector | toYaml | nindent 4 }} + {{- end }} + #nodeSelector: + # node-role.kubernetes.io/worker: "" +{{- end }} diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/metallbNS.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/metallbNS.yaml new file mode 100644 index 00000000..b6a6c36f --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/metallbNS.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: metallb-system + labels: + openshift.io/cluster-monitoring: "true" diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/metallbOperGroup.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/metallbOperGroup.yaml new file mode 100644 index 00000000..838e4738 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/metallbOperGroup.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: metallb-operator + namespace: metallb-system diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/metallbSubscription.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/metallbSubscription.yaml new file mode 100644 index 00000000..07792054 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/metallb/metallbSubscription.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: metallb-operator-sub + namespace: metallb-system +spec: + channel: stable + name: metallb-operator + source: {{ .spec.source }} + sourceNamespace: openshift-marketplace + installPlanApproval: Manual + diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/multinetworkpolicy/multiNetworkPolicyAllowPortProtocol.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/multinetworkpolicy/multiNetworkPolicyAllowPortProtocol.yaml new file mode 100644 index 00000000..c8793554 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/multinetworkpolicy/multiNetworkPolicyAllowPortProtocol.yaml @@ -0,0 +1,20 @@ +apiVersion: k8s.cni.cncf.io/v1beta1 +kind: MultiNetworkPolicy +metadata: + name: allow-port-and-protocol + annotations: + {{ if .metadata.annotations }} + k8s.v1.cni.cncf.io/policy-for: {{ index .metadata.annotations "k8s.v1.cni.cncf.io/policy-for" }} + {{ end }} +spec: + podSelector: + matchLabels: # Define which pods are affected by this policy + {{ .spec.podSelector.matchLabels | toYaml | indent 6}} + policyTypes: + - Ingress + ingress: + - ports: + {{ if .spec.ingress }} + - protocol: {{ if index (index .spec.ingress 0).ports 0 }}{{ index (index .spec.ingress 0).ports 0 "protocol" }}{{ else }}nil{{ end }} + port: {{ if index (index .spec.ingress 0).ports 0 }}{{ index (index .spec.ingress 0).ports 0 "port" }}{{ else }}nil{{ end }} + {{ end }} diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/multinetworkpolicy/multiNetworkPolicyDenyAll.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/multinetworkpolicy/multiNetworkPolicyDenyAll.yaml new file mode 100644 index 00000000..436533bf --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/multinetworkpolicy/multiNetworkPolicyDenyAll.yaml @@ -0,0 +1,27 @@ +apiVersion: k8s.cni.cncf.io/v1beta1 +kind: MultiNetworkPolicy +metadata: + name: deny-all +{{- if .metadata.annotations }} + {{- if ne (index .metadata.annotations "k8s.v1.cni.cncf.io/policy-for") nil }} + annotations: + k8s.v1.cni.cncf.io/policy-for: {{ index .metadata.annotations "k8s.v1.cni.cncf.io/policy-for" }} + {{- end }} +{{- end }} +spec: + podSelector: + {{ if .spec.podSelector.matchLabels }} + matchLabels: # Define which pods are affected by this policy + {{ range $labelKey, $labelValue := .spec.podSelector.matchLabels }} + {{ $labelKey}} : {{ $labelValue }} + {{ end }} + {{ else }} + matchLabels: {} + {{ end }} + policyTypes: + - Ingress + {{- if and (.spec.ingress) (ne (len .spec.ingress) 0) }} + ingress: Ingress list must be of size 0 to deny all ingress traffic + {{- else }} + ingress: [] # Deny all ingress traffic + {{- end }} diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/nodeNetworkConfigurationPolicy.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/nodeNetworkConfigurationPolicy.yaml new file mode 100644 index 00000000..faee1a51 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/nodeNetworkConfigurationPolicy.yaml @@ -0,0 +1,49 @@ +# optional +# copies: 0-N (up to one per node) +apiVersion: nmstate.io/v1 +kind: NodeNetworkConfigurationPolicy +metadata: + name: {{ .metadata.name }} +spec: + {{- if (hasKey .spec "nodeSelector") }} + nodeSelector: + {{- .spec.nodeSelector | toYaml | nindent 4 }} + {{- end }} + desiredState: + {{- if (hasKey .spec.desiredState "interfaces") }} + interfaces: + {{- .spec.desiredState.interfaces | toYaml | nindent 6 }} + {{- end }} +# - name: ens8f0.169 +# type: vlan +# state: up +# ipv4: +# address: +# - ip: 192.168.1.1 +# prefix-length: 24 +# dhcp: false +# enabled: true +# vlan: +# base-iface: ens8f0 +# id: 169 + +# Example, shared host networking. +# - name: ens5f0 +# description: Create VFs +# type: ethernet +# state: up +# ethernet: +# sr-iov: +# total-vfs: 5 + {{- if .spec.desiredState.routes }} + routes: + {{- if (hasKey .spec.desiredState.routes "config") }} + config: + {{- .spec.desiredState.routes.config | toYaml | nindent 8 }} + {{- end }} + {{- end }} +# - destination: 172.16.0.0/24 +# metric: 150 +# next-hop-address: 192.168.1.2 +# next-hop-interface: ens8f0.169 +# table-id: 254 diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/SriovOperatorConfig.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/SriovOperatorConfig.yaml new file mode 100644 index 00000000..ffbbb124 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/SriovOperatorConfig.yaml @@ -0,0 +1,15 @@ +# required +# count: 1 +--- +apiVersion: sriovnetwork.openshift.io/v1 +kind: SriovOperatorConfig +metadata: + name: default + namespace: openshift-sriov-network-operator +spec: + configDaemonNodeSelector: + {{ range $key, $val := .spec.configDaemonNodeSelector }} + {{ $key }}: {{ quote $val}} + {{- end }} + enableInjector: true + enableOperatorWebhook: true diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/SriovSubscription.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/SriovSubscription.yaml new file mode 100644 index 00000000..4c9c9538 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/SriovSubscription.yaml @@ -0,0 +1,13 @@ +# required: yes +# count: 1 +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: sriov-network-operator-subscription + namespace: openshift-sriov-network-operator +spec: + channel: "stable" + name: sriov-network-operator + source: {{ .spec.source }} + sourceNamespace: openshift-marketplace + installPlanApproval: Manual diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/SriovSubscriptionNS.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/SriovSubscriptionNS.yaml new file mode 100644 index 00000000..55cf184a --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/SriovSubscriptionNS.yaml @@ -0,0 +1,6 @@ +# required: yes +# count: 1 +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-sriov-network-operator diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/SriovSubscriptionOperGroup.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/SriovSubscriptionOperGroup.yaml new file mode 100644 index 00000000..31e7bb1a --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/SriovSubscriptionOperGroup.yaml @@ -0,0 +1,10 @@ +# required: yes +# count: 1 +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: sriov-network-operators + namespace: openshift-sriov-network-operator +spec: + targetNamespaces: + - openshift-sriov-network-operator diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/sriovNetwork.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/sriovNetwork.yaml new file mode 100644 index 00000000..4ac8c2e6 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/sriovNetwork.yaml @@ -0,0 +1,13 @@ +# optional (though expected for all) +# count: 0-N +apiVersion: sriovnetwork.openshift.io/v1 +kind: SriovNetwork +metadata: + name: {{ .metadata.name }} # eg sriov-network-abcd + namespace: openshift-sriov-network-operator +spec: +{{ .spec | toYaml | indent 2 }} +# capabilities: "$capabilities" # eg '{"mac": true, "ips": true}' +# ipam: "$ipam" # eg '{ "type": "host-local", "subnet": "10.3.38.0/24" }' +# networkNamespace: $nns # eg cni-test +# resourceName: $resource # eg resourceTest diff --git a/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/sriovNetworkNodePolicy.yaml b/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/sriovNetworkNodePolicy.yaml new file mode 100644 index 00000000..9ff31507 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/networking/sriov/sriovNetworkNodePolicy.yaml @@ -0,0 +1,24 @@ +# optional (though expected in all deployments) +# count: 0-N +apiVersion: sriovnetwork.openshift.io/v1 +kind: SriovNetworkNodePolicy +metadata: + name: {{ .metadata.name }} + namespace: openshift-sriov-network-operator +spec: # $spec +{{ .spec | toYaml | indent 2 }} + # eg + #deviceType: netdevice + #nicSelector: + # deviceID: "1593" + # pfNames: + # - ens8f0np0#0-9 + # rootDevices: + # - 0000:d8:00.0 + # vendor: "8086" + #nodeSelector: + # kubernetes.io/hostname: host.sample.lab + #numVfs: 20 + #priority: 99 + #excludeTopology: true + #resourceName: resourceNameABCD diff --git a/telco-core/configuration/reference-crs-kube-compare/required/other/catalog-source.yaml b/telco-core/configuration/reference-crs-kube-compare/required/other/catalog-source.yaml new file mode 100644 index 00000000..f52db62b --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/other/catalog-source.yaml @@ -0,0 +1,20 @@ +# required +# count: 1..N +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: redhat-operators-disconnected + namespace: openshift-marketplace +spec: + displayName: Red Hat Disconnected Operators Catalog + image: {{ .spec.image }} # $imageUrl + publisher: Red Hat + sourceType: grpc + {{ if .spec.updateStrategy }} + updateStrategy: + registryPoll: + interval: 1h + {{ end }} +status: + connectionState: + lastObservedState: READY diff --git a/telco-core/configuration/reference-crs-kube-compare/required/other/icsp.yaml b/telco-core/configuration/reference-crs-kube-compare/required/other/icsp.yaml new file mode 100644 index 00000000..002a8a3d --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/other/icsp.yaml @@ -0,0 +1,11 @@ +# required +# count: 1 +apiVersion: operator.openshift.io/v1alpha1 +kind: ImageContentSourcePolicy +metadata: + name: {{ .metadata.name }} +spec: + {{- if hasKey .spec "repositoryDigestMirrors" }} + repositoryDigestMirrors: + {{- .spec.repositoryDigestMirrors | toYaml | nindent 4 }} + {{- end }} diff --git a/telco-core/configuration/reference-crs-kube-compare/required/other/operator-hub.yaml b/telco-core/configuration/reference-crs-kube-compare/required/other/operator-hub.yaml new file mode 100644 index 00000000..1af2f8c1 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/other/operator-hub.yaml @@ -0,0 +1,9 @@ +# required +# count: 1 +apiVersion: config.openshift.io/v1 +kind: OperatorHub +metadata: + name: cluster +spec: + disableAllDefaultSources: true + diff --git a/telco-core/configuration/reference-crs-kube-compare/required/performance/PerformanceProfile.yaml b/telco-core/configuration/reference-crs-kube-compare/required/performance/PerformanceProfile.yaml new file mode 100644 index 00000000..9a7b29a1 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/performance/PerformanceProfile.yaml @@ -0,0 +1,58 @@ +# required +# count: 1 +apiVersion: performance.openshift.io/v2 +kind: PerformanceProfile +metadata: + name: {{ .metadata.name }} + annotations: + # Some pods want the kernel stack to ignore IPv6 router Advertisement. + kubeletconfig.experimental: | + {"allowedUnsafeSysctls":["net.ipv6.conf.all.accept_ra"]} +spec: + cpu: + # node0 CPUs: 0-17,36-53 + # node1 CPUs: 18-34,54-71 + # siblings: (0,36), (1,37)... + # we want to reserve the first Core of each NUMA socket + # + # no CPU left behind! all-cpus == isolated + reserved + isolated: {{ .spec.cpu.isolated }} # eg 1-17,19-35,37-53,55-71 + reserved: {{ .spec.cpu.reserved }} # eg 0,18,36,54 + # Guaranteed QoS pods will disable IRQ balancing for cores allocated to the pod. + # default value of globallyDisableIrqLoadBalancing is false + globallyDisableIrqLoadBalancing: false + hugepages: + defaultHugepagesSize: 1G + pages: + {{- range .spec.hugepages.pages }} + - size: {{ .size }} + count: {{ .count }} + {{if ne (index . "node") nil }} + node: {{ .node }} + {{- end }} + {{- end }} + {{- if hasKey .spec "machineConfigPoolSelector" }} + machineConfigPoolSelector: + {{- .spec.machineConfigPoolSelector | toYaml | nindent 4 }} +# # For SNO: machineconfiguration.openshift.io/role: 'master' +# pools.operator.machineconfiguration.openshift.io/worker: '' + {{- end }} + {{- if hasKey .spec "nodeSelector" }} + nodeSelector: + {{- .spec.nodeSelector | toYaml | nindent 4 }} +# # For SNO: node-role.kubernetes.io/master: "" +# node-role.kubernetes.io/worker: "" + {{- end }} + workloadHints: + realTime: false + highPowerConsumption: false + perPodPowerManagement: true + realTimeKernel: + enabled: false + numa: + # All guaranteed QoS containers get resources from a single NUMA node + topologyPolicy: "single-numa-node" + {{- if hasKey .spec.net "userLevelNetworking" }} + net: + userLevelNetworking: false + {{- end }} diff --git a/telco-core/configuration/reference-crs-kube-compare/required/scheduling/NROPSubscription.yaml b/telco-core/configuration/reference-crs-kube-compare/required/scheduling/NROPSubscription.yaml new file mode 100644 index 00000000..c8dcbb16 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/scheduling/NROPSubscription.yaml @@ -0,0 +1,12 @@ +# required +# count: 1 +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: numaresources-operator + namespace: openshift-numaresources +spec: + channel: "{{ .spec.channel }}" + name: numaresources-operator + source: {{ if .spec.source }} {{ .spec.source }} {{ else }} "redhat-operators-disconnected " {{ end }} + sourceNamespace: {{ if .spec.sourceNamespace }} {{ .spec.sourceNamespace }} {{ else }} "openshift-marketplace" {{ end }} diff --git a/telco-core/configuration/reference-crs-kube-compare/required/scheduling/NROPSubscriptionNS.yaml b/telco-core/configuration/reference-crs-kube-compare/required/scheduling/NROPSubscriptionNS.yaml new file mode 100644 index 00000000..e35801a9 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/scheduling/NROPSubscriptionNS.yaml @@ -0,0 +1,6 @@ +# required: yes +# count: 1 +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-numaresources diff --git a/telco-core/configuration/reference-crs-kube-compare/required/scheduling/NROPSubscriptionOperGroup.yaml b/telco-core/configuration/reference-crs-kube-compare/required/scheduling/NROPSubscriptionOperGroup.yaml new file mode 100644 index 00000000..619582a0 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/scheduling/NROPSubscriptionOperGroup.yaml @@ -0,0 +1,10 @@ +# required: yes +# count: 1 +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: numaresources-operator + namespace: openshift-numaresources +spec: + targetNamespaces: + - openshift-numaresources diff --git a/telco-core/configuration/reference-crs-kube-compare/required/scheduling/Scheduler.yaml b/telco-core/configuration/reference-crs-kube-compare/required/scheduling/Scheduler.yaml new file mode 100644 index 00000000..89bae112 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/scheduling/Scheduler.yaml @@ -0,0 +1,10 @@ +apiVersion: config.openshift.io/v1 +kind: Scheduler +metadata: + name: cluster +spec: + # non-schedulable control plane is the default. This ensures + # compliance. + mastersSchedulable: false + policy: + name: "" diff --git a/telco-core/configuration/reference-crs-kube-compare/required/scheduling/nrop.yaml b/telco-core/configuration/reference-crs-kube-compare/required/scheduling/nrop.yaml new file mode 100644 index 00000000..3e9b8a68 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/scheduling/nrop.yaml @@ -0,0 +1,24 @@ +# Optional +# count: 1 +apiVersion: nodetopology.openshift.io/v1 +kind: NUMAResourcesOperator +metadata: + name: numaresourcesoperator +spec: + nodeGroups: + {{- if .spec.nodeGroups }} + {{- range .spec.nodeGroups }} + - config: + # Periodic is the default setting + infoRefreshMode: Periodic + {{- if .machineConfigPoolSelector }} + machineConfigPoolSelector: + {{- .machineConfigPoolSelector | toYaml | nindent 8 }} + {{- end }} +# matchLabels: +# # This label must match the pool(s) you want to run NUMA-aligned workloads +# pools.operator.machineconfiguration.openshift.io/worker: "" + {{- end }} + {{- else }} + [] + {{- end }} diff --git a/telco-core/configuration/reference-crs-kube-compare/required/scheduling/sched.yaml b/telco-core/configuration/reference-crs-kube-compare/required/scheduling/sched.yaml new file mode 100644 index 00000000..bf2d1bbf --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/scheduling/sched.yaml @@ -0,0 +1,12 @@ +# Optional +# count: 1 +apiVersion: nodetopology.openshift.io/v1 +kind: NUMAResourcesScheduler +metadata: + name: numaresourcesscheduler +spec: + #cacheResyncPeriod: "0" + # Image spec should be the latest for the release + imageSpec: {{ .spec.imageSpec }} # "registry.redhat.io/openshift4/noderesourcetopology-scheduler-rhel9:v4.14.0" + #logLevel: "Trace" + schedulerName: topo-aware-scheduler diff --git a/telco-core/configuration/reference-crs-kube-compare/required/storage/odf-external/01-rook-ceph-external-cluster-details.secret.yaml b/telco-core/configuration/reference-crs-kube-compare/required/storage/odf-external/01-rook-ceph-external-cluster-details.secret.yaml new file mode 100644 index 00000000..7db9d44f --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/storage/odf-external/01-rook-ceph-external-cluster-details.secret.yaml @@ -0,0 +1,12 @@ +# required +# count: 1 +--- +apiVersion: v1 +kind: Secret +metadata: + name: rook-ceph-external-cluster-details + namespace: openshift-storage +type: Opaque +data: + # encoded content has been made generic + external_cluster_details: {{ .data.external_cluster_details }} diff --git a/telco-core/configuration/reference-crs-kube-compare/required/storage/odf-external/02-ocs-external-storagecluster.yaml b/telco-core/configuration/reference-crs-kube-compare/required/storage/odf-external/02-ocs-external-storagecluster.yaml new file mode 100644 index 00000000..a8126230 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/storage/odf-external/02-ocs-external-storagecluster.yaml @@ -0,0 +1,14 @@ +# required +# count: 1 +--- +apiVersion: ocs.openshift.io/v1 +kind: StorageCluster +metadata: + name: ocs-external-storagecluster + namespace: openshift-storage +spec: + externalStorage: + enable: true + labelSelector: {} +status: + phase: Ready diff --git a/telco-core/configuration/reference-crs-kube-compare/required/storage/odf-external/odfNS.yaml b/telco-core/configuration/reference-crs-kube-compare/required/storage/odf-external/odfNS.yaml new file mode 100644 index 00000000..12e7a47e --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/storage/odf-external/odfNS.yaml @@ -0,0 +1,9 @@ +# required: yes +# count: 1 +--- +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-storage + labels: + openshift.io/cluster-monitoring: "true" diff --git a/telco-core/configuration/reference-crs-kube-compare/required/storage/odf-external/odfOperGroup.yaml b/telco-core/configuration/reference-crs-kube-compare/required/storage/odf-external/odfOperGroup.yaml new file mode 100644 index 00000000..4efc3206 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/storage/odf-external/odfOperGroup.yaml @@ -0,0 +1,11 @@ +# required: yes +# count: 1 +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: openshift-storage-operatorgroup + namespace: openshift-storage +spec: + targetNamespaces: + - openshift-storage diff --git a/telco-core/configuration/reference-crs-kube-compare/required/storage/odf-external/odfSubscription.yaml b/telco-core/configuration/reference-crs-kube-compare/required/storage/odf-external/odfSubscription.yaml new file mode 100644 index 00000000..3311faf3 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/required/storage/odf-external/odfSubscription.yaml @@ -0,0 +1,20 @@ +# required: yes +# count: 1 +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: odf-operator + namespace: openshift-storage +spec: + {{- if and (.spec.channel) (hasPrefix "stable" .spec.channel) }} + channel: {{ .spec.channel }} + {{- else }} + channel: "stable-.*" + {{- end }} + name: odf-operator + source: redhat-operators-disconnected + sourceNamespace: openshift-marketplace + installPlanApproval: Manual +status: + state: AtLatestKnown diff --git a/telco-core/configuration/reference-crs-kube-compare/version_match.tmpl b/telco-core/configuration/reference-crs-kube-compare/version_match.tmpl new file mode 100644 index 00000000..c5af9af7 --- /dev/null +++ b/telco-core/configuration/reference-crs-kube-compare/version_match.tmpl @@ -0,0 +1,9 @@ +{{- define "versionMatch" }} + {{- $version := semver (index . 0 | default "0.0.0") }} + {{- $target := semver (index . 1) }} + {{- $result := print ($target.Original) ".*" }} + {{- if and (eq $version.Major $target.Major) (eq $version.Minor $target.Minor) }} + {{- $result = $version.Original }} + {{- end }} + {{- $result }} +{{- end }} diff --git a/telco-core/configuration/reference-crs/optional/other/ClusterVersion.yaml b/telco-core/configuration/reference-crs/optional/other/ClusterVersion.yaml new file mode 100644 index 00000000..0689c185 --- /dev/null +++ b/telco-core/configuration/reference-crs/optional/other/ClusterVersion.yaml @@ -0,0 +1,5 @@ +# Placeholder to allow cluster-compare to have a custom ClusterVersion to enforce version-checking +apiVersion: config.openshift.io/v1 +kind: ClusterVersion +metadata: + name: version