Merge pull request #453 from vshn/billing-managed

Adjust appuio managed billing
vshn · Aug 29, 2024 · 715fb51 · 715fb51
2 parents bb6e8f9 + b4c0253
commit 715fb51
Show file tree

Hide file tree

Showing 90 changed files with 4,645 additions and 30 deletions.
diff --git a/class/defaults.yml b/class/defaults.yml
@@ -147,13 +147,12 @@ parameters:
       cloudZone: ""
       instanceUOM: uom_uom_45_1e112771
       enableMockOrgInfo: false
-      salesOrder: "" # TODO: Update once the fact is available in Leutenant
+      salesOrder: ${facts:sales_order}
       vshn:
-        # The reporting for VSHN service should only run on APPUiO Cloud LPG2
         enableCronjobs: false
         schedule: 30 * * * *
         # Deploy metering prometheus rules
-        meteringRules: false
+        meteringRules: true
       cloud:
         secrets:
           exoscale:

diff --git a/component/billing.jsonnet b/component/billing.jsonnet
@@ -169,17 +169,16 @@ local generateCloudAndManaged = function(name)
       sla: 'guaranteed',
       type: 'cloud',
     },
-    // Currently appcat on appuio managed isn't billed, so we don't need the permutations
-    // {
-    //   query: managedQuery % 'besteffort',
-    //   sla: 'besteffort',
-    //   type: 'managed',
-    // },
-    // {
-    //   query: managedQuery % 'guaranteed',
-    //   sla: 'guaranteed',
-    //   type: 'managed',
-    // },
+    {
+      query: managedQuery % 'besteffort',
+      sla: 'besteffort',
+      type: 'managed',
+    },
+    {
+      query: managedQuery % 'guaranteed',
+      sla: 'guaranteed',
+      type: 'managed',
+    },
   ];
 
   std.flatMap(function(r) [ backfillCJ(name, r.query, r.sla, r.type) ], permutations);

diff --git a/component/cloud_billing.jsonnet b/component/cloud_billing.jsonnet
@@ -7,7 +7,7 @@ local kube = import 'lib/kube.libjsonnet';
 local com = import 'lib/commodore.libjsonnet';
 local collectorImage = '%(registry)s/%(repository)s:%(tag)s' % inv.parameters.appcat.images.collector;
 local component_name = 'billing-collector-cloudservices';
-local appuioManaged = if params.salesOrder == '' then false else true;
+local appuioManaged = inv.parameters.appcat.appuioManaged;
 
 local labels = {
   'app.kubernetes.io/name': component_name,

diff --git a/component/main.jsonnet b/component/main.jsonnet
@@ -115,7 +115,7 @@ local promQuery = std.strReplace(promQueryWithLabel, '{{TENANT_REPLACE}}', tenan
 
 local meteringQueryCloud = importstr 'promql/metering_cloud.promql';
 local meteringQueryManagedRaw = importstr 'promql/metering_managed.promql';
-local meteringQueryManaged = std.strReplace(meteringQueryManagedRaw, '{{salesOrder}}', params.salesOrder);
+local meteringQueryManaged = std.strReplace(meteringQueryManagedRaw, '{{salesOrderID}}', params.billing.salesOrder);
 
 local legacyBillingRule = std.prune(kube._Object('monitoring.coreos.com/v1', 'PrometheusRule', 'appcat-billing') {
   metadata+: {

diff --git a/tests/apiserver.yml b/tests/apiserver.yml
@@ -8,6 +8,7 @@ parameters:
   facts:
     cloud: cloudscale
     distribution: openshift4
+    sales_order: "10431"
 
   global:
     appuio_metered_billing_zone_label_map:

diff --git a/tests/billing.yml b/tests/billing.yml
@@ -0,0 +1,41 @@
+parameters:
+  kapitan:
+    dependencies:
+      - type: https
+        source: https://raw.githubusercontent.com/projectsyn/component-crossplane/v2.3.0/lib/crossplane.libsonnet
+        output_path: vendor/lib/crossplane.libsonnet
+      - type: https
+        source: https://raw.githubusercontent.com/appuio/component-openshift4-operators/v1.4.0/lib/openshift4-operators.libsonnet
+        output_path: vendor/lib/openshift4-operators.libsonnet
+
+  facts:
+    cloud: cloudscale
+    sales_order: "10431"
+
+  global:
+    appuio_metered_billing_zone_label_map:
+      c-green-test-1234: 'Kind - Local Test 0'
+
+  crossplane:
+    namespace: syn-crossplane
+
+  appcat:
+
+    grpcEndpoint: host.docker.internal:9443
+    proxyFunction: false
+
+    quotasEnabled: false
+    appuioManaged: true
+    billing:
+      salesOrder: ST10120
+      vshn:
+        enableCronjobs: true
+        meteringRules: true
+      enableMockOrgInfo: true
+      instanceUOM: uom_uom_45_1e112771
+      network_policies:
+        target_namespaces:
+          vshn-appuio-mimir: false
+      prometheus:
+        url: http://prometheus-operated.prometheus-system:9090/prometheus
+      cloudZone: ${global:appuio_metered_billing_zone_label_map:${cluster:name}}
diff --git a/tests/cloudscale-metrics-collector-cloud.yml b/tests/cloudscale-metrics-collector-cloud.yml
@@ -11,6 +11,7 @@ parameters:
 
   facts:
     cloud: cloudscale
+    sales_order: "10431"
 
   global:
     appuio_metered_billing_zone_label_map:

diff --git a/tests/cloudscale-metrics-collector-managed.yml b/tests/cloudscale-metrics-collector-managed.yml
@@ -11,14 +11,14 @@ parameters:
 
   facts:
     cloud: cloudscale
+    sales_order: "10431"
 
   global:
     appuio_metered_billing_zone_label_map:
       c-green-test-1234: 'Kind - Local Test 0'
 
   appcat:
     billing:
-      salesOrderID: "12345"
       cloud:
         cloudscale:
           enabled: true

diff --git a/tests/cloudscale.yml b/tests/cloudscale.yml
@@ -7,6 +7,7 @@ parameters:
 
   facts:
     cloud: cloudscale
+    sales_order: ""
 
   global:
     appuio_metered_billing_zone_label_map:
@@ -17,6 +18,7 @@ parameters:
     namespace: syn-crossplane
 
   appcat:
+    appuioManaged: false
     apiserver:
       env:
         APPCAT_HANDLER_ENABLED: "true"

diff --git a/tests/controllers.yml b/tests/controllers.yml
@@ -8,6 +8,7 @@ parameters:
   facts:
     cloud: cloudscale
     distribution: openshift4
+    sales_order: "10431"
 
   global:
     appuio_metered_billing_zone_label_map:

diff --git a/tests/defaults.yml b/tests/defaults.yml
@@ -7,6 +7,7 @@ parameters:
 
   facts:
     cloud: cloudscale
+    sales_order: "10431"
 
   global:
     appuio_metered_billing_zone_label_map:

diff --git a/tests/exoscale-metrics-collector-cloud.yml b/tests/exoscale-metrics-collector-cloud.yml
@@ -11,6 +11,7 @@ parameters:
 
   facts:
     cloud: exoscale
+    sales_order: "10431"
 
   global:
     appuio_metered_billing_zone_label_map:

diff --git a/tests/exoscale-metrics-collector-managed.yml b/tests/exoscale-metrics-collector-managed.yml
@@ -11,6 +11,7 @@ parameters:
 
   facts:
     cloud: exoscale
+    sales_order: "10431"
 
   global:
     appuio_metered_billing_zone_label_map:

diff --git a/tests/exoscale.yml b/tests/exoscale.yml
@@ -7,6 +7,7 @@ parameters:
 
   facts:
     cloud: exoscale
+    sales_order: ""
 
   global:
     appuio_metered_billing_zone_label_map:
@@ -16,6 +17,7 @@ parameters:
     namespace: crossplane-system
 
   appcat:
+    appuioManaged: false
     providers:
       exoscale:
         enabled: true

diff --git a/tests/golden/apiserver/appcat/appcat/10_appcat_metering_recording_rule.yaml b/tests/golden/apiserver/appcat/appcat/10_appcat_metering_recording_rule.yaml
@@ -0,0 +1,38 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    name: appcat-metering
+  name: appcat-metering
+  namespace: syn-appcat
+spec:
+  groups:
+    - name: appcat-metering-rules
+      rules:
+        - expr: |
+            sum by (label_appcat_vshn_io_claim_namespace,label_appcat_vshn_io_sla,label_appuio_io_billing_name,label_appcat_vshn_io_claim_name,sales_order,label_appuio_io_billing_name) (
+              # Sum values over one hour and get mean
+              sum_over_time(
+                # Default appcat.vshn.io/sla to besteffort if it is not set
+                label_replace(
+                    # Fetch all namespaces with label label_appuio_io_billing_name=~"appcat-.+"
+                    (
+                      # We hardcode the tenant ID here, so that we have the label to do the join.
+                      label_replace(
+                          kube_namespace_labels{ label_appuio_io_billing_name=~"appcat-.+"} *
+                          on (namespace) group_right(label_appcat_vshn_io_claim_namespace,label_appcat_vshn_io_sla, label_appuio_io_billing_name,label_appcat_vshn_io_claim_name,sales_order,label_appuio_io_billing_name)
+                          kube_pod_info{created_by_kind!="Job"},
+                        "sales_order",
+                        "10431",
+                        "",
+                        ""
+                      )
+                    ),
+                  "label_appcat_vshn_io_sla",
+                  "besteffort",
+                  "label_appcat_vshn_io_sla",
+                  "^$"
+                )[60m:1m]
+              )/60
+            )
+          record: appcat:metering
diff --git a/tests/golden/billing/appcat/appcat/10_appcat_backup_monitoring.yaml b/tests/golden/billing/appcat/appcat/10_appcat_backup_monitoring.yaml
@@ -0,0 +1,21 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: appcat-backup
+  namespace: syn-appcat
+spec:
+  groups:
+    - name: appcat-backup
+      rules:
+        - alert: AppCatBackupJobError
+          annotations:
+            description: The backup job {{ $labels.job_name }} in namespace {{ $labels.namespace
+              }} has failed.
+            runbook_url: https://kb.vshn.ch/app-catalog/how-tos/appcat/AppCatBackupJobError.html
+            summary: AppCat service backup failed.
+          expr: kube_job_failed{job_name=~".*backup.*", namespace=~"vshn-(postgresql|redis)-.*"}
+            > 0
+          for: 1m
+          labels:
+            severity: warning
+            syn_team: schedar
diff --git a/tests/golden/billing/appcat/appcat/10_appcat_ha_monitoring.yaml b/tests/golden/billing/appcat/appcat/10_appcat_ha_monitoring.yaml
@@ -0,0 +1,37 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: appcat-ha
+  namespace: syn-appcat
+spec:
+  groups:
+    - name: appcat-ha
+      rules:
+        - alert: AppCatHighAvailableDeploymentWarning
+          annotations:
+            description: The deployment {{ $labels.deployment }} in namespace {{ $labels.namespace
+              }} has less replicas than expected.
+            runbook_url: https://kb.vshn.ch/app-catalog/how-tos/appcat/vshn/AppCatHighAvailableDeploymentWarning.html
+            summary: AppCat service instance has unavailable pods.
+          expr: kube_deployment_status_replicas{namespace=~"vshn-(postgresql|redis)-.*"}
+            > 1 AND kube_deployment_status_replicas{namespace=~"vshn-(postgresql|redis)-.*"}
+            - kube_deployment_status_replicas_ready{namespace=~"vshn-(postgresql|redis)-.*"}
+            > 0
+          for: 1m
+          labels:
+            severity: warning
+            syn_team: schedar
+        - alert: AppCatHighAvailableStatefulsetWarning
+          annotations:
+            description: The statefulset {{ $labels.statefulset }} in namespace {{
+              $labels.namespace }} has less replicas than expected.
+            runbook_url: https://kb.vshn.ch/app-catalog/how-tos/appcat/vshn/AppCatHighAvailableStatefulsetWarning.html
+            summary: AppCat service instance has unavailable pods.
+          expr: kube_statefulset_status_replicas{namespace=~"vshn-(postgresql|redis)-.*"}
+            > 1 AND kube_statefulset_status_replicas{namespace=~"vshn-(postgresql|redis)-.*"}
+            - kube_statefulset_status_replicas_ready{namespace=~"vshn-(postgresql|redis)-.*"}
+            > 0
+          for: 1m
+          labels:
+            severity: warning
+            syn_team: schedar
diff --git a/tests/golden/billing/appcat/appcat/10_appcat_legacy_billing_recording_rule.yaml b/tests/golden/billing/appcat/appcat/10_appcat_legacy_billing_recording_rule.yaml
@@ -0,0 +1,84 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    name: appcat-billing
+  name: appcat-billing
+  namespace: syn-appcat
+spec:
+  groups:
+    - name: appcat-billing-rules
+      rules:
+        - expr: |
+            sum by (label_appcat_vshn_io_claim_namespace,label_appcat_vshn_io_sla,product,provider,architecture, category, claim_namespace,tenant_id) (
+            # Sum values over one hour and get mean
+            sum_over_time(
+              # Udpate label product: $product:$provider:$tenant_id:$claim_namespace:$architecture
+              label_join(
+                # Add label category: $provider:$claim_namespace
+                label_join(
+                  # Add label architecture: $SLA, where $SLA is the content of label appcat.vshn.io/sla
+                  label_replace(
+                    # Add label provider: vshn
+                    label_replace(
+                      # Add label product: postgres
+                      label_replace(
+                        # Default appcat.vshn.io/sla to besteffort if it is not set
+                        label_replace(
+                          # Copy label appcat.vshn.io/namespace to label claim_namespace
+                          label_replace(
+                            # Populate tenant_id
+                            label_replace(
+                              # Fetch all namespaces with label label_appuio_io_billing_name=~"appcat-.+"
+                              kube_namespace_labels{ label_appuio_io_billing_name=~"appcat-.+"} *
+                              on (namespace) group_right(label_appuio_io_organization,label_appcat_vshn_io_claim_namespace,label_appcat_vshn_io_sla, label_appuio_io_billing_name)
+                              kube_pod_info{created_by_kind!="Job"},
+                              "tenant_id",
+            "t-silent-test-1234",
+            "",
+            ""
+
+                            ),
+                            "claim_namespace",
+                            "$1",
+                            "label_appcat_vshn_io_claim_namespace",
+                            "(.*)"
+                          ),
+                          "label_appcat_vshn_io_sla",
+                          "besteffort",
+                          "label_appcat_vshn_io_sla",
+                          "^$"
+                        ),
+                        "product",
+                        "appcat_$1",
+                        "label_appuio_io_billing_name",
+                        "appcat-(.+)"
+                      ),
+                      "provider",
+                      "vshn",
+                      "",
+                      ""
+                    ),
+                    "sla",
+                    "$1",
+                    "label_appcat_vshn_io_sla",
+                    "(.*)"
+                  ),
+                  "category",
+                  ":",
+                  "provider",
+                  "claim_namespace"
+                ),
+                "product",
+                ":",
+                "product",
+                "provider",
+                "tenant_id",
+                "claim_namespace",
+                "sla"
+              # other billing queries have [59m:1m] here. This is due to some
+              # obscure discrepancies between how the cloud-reporting evaluates the query
+              # and how the GUI/recording rules evaluate the query.
+              )[60m:1m]
+            )/60 )
+          record: appcat:billing