From 47b8feeaad101bafa7c88b2a58d28b65cbef28d0 Mon Sep 17 00:00:00 2001 From: e-minguez Date: Tue, 24 Sep 2024 16:47:27 +0200 Subject: [PATCH] Replaced Bird by native MetalLB --- examples/demo_cluster/variables.tf | 14 +- main.tf | 12 +- .../{k3s_cluster => kube_cluster}/README.md | 0 modules/{k3s_cluster => kube_cluster}/main.tf | 40 +-- .../{k3s_cluster => kube_cluster}/outputs.tf | 4 +- .../templates/user-data.tftpl | 299 ++++++++++++------ .../variables.tf | 32 +- .../{k3s_cluster => kube_cluster}/versions.tf | 0 outputs.tf | 6 +- variables.tf | 14 +- 10 files changed, 262 insertions(+), 159 deletions(-) rename modules/{k3s_cluster => kube_cluster}/README.md (100%) rename modules/{k3s_cluster => kube_cluster}/main.tf (84%) rename modules/{k3s_cluster => kube_cluster}/outputs.tf (71%) rename modules/{k3s_cluster => kube_cluster}/templates/user-data.tftpl (65%) rename modules/{k3s_cluster => kube_cluster}/variables.tf (69%) rename modules/{k3s_cluster => kube_cluster}/versions.tf (100%) diff --git a/examples/demo_cluster/variables.tf b/examples/demo_cluster/variables.tf index 527256d..7e4dee3 100644 --- a/examples/demo_cluster/variables.tf +++ b/examples/demo_cluster/variables.tf @@ -22,20 +22,20 @@ variable "deploy_demo" { } variable "clusters" { - description = "K3s cluster definition" + description = "Cluster definition" type = list(object({ - name = optional(string, "K3s demo cluster") + name = optional(string, "Demo cluster") metro = optional(string, "FR") plan_control_plane = optional(string, "c3.small.x86") plan_node = optional(string, "c3.small.x86") node_count = optional(number, 0) - k3s_ha = optional(bool, false) + ha = optional(bool, false) os = optional(string, "debian_11") - control_plane_hostnames = optional(string, "k3s-cp") - node_hostnames = optional(string, "k3s-node") - custom_k3s_token = optional(string, "") + control_plane_hostnames = optional(string, "cp") + node_hostnames = optional(string, "node") + custom_token = optional(string, "") ip_pool_count = optional(number, 0) - k3s_version = optional(string, "") + kube_version = optional(string, "") metallb_version = optional(string, "") })) default = [{}] diff --git a/main.tf b/main.tf index d64c0e7..d8cb435 100644 --- a/main.tf +++ b/main.tf @@ -5,11 +5,11 @@ locals { } ################################################################################ -# K3S Cluster In-line Module +# K8s Cluster In-line Module ################################################################################ -module "k3s_cluster" { - source = "./modules/k3s_cluster" +module "kube_cluster" { + source = "./modules/kube_cluster" for_each = { for cluster in var.clusters : cluster.name => cluster } @@ -18,12 +18,12 @@ module "k3s_cluster" { plan_control_plane = each.value.plan_control_plane plan_node = each.value.plan_node node_count = each.value.node_count - k3s_ha = each.value.k3s_ha + ha = each.value.ha os = each.value.os control_plane_hostnames = each.value.control_plane_hostnames node_hostnames = each.value.node_hostnames - custom_k3s_token = each.value.custom_k3s_token - k3s_version = each.value.k3s_version + custom_token = each.value.custom_token + kube_version = each.value.kube_version metallb_version = each.value.metallb_version ip_pool_count = each.value.ip_pool_count metal_project_id = var.metal_project_id diff --git a/modules/k3s_cluster/README.md b/modules/kube_cluster/README.md similarity index 100% rename from modules/k3s_cluster/README.md rename to modules/kube_cluster/README.md diff --git a/modules/k3s_cluster/main.tf b/modules/kube_cluster/main.tf similarity index 84% rename from modules/k3s_cluster/main.tf rename to modules/kube_cluster/main.tf index b9906fa..2572c0e 100644 --- a/modules/k3s_cluster/main.tf +++ b/modules/kube_cluster/main.tf @@ -1,10 +1,10 @@ locals { - k3s_token = coalesce(var.custom_k3s_token, random_string.random_k3s_token.result) - api_vip = var.k3s_ha ? equinix_metal_reserved_ip_block.api_vip_addr[0].address : equinix_metal_device.all_in_one[0].network[0].address + token = coalesce(var.custom_token, random_string.random_token.result) + api_vip = var.ha ? equinix_metal_reserved_ip_block.api_vip_addr[0].address : equinix_metal_device.all_in_one[0].network[0].address ip_pool_cidr = var.ip_pool_count > 0 ? equinix_metal_reserved_ip_block.ip_pool[0].cidr_notation : "" } -resource "random_string" "random_k3s_token" { +resource "random_string" "random_token" { length = 16 special = false } @@ -20,14 +20,14 @@ resource "equinix_metal_device" "control_plane_master" { operating_system = var.os billing_cycle = "hourly" project_id = var.metal_project_id - count = var.k3s_ha ? 1 : 0 + count = var.ha ? 1 : 0 description = var.cluster_name user_data = templatefile("${path.module}/templates/user-data.tftpl", { - k3s_token = local.k3s_token, + token = local.token, API_IP = local.api_vip, global_ip_cidr = var.global_ip_cidr, ip_pool = local.ip_pool_cidr, - k3s_version = var.k3s_version, + kube_version = var.kube_version, metallb_version = var.metallb_version, deploy_demo = var.deploy_demo, node_type = "control-plane-master" }) @@ -36,16 +36,16 @@ resource "equinix_metal_device" "control_plane_master" { resource "equinix_metal_bgp_session" "control_plane_master" { device_id = equinix_metal_device.control_plane_master[0].id address_family = "ipv4" - count = var.k3s_ha ? 1 : 0 + count = var.ha ? 1 : 0 } resource "equinix_metal_reserved_ip_block" "api_vip_addr" { - count = var.k3s_ha ? 1 : 0 + count = var.ha ? 1 : 0 project_id = var.metal_project_id metro = var.metal_metro type = "public_ipv4" quantity = 1 - description = "K3s API IP" + description = "Kubernetes API IP" } resource "equinix_metal_device" "control_plane_others" { @@ -55,15 +55,15 @@ resource "equinix_metal_device" "control_plane_others" { operating_system = var.os billing_cycle = "hourly" project_id = var.metal_project_id - count = var.k3s_ha ? 2 : 0 + count = var.ha ? 2 : 0 description = var.cluster_name depends_on = [equinix_metal_device.control_plane_master] user_data = templatefile("${path.module}/templates/user-data.tftpl", { - k3s_token = local.k3s_token, + token = local.token, API_IP = local.api_vip, global_ip_cidr = "", ip_pool = "", - k3s_version = var.k3s_version, + kube_version = var.kube_version, metallb_version = var.metallb_version, deploy_demo = false, node_type = "control-plane" }) @@ -72,13 +72,13 @@ resource "equinix_metal_device" "control_plane_others" { resource "equinix_metal_bgp_session" "control_plane_second" { device_id = equinix_metal_device.control_plane_others[0].id address_family = "ipv4" - count = var.k3s_ha ? 1 : 0 + count = var.ha ? 1 : 0 } resource "equinix_metal_bgp_session" "control_plane_third" { device_id = equinix_metal_device.control_plane_others[1].id address_family = "ipv4" - count = var.k3s_ha ? 1 : 0 + count = var.ha ? 1 : 0 } ################################################################################ @@ -109,11 +109,11 @@ resource "equinix_metal_device" "nodes" { description = var.cluster_name depends_on = [equinix_metal_device.control_plane_master] user_data = templatefile("${path.module}/templates/user-data.tftpl", { - k3s_token = local.k3s_token, + token = local.token, API_IP = local.api_vip, global_ip_cidr = "", ip_pool = "", - k3s_version = var.k3s_version, + kube_version = var.kube_version, metallb_version = var.metallb_version, deploy_demo = false, node_type = "node" }) @@ -130,14 +130,14 @@ resource "equinix_metal_device" "all_in_one" { operating_system = var.os billing_cycle = "hourly" project_id = var.metal_project_id - count = var.k3s_ha ? 0 : 1 + count = var.ha ? 0 : 1 description = var.cluster_name user_data = templatefile("${path.module}/templates/user-data.tftpl", { - k3s_token = local.k3s_token, + token = local.token, global_ip_cidr = var.global_ip_cidr, ip_pool = local.ip_pool_cidr, API_IP = "", - k3s_version = var.k3s_version, + kube_version = var.kube_version, metallb_version = var.metallb_version, deploy_demo = var.deploy_demo, node_type = "all-in-one" }) @@ -147,5 +147,5 @@ resource "equinix_metal_device" "all_in_one" { resource "equinix_metal_bgp_session" "all_in_one" { device_id = equinix_metal_device.all_in_one[0].id address_family = "ipv4" - count = var.k3s_ha ? 0 : 1 + count = var.ha ? 0 : 1 } diff --git a/modules/k3s_cluster/outputs.tf b/modules/kube_cluster/outputs.tf similarity index 71% rename from modules/k3s_cluster/outputs.tf rename to modules/kube_cluster/outputs.tf index 7e6cb62..8d83951 100644 --- a/modules/k3s_cluster/outputs.tf +++ b/modules/kube_cluster/outputs.tf @@ -1,4 +1,4 @@ -output "k3s_api_ip" { +output "kube_api_ip" { value = try(equinix_metal_reserved_ip_block.api_vip_addr[0].address, equinix_metal_device.all_in_one[0].network[0].address) - description = "K3s API IPs" + description = "K8s API IPs" } diff --git a/modules/k3s_cluster/templates/user-data.tftpl b/modules/kube_cluster/templates/user-data.tftpl similarity index 65% rename from modules/k3s_cluster/templates/user-data.tftpl rename to modules/kube_cluster/templates/user-data.tftpl index 6980e0a..1a8f3e5 100644 --- a/modules/k3s_cluster/templates/user-data.tftpl +++ b/modules/kube_cluster/templates/user-data.tftpl @@ -1,104 +1,155 @@ #!/usr/bin/env bash set -euo pipefail -wait_for_k3s_api(){ +die(){ + echo $${1} >&2 + exit $${2} +} + +prechecks(){ + # Set OS + source /etc/os-release + case $${ID} in + "debian") + export PKGMANAGER="apt" + ;; + "sles") + export PKGMANAGER="zypper" + ;; + "sle-micro") + export PKGMANAGER="transactional-update" + ;; + *) + die "Unsupported OS $${ID}" 1 + ;; + esac + # Set ARCH + ARCH=$(uname -m) + case $${ARCH} in + "amd64") + export ARCH=amd64 + export SUFFIX= + ;; + "x86_64") + export ARCH=amd64 + export SUFFIX= + ;; + "arm64") + export ARCH=arm64 + export SUFFIX=-$${ARCH} + ;; + "s390x") + export ARCH=s390x + export SUFFIX=-$${ARCH} + ;; + "aarch64") + export ARCH=arm64 + export SUFFIX=-$${ARCH} + ;; + "arm*") + export ARCH=arm + export SUFFIX=-$${ARCH}hf + ;; + *) + die "Unsupported architecture $${ARCH}" 1 + ;; + esac +} + +prereqs(){ + # Required packages + case $${PKGMANAGER} in + "apt") + apt update + apt install -y jq curl + ;; + "zypper") + zypper refresh + zypper install -y jq curl + ;; + esac +} + +wait_for_kube_api(){ # Wait for the node to be available, meaning the K8s API is available while ! kubectl wait --for condition=ready node $(cat /etc/hostname | tr '[:upper:]' '[:lower:]') --timeout=60s; do sleep 2 ; done } -install_bird(){ - # Install bird - apt update && apt install bird jq -y +install_eco(){ + # Wait for K3s to be up. It should be up already but just in case. + wait_for_kube_api - # In order to configure bird, the metadata information is required. - # BGP info can take a few seconds to be populated, retry if that's the case - INTERNAL_IP="null" - while [ $${INTERNAL_IP} == "null" ]; do - echo "BGP data still not available..." - sleep 5 - METADATA=$(curl -s https://metadata.platformequinix.com/metadata) - INTERNAL_IP=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].customer_ip') - done - PEER_IP_1=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].peer_ips[0]') - PEER_IP_2=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].peer_ips[1]') - ASN=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].customer_as') - ASN_AS=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].peer_as') - MULTIHOP=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].multihop') - GATEWAY=$(echo $${METADATA} | jq -r '.network.addresses[] | select(.public == true and .address_family == 4) | .gateway') - - # Generate the bird configuration based on the metadata values - # https://deploy.equinix.com/developers/guides/configuring-bgp-with-bird/ - cat <<-EOF >/etc/bird/bird.conf - router id $${INTERNAL_IP}; - - protocol direct { - interface "lo"; - } - - protocol kernel { - persist; - scan time 60; - import all; - export all; - } - - protocol device { - scan time 60; - } - - protocol static { - route $${PEER_IP_1}/32 via $${GATEWAY}; - route $${PEER_IP_2}/32 via $${GATEWAY}; - } - - filter metal_bgp { - accept; - } - - protocol bgp neighbor_v4_1 { - export filter metal_bgp; - local as $${ASN}; - multihop; - neighbor $${PEER_IP_1} as $${ASN_AS}; - } - - protocol bgp neighbor_v4_2 { - export filter metal_bgp; - local as $${ASN}; - multihop; - neighbor $${PEER_IP_2} as $${ASN_AS}; - } - EOF + # Download helm as required to install endpoint-copier-operator + command -v helm || curl -fsSL https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 |bash - # Wait for K3s to be up, otherwise the second and third control plane nodes will try to join localhost - wait_for_k3s_api + # Add the SUSE Edge charts and deploy ECO + helm repo add suse-edge https://suse-edge.github.io/charts + helm install --create-namespace -n endpoint-copier-operator endpoint-copier-operator suse-edge/endpoint-copier-operator - # Configure the BGP interface - # https://deploy.equinix.com/developers/guides/configuring-bgp-with-bird/ - if ! grep -q 'lo:0' /etc/network/interfaces; then - cat <<-EOF >>/etc/network/interfaces + # Configure the MetalLB IP Address pool for the VIP + cat <<-EOF | kubectl apply -f - + apiVersion: metallb.io/v1beta1 + kind: IPAddressPool + metadata: + name: kubernetes-vip-ip-pool + namespace: metallb-system + spec: + addresses: + - ${API_IP}/32 + serviceAllocation: + priority: 100 + namespaces: + - default + EOF - auto lo:0 - iface lo:0 inet static - address ${API_IP} - netmask 255.255.255.255 + # Create the kubernetes-vip service that will be updated by e-c-o with the control plane hosts + if [[ $${KUBETYPE} == "k3s" ]]; then + cat <<-EOF | kubectl apply -f - + apiVersion: v1 + kind: Service + metadata: + name: kubernetes-vip + namespace: default + spec: + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: k8s-api + port: 6443 + protocol: TCP + targetPort: 6443 + type: LoadBalancer + EOF + fi + if [[ $${KUBETYPE} == "rke2" ]]; then + cat <<-EOF | kubectl apply -f - + apiVersion: v1 + kind: Service + metadata: + name: kubernetes-vip + namespace: default + spec: + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: k8s-api + port: 6443 + protocol: TCP + targetPort: 6443 + - name: rke2-api + port: 9345 + protocol: TCP + targetPort: 9345 + type: LoadBalancer EOF - ifup lo:0 fi - - # Enable IP forward for bird - # TODO: Check if this is done automatically with K3s, it doesn't hurt however - echo "net.ipv4.ip_forward=1" | tee /etc/sysctl.d/99-ip-forward.conf - sysctl --load /etc/sysctl.d/99-ip-forward.conf - - # Debian usually starts the service after being installed, but just in case - systemctl enable bird - systemctl restart bird } install_metallb(){ - apt update && apt install -y curl jq - %{ if metallb_version != "" ~} export METALLB_VERSION=${metallb_version} %{ else ~} @@ -106,7 +157,7 @@ install_metallb(){ %{ endif ~} # Wait for K3s to be up. It should be up already but just in case. - wait_for_k3s_api + wait_for_kube_api # Apply the MetalLB manifest kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/$${METALLB_VERSION}/config/manifests/metallb-native.yaml @@ -200,8 +251,6 @@ install_metallb(){ } install_k3s(){ - apt update && apt install curl -y - # Download the K3s installer script curl -L --output k3s_installer.sh https://get.k3s.io && install -m755 k3s_installer.sh /usr/local/bin/ @@ -215,8 +264,9 @@ install_k3s(){ while ! curl -m 10 -s -k -o /dev/null https://${API_IP}:6443 ; do echo "API still not reachable"; sleep 2 ; done %{ endif ~} + export INSTALL_K3S_SKIP_ENABLE=false export INSTALL_K3S_SKIP_START=false - export K3S_TOKEN="${k3s_token}" + export K3S_TOKEN="${token}" %{ if node_type == "all-in-one" ~} %{ if global_ip_cidr != "" ~} export INSTALL_K3S_EXEC="server --write-kubeconfig-mode=644 --disable=servicelb" @@ -237,16 +287,20 @@ install_k3s(){ %{ if node_type == "node" ~} export K3S_URL="https://${API_IP}:6443" %{ endif ~} -%{ if k3s_version != "" ~} - export INSTALL_K3S_VERSION=${k3s_version} +%{ if kube_version != "" ~} + export INSTALL_K3S_VERSION="${kube_version}" %{ endif ~} /usr/local/bin/k3s_installer.sh +} - systemctl enable --now k3s +install_rke2(){ + echo "Not yet" } deploy_demo(){ - kubectl annotate svc -n kube-system traefik "metallb.universe.tf/address-pool=anycast-ip" + # Check if the demo is already deployed + if kubectl get deployment -n hello-kubernetes hello-kubernetes -o name > /dev/null 2>&1; then exit 0; fi + [[ $${KUBETYPE} == "k3s" ]] && kubectl annotate svc -n kube-system traefik "metallb.universe.tf/address-pool=anycast-ip" # I cannot make split work in Terraform templates IP=$(echo ${global_ip_cidr} | cut -d/ -f1) @@ -356,29 +410,66 @@ deploy_demo(){ EOF } -install_k3s +setup_kubectl(){ + case $${KUBETYPE} in + "k3s") + alias kubectl="/usr/local/bin/kubectl" + alias k="kubectl" + export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + ;; + "rke2") + alias kubectl="/var/lib/rancher/rke2/bin/kubectl" + alias k="kubectl" + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + ;; + *) + die "Kubernetes type $${KUBETYPE} not found" 2 + ;; + esac +} + +prechecks +prereqs + +if [[ "${kube_version}" =~ .*"k3s".* ]] || [[ "${kube_version}" == "" ]]; then + export KUBETYPE="k3s" + install_k3s +elif [[ "${kube_version}" =~ .*"rke2".* ]]; then + export KUBETYPE="rke2" + install_rke2 +else + die "Kubernetes version ${kube_version} not valid" 2 +fi +setup_kubectl + +DEPLOY_DEMO=false +INSTALL_METALLB=false %{ if node_type == "control-plane-master" ~} -install_bird -install_metallb +INSTALL_METALLB=true +%{ if deploy_demo != "" ~} +DEPLOY_DEMO=true %{ endif ~} -%{ if node_type == "control-plane" ~} -install_bird -install_metallb %{ endif ~} %{ if node_type == "all-in-one" ~} %{ if global_ip_cidr != "" ~} INSTALL_METALLB=true -%{ else } +%{ endif } %{ if ip_pool != "" ~} INSTALL_METALLB=true -%{ else } -INSTALL_METALLB=false +%{ endif } +%{ if deploy_demo != "" ~} +DEPLOY_DEMO=true %{ endif ~} %{ endif ~} + [ $${INSTALL_METALLB} == true ] && install_metallb || true + +%{ if API_IP != "" ~} +%{ if node_type == "control-plane-master" ~} +install_eco %{ endif ~} -%{ if deploy_demo != "" ~} -deploy_demo %{ endif ~} + +[ $${DEPLOY_DEMO} == true ] && deploy_demo || true diff --git a/modules/k3s_cluster/variables.tf b/modules/kube_cluster/variables.tf similarity index 69% rename from modules/k3s_cluster/variables.tf rename to modules/kube_cluster/variables.tf index c3860a4..88ac6a3 100644 --- a/modules/k3s_cluster/variables.tf +++ b/modules/kube_cluster/variables.tf @@ -17,30 +17,30 @@ variable "deploy_demo" { variable "cluster_name" { type = string description = "Cluster name" - default = "K3s cluster" + default = "Cluster" } variable "plan_control_plane" { type = string - description = "K3s control plane type/size" + description = "Control plane type/size" default = "c3.small.x86" } variable "plan_node" { type = string - description = "K3s node type/size" + description = "Node type/size" default = "c3.small.x86" } variable "node_count" { type = number - description = "Number of K3s nodes" + description = "Number of nodes" default = "0" } -variable "k3s_ha" { +variable "ha" { type = bool - description = "K3s HA (aka 3 control plane nodes)" + description = "HA (aka 3 control plane nodes)" default = false } @@ -62,9 +62,9 @@ variable "node_hostnames" { default = "node" } -variable "custom_k3s_token" { +variable "custom_token" { type = string - description = "K3s token used for nodes to join the cluster (autogenerated otherwise)" + description = "Token used for nodes to join the cluster (autogenerated otherwise)" default = null } @@ -80,9 +80,9 @@ variable "global_ip_cidr" { default = null } -variable "k3s_version" { +variable "kube_version" { type = string - description = "K3s version to be installed. Empty for latest" + description = "K3s/RKE2 version to be installed. Empty for latest K3s" default = "" } @@ -91,3 +91,15 @@ variable "metallb_version" { description = "MetalLB version to be installed. Empty for latest" default = "" } + +variable "rancher_version" { + type = string + description = "Rancher version to be installed (vX.Y.Z). Empty for latest" + default = "" +} + +variable "rancher_flavor" { + type = string + description = "Rancher flavor to be installed (prime, latest or stable). Empty to not install it" + default = "" +} \ No newline at end of file diff --git a/modules/k3s_cluster/versions.tf b/modules/kube_cluster/versions.tf similarity index 100% rename from modules/k3s_cluster/versions.tf rename to modules/kube_cluster/versions.tf diff --git a/outputs.tf b/outputs.tf index 432a199..efc6243 100644 --- a/outputs.tf +++ b/outputs.tf @@ -8,9 +8,9 @@ output "demo_url" { description = "URL of the demo application to demonstrate a global IP shared across Metros" } -output "k3s_api" { +output "kube_api" { value = { - for cluster in var.clusters : cluster.name => module.k3s_cluster[cluster.name].k3s_api_ip + for cluster in var.clusters : cluster.name => module.kube_cluster[cluster.name].kube_api_ip } - description = "List of Clusters => K3s APIs" + description = "List of Clusters => K8s APIs" } diff --git a/variables.tf b/variables.tf index 490abdb..f72562d 100644 --- a/variables.tf +++ b/variables.tf @@ -16,20 +16,20 @@ variable "deploy_demo" { } variable "clusters" { - description = "K3s cluster definition" + description = "Cluster definition" type = list(object({ - name = optional(string, "K3s demo cluster") + name = optional(string, "Demo cluster") metro = optional(string, "FR") plan_control_plane = optional(string, "c3.small.x86") plan_node = optional(string, "c3.small.x86") node_count = optional(number, 0) - k3s_ha = optional(bool, false) + ha = optional(bool, false) os = optional(string, "debian_11") - control_plane_hostnames = optional(string, "k3s-cp") - node_hostnames = optional(string, "k3s-node") - custom_k3s_token = optional(string, "") + control_plane_hostnames = optional(string, "cp") + node_hostnames = optional(string, "node") + custom_token = optional(string, "") ip_pool_count = optional(number, 0) - k3s_version = optional(string, "") + kube_version = optional(string, "") metallb_version = optional(string, "") })) default = [{}]