diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 011d9e552..e8b50acb6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -91,14 +91,16 @@ This gem uses subclasses of `KubernetesResource` to implement custom success/fai 1. Create a file for your type in `lib/krane/kubernetes_resource/` 2. Create a new class that inherits from `KubernetesResource`. Minimally, it should implement the following methods: * `sync` -- Gather the data you'll need to determine `deploy_succeeded?` and `deploy_failed?`. The superclass's implementation fetches the corresponding resource, parses it and stores it in `@instance_data`. You can define your own implementation if you need something else. + * `predeployed?` -- Whether the resource should be [predeployed](README.md#phase-3-predeploying-priority-resources) by default. * `deploy_succeeded?` * `deploy_failed?` 3. Adjust the `TIMEOUT` constant to an appropriate value for this type. 4. Add the new class to list of resources in - [`deploy_task.rb`](https://github.com/Shopify/krane/blob/main/lib/krane/deploy_task.rb#L8) -5. Add the new resource to the [prune whitelist](https://github.com/Shopify/krane/blob/main/lib/krane/deploy_task.rb#L81) -6. Add a basic example of the type to the hello-cloud [fixture set](https://github.com/Shopify/krane/tree/main/test/fixtures/hello-cloud) and appropriate assertions to `#assert_all_up` in [`hello_cloud.rb`](https://github.com/Shopify/krane/blob/main/test/helpers/fixture_sets/hello_cloud.rb). This will get you coverage in several existing tests, such as `test_full_hello_cloud_set_deploy_succeeds`. -7. Add tests for any edge cases you foresee. + [`deploy_task.rb`](https://github.com/Shopify/krane/blob/main/lib/krane/deploy_task.rb#L12) +5. Add the new resource to the "after_crs" array in [`deploy_task.rb`](https://github.com/Shopify/krane/blob/main/lib/krane/deploy_task.rb#L72) +6. Add the new resource to the [prune whitelist](https://github.com/Shopify/krane/blob/main/lib/krane/deploy_task.rb#L93) +7. Add a basic example of the type to the hello-cloud [fixture set](https://github.com/Shopify/krane/tree/main/test/fixtures/hello-cloud) and appropriate assertions to `#assert_all_up` in [`hello_cloud.rb`](https://github.com/Shopify/krane/blob/main/test/helpers/fixture_sets/hello_cloud.rb). This will get you coverage in several existing tests, such as `test_full_hello_cloud_set_deploy_succeeds`. +8. Add tests for any edge cases you foresee. ### Contributor License Agreement diff --git a/README.md b/README.md index 11de54aa9..0a50e398e 100644 --- a/README.md +++ b/README.md @@ -167,11 +167,11 @@ before the deployment is considered successful. - Percent (e.g. 90%): The deploy is successful when the number of new pods that are ready is equal to `spec.replicas` * Percent. - _Compatibility_: StatefulSet - `full`: The deployment is successful when all pods are ready. -- `krane.shopify.io/predeployed`: Causes a Custom Resource to be deployed in the pre-deploy phase. - - _Compatibility_: Custom Resource Definition - - _Default_: `true` - - `true`: The custom resource will be deployed in the pre-deploy phase. - - All other values: The custom resource will be deployed in the main deployment phase. +- `krane.shopify.io/predeployed`: Causes a Custom Resource or [other resources](https://github.com/shopify/krane/tree/main/lib/krane/kubernetes_resource) to be deployed in the pre-deploy phase. + - _Compatibility_: CronJob, CustomResource, CustomResourceDefinition, DaemonSet, Deployment, HorizontalPodAutoscaler, Ingress, Job, Pod, PodDisruptionBudget, PodSetBase, PodTemplate, ReplicaSet, Service, StatefulSet + - _Default_: `false` + - `true`: The custom resource or other resource will be deployed in the pre-deploy phase. + - All other values: The custom resource or other resource will be deployed in the main deployment phase. - `krane.shopify.io/deploy-method-override`: Cause a resource to be deployed by the specified `kubectl` command, instead of the default `apply`. - _Compatibility_: Cannot be used for `PodDisruptionBudget`, since it always uses `create/replace-force` - _Accepted values_: `create`, `replace`, and `replace-force` diff --git a/lib/krane/deploy_task.rb b/lib/krane/deploy_task.rb index 4973b48ac..9380bbfc5 100644 --- a/lib/krane/deploy_task.rb +++ b/lib/krane/deploy_task.rb @@ -70,7 +70,19 @@ def predeploy_sequence ).map { |r| [r, default_group] } after_crs = %w( + Deployment + Service + Ingress Pod + Job + CronJob + DaemonSet + HorizontalPodAutoscaler + PodDisruptionBudget + PodSetBase + PodTemplate + ReplicaSet + StatefulSet ).map { |r| [r, default_group] } crs = cluster_resource_discoverer.crds.select(&:predeployed?).map { |cr| [cr.kind, { group: cr.group }] } diff --git a/lib/krane/kubernetes_resource.rb b/lib/krane/kubernetes_resource.rb index 2636ae9b0..eea0dfdca 100644 --- a/lib/krane/kubernetes_resource.rb +++ b/lib/krane/kubernetes_resource.rb @@ -381,6 +381,23 @@ def use_generated_name(instance_data) @file = create_definition_tempfile end + PREDEPLOYED_RESOURCE_TYPES = [ + "ResourceQuota", + "NetworkPolicy", + "ConfigMap", + "PersistentVolumeClaim", + "ServiceAccount", + "Role", + "RoleBinding", + "Secret", + "Pod" + ] + + def predeployed? + predeployed = krane_annotation_value("predeployed") + PREDEPLOYED_RESOURCE_TYPES.include?(type) || predeployed == "true" + end + class Event EVENT_SEPARATOR = "ENDEVENT--BEGINEVENT" FIELD_SEPARATOR = "ENDFIELD--BEGINFIELD" diff --git a/lib/krane/kubernetes_resource/custom_resource.rb b/lib/krane/kubernetes_resource/custom_resource.rb index 453ce3f8d..e5372aadb 100644 --- a/lib/krane/kubernetes_resource/custom_resource.rb +++ b/lib/krane/kubernetes_resource/custom_resource.rb @@ -62,6 +62,11 @@ def type kind end + def predeployed? + predeployed = krane_annotation_value("predeployed") + predeployed.nil? || predeployed == "true" + end + def validate_definition(*, **) super diff --git a/lib/krane/resource_deployer.rb b/lib/krane/resource_deployer.rb index 3c79ea6ba..f44998db7 100644 --- a/lib/krane/resource_deployer.rb +++ b/lib/krane/resource_deployer.rb @@ -49,7 +49,8 @@ def predeploy_priority_resources(resource_list, predeploy_sequence) predeploy_sequence.each do |resource_type, attributes| matching_resources = resource_list.select do |r| r.type == resource_type && - (!attributes[:group] || r.group == attributes[:group]) + (!attributes[:group] || r.group == attributes[:group]) && + r.predeployed? end StatsD.client.gauge('priority_resources.count', matching_resources.size, tags: statsd_tags) @@ -92,40 +93,42 @@ def deploy_resources(resources, prune: false, verify:, record_summary: true) logger.info("Deploying #{resource.id} (#{resource.pretty_timeout_type})") end - # Apply can be done in one large batch, the rest have to be done individually - applyables, individuals = resources.partition { |r| r.deploy_method == :apply } - # Prunable resources should also applied so that they can be pruned - pruneable_types = @prune_allowlist.map { |t| t.split("/").last } - applyables += individuals.select { |r| pruneable_types.include?(r.type) && !r.deploy_method_override } - - individuals.each do |individual_resource| - individual_resource.deploy_started_at = Time.now.utc - case individual_resource.deploy_method - when :create - err, status = create_resource(individual_resource) - when :replace - err, status = replace_or_create_resource(individual_resource) - when :replace_force - err, status = replace_or_create_resource(individual_resource, force: true) - else - # Fail Fast! This is a programmer mistake. - raise ArgumentError, "Unexpected deploy method! (#{individual_resource.deploy_method.inspect})" - end + StatsD.client.measure('sync.duration', tags: statsd_tags) do + # Apply can be done in one large batch, the rest have to be done individually + applyables, individuals = resources.partition { |r| r.deploy_method == :apply } + # Prunable resources should also applied so that they can be pruned + pruneable_types = @prune_allowlist.map { |t| t.split("/").last } + applyables += individuals.select { |r| pruneable_types.include?(r.type) && !r.deploy_method_override } + + individuals.each do |individual_resource| + individual_resource.deploy_started_at = Time.now.utc + case individual_resource.deploy_method + when :create + err, status = create_resource(individual_resource) + when :replace + err, status = replace_or_create_resource(individual_resource) + when :replace_force + err, status = replace_or_create_resource(individual_resource, force: true) + else + # Fail Fast! This is a programmer mistake. + raise ArgumentError, "Unexpected deploy method! (#{individual_resource.deploy_method.inspect})" + end - next if status.success? + next if status.success? - raise FatalDeploymentError, <<~MSG - Failed to replace or create resource: #{individual_resource.id} - #{individual_resource.sensitive_template_content? ? '' : err} - MSG - end + raise FatalDeploymentError, <<~MSG + Failed to replace or create resource: #{individual_resource.id} + #{individual_resource.sensitive_template_content? ? '' : err} + MSG + end - apply_all(applyables, prune) + apply_all(applyables, prune) - if verify - watcher = Krane::ResourceWatcher.new(resources: resources, deploy_started_at: deploy_started_at, - timeout: @global_timeout, task_config: @task_config, sha: @current_sha) - watcher.run(record_summary: record_summary) + if verify + watcher = Krane::ResourceWatcher.new(resources: resources, deploy_started_at: deploy_started_at, + timeout: @global_timeout, task_config: @task_config, sha: @current_sha) + watcher.run(record_summary: record_summary) + end end end diff --git a/test/helpers/mock_resource.rb b/test/helpers/mock_resource.rb index da53e0fb4..5f9b65257 100644 --- a/test/helpers/mock_resource.rb +++ b/test/helpers/mock_resource.rb @@ -25,6 +25,10 @@ def group "core" end + def predeployed? + false + end + def pretty_timeout_type end diff --git a/test/integration-serial/serial_deploy_test.rb b/test/integration-serial/serial_deploy_test.rb index 39a5553b3..e2438d117 100644 --- a/test/integration-serial/serial_deploy_test.rb +++ b/test/integration-serial/serial_deploy_test.rb @@ -344,8 +344,7 @@ def test_cr_success_with_service assert_deploy_success(deploy_fixtures("crd", subset: %w(web.yml))) - refute_logs_match(/Predeploying priority resources/) - assert_logs_match_all([/Phase 3: Deploying all resources/]) + assert_logs_match_all([/Phase 4: Deploying all resources/]) ensure build_kubectl.run("delete", "-f", filepath, use_namespace: false, log_failure: false) end @@ -526,6 +525,163 @@ def test_resource_discovery_stops_deploys_when_fetch_crds_kubectl_errs ], in_order: true) end + def test_deployment_with_predeploy_annotation_is_predeployed + # Deploy the fixtures with a modified deployment that has the predeploy annotation + result = deploy_fixtures("hello-cloud", subset: ["configmap-data.yml", "web.yml.erb"], render_erb: true) do |fixtures| + deployment = fixtures["web.yml.erb"]["Deployment"].first + deployment["metadata"]["annotations"] = { + "krane.shopify.io/predeployed" => "true" + } + end + + assert_deploy_success(result) + + # Verify the deployment was predeployed before other resources by checking log order + assert_logs_match_all([ + "Phase 3: Predeploying priority resources", + %r{Successfully deployed in \d+.\ds: Deployment/web}, + "Phase 4: Deploying all resources", + /Successfully deployed in \d+.\ds: ConfigMap\/hello-cloud-configmap-data, Deployment\/web, Ingress\/web, Service\/web/, + ], in_order: true) + end + + def test_service_with_predeploy_annotation_is_predeployed + result = deploy_fixtures("hello-cloud", subset: ["configmap-data.yml", "web.yml.erb"], render_erb: true) do |fixtures| + service = fixtures["web.yml.erb"]["Service"].first + service["metadata"]["annotations"] = { + "krane.shopify.io/predeployed" => "true", + "krane.shopify.io/skip-endpoint-validation" => "true" + } + end + + assert_deploy_success(result) + assert_logs_match_all([ + "Phase 3: Predeploying priority resources", + %r{Successfully deployed in \d+.\ds: Service/web}, + "Phase 4: Deploying all resources", + /Successfully deployed in \d+.\ds: ConfigMap\/hello-cloud-configmap-data, Deployment\/web, Ingress\/web, Service\/web/, + ], in_order: true) + end + + def test_ingress_with_predeploy_annotation_is_predeployed + result = deploy_fixtures("hello-cloud", subset: ["configmap-data.yml", "web.yml.erb"], render_erb: true) do |fixtures| + ingress = fixtures["web.yml.erb"]["Ingress"].first + ingress["metadata"]["annotations"] = { + "krane.shopify.io/predeployed" => "true" + } + end + + assert_deploy_success(result) + assert_logs_match_all([ + "Phase 3: Predeploying priority resources", + %r{Successfully deployed in \d+.\ds: Ingress/web}, + "Phase 4: Deploying all resources", + /Successfully deployed in \d+.\ds: ConfigMap\/hello-cloud-configmap-data, Deployment\/web, Ingress\/web, Service\/web/, + ], in_order: true) + end + + def test_job_with_predeploy_annotation_is_predeployed + result = deploy_fixtures("hello-cloud", subset: ["configmap-data.yml", "job.yml"]) do |fixtures| + job = fixtures["job.yml"]["Job"].first + job["metadata"]["annotations"] = { + "krane.shopify.io/predeployed" => "true" + } + end + + assert_deploy_success(result) + assert_logs_match_all([ + "Phase 3: Predeploying priority resources", + %r{Successfully deployed in \d+.\ds: Job/hello-job}, + "Phase 4: Deploying all resources", + /Successfully deployed in \d+.\ds: ConfigMap\/hello-cloud-configmap-data, Job\/hello-job/, + ], in_order: true) + end + + def test_daemonset_with_predeploy_annotation_is_predeployed + result = deploy_fixtures("hello-cloud", subset: ["configmap-data.yml", "daemon_set.yml"]) do |fixtures| + daemon_set = fixtures["daemon_set.yml"]["DaemonSet"].first + daemon_set["metadata"]["annotations"] = { + "krane.shopify.io/predeployed" => "true" + } + end + + assert_deploy_success(result) + assert_logs_match_all([ + "Phase 3: Predeploying priority resources", + %r{Successfully deployed in \d+.\ds: DaemonSet/ds-app}, + "Phase 4: Deploying all resources", + /Successfully deployed in \d+.\ds: ConfigMap\/hello-cloud-configmap-data, DaemonSet\/ds-app/, + ], in_order: true) + end + + def test_pod_disruption_budget_with_predeploy_annotation_is_predeployed + result = deploy_fixtures("hello-cloud", subset: ["configmap-data.yml", "disruption-budgets.yml"]) do |fixtures| + pdb = fixtures["disruption-budgets.yml"]["PodDisruptionBudget"].first + pdb["metadata"]["annotations"] = { + "krane.shopify.io/predeployed" => "true" + } + end + + assert_deploy_success(result) + assert_logs_match_all([ + "Phase 3: Predeploying priority resources", + %r{Successfully deployed in \d+.\ds: PodDisruptionBudget/test}, + "Phase 4: Deploying all resources", + /Successfully deployed in \d+.\ds: ConfigMap\/hello-cloud-configmap-data, PodDisruptionBudget\/test/, + ], in_order: true) + end + + def test_pod_template_with_predeploy_annotation_is_predeployed + result = deploy_fixtures("hello-cloud", subset: ["configmap-data.yml", "template-runner.yml"]) do |fixtures| + template = fixtures["template-runner.yml"]["PodTemplate"].first + template["metadata"]["annotations"] = { + "krane.shopify.io/predeployed" => "true" + } + end + + assert_deploy_success(result) + assert_logs_match_all([ + "Phase 3: Predeploying priority resources", + %r{Successfully deployed in \d+.\ds: PodTemplate/hello-cloud-template-runner}, + "Phase 4: Deploying all resources", + /Successfully deployed in \d+.\ds: ConfigMap\/hello-cloud-configmap-data, PodTemplate\/hello-cloud-template-runner/, + ], in_order: true) + end + + def test_replica_set_with_predeploy_annotation_is_predeployed + result = deploy_fixtures("hello-cloud", subset: ["configmap-data.yml", "bare_replica_set.yml"]) do |fixtures| + rs = fixtures["bare_replica_set.yml"]["ReplicaSet"].first + rs["metadata"]["annotations"] = { + "krane.shopify.io/predeployed" => "true" + } + end + + assert_deploy_success(result) + assert_logs_match_all([ + "Phase 3: Predeploying priority resources", + %r{Successfully deployed in \d+.\ds: ReplicaSet/bare-replica-set}, + "Phase 4: Deploying all resources", + /Successfully deployed in \d+.\ds: ConfigMap\/hello-cloud-configmap-data, ReplicaSet\/bare-replica-set/, + ], in_order: true) + end + + def test_stateful_set_with_predeploy_annotation_is_predeployed + result = deploy_fixtures("hello-cloud", subset: ["configmap-data.yml", "stateful_set.yml"]) do |fixtures| + stateful_set = fixtures["stateful_set.yml"]["StatefulSet"].first + stateful_set["metadata"]["annotations"] = { + "krane.shopify.io/predeployed" => "true" + } + end + + assert_deploy_success(result) + assert_logs_match_all([ + "Phase 3: Predeploying priority resources", + %r{Successfully deployed in \d+.\ds: StatefulSet/stateful-busybox}, + "Phase 4: Deploying all resources", + /Successfully deployed in \d+.\ds: ConfigMap\/hello-cloud-configmap-data, Service\/stateful-busybox, StatefulSet\/stateful-busybox/, + ], in_order: true) + end + private def rollout_conditions_annotation_key diff --git a/test/unit/krane/resource_deployer_test.rb b/test/unit/krane/resource_deployer_test.rb index fe1f73dd3..7e5b8a10e 100644 --- a/test/unit/krane/resource_deployer_test.rb +++ b/test/unit/krane/resource_deployer_test.rb @@ -66,6 +66,14 @@ def test_deploy_verify_false_no_failure_error def test_predeploy_priority_resources_respects_pre_deploy_list kind = "MockResource" resource = build_mock_resource + priority_list = { kind => { group: "not-#{resource.group}" } } + resource_deployer(kubectl_times: 0).predeploy_priority_resources([resource], priority_list) + end + + def test_predeploy_priority_resources_respects_pre_deploy_list_and_predeployed_true_annotation + kind = "MockResource" + resource = build_mock_resource + resource.expects(:predeployed?).returns(true) watcher = mock("ResourceWatcher") watcher.expects(:run).returns(true) # ResourceDeployer only creates a ResourceWatcher if one or more resources @@ -76,6 +84,18 @@ def test_predeploy_priority_resources_respects_pre_deploy_list resource_deployer.predeploy_priority_resources([resource], priority_list) end + def test_predeploy_priority_resources_respects_pre_deploy_list_and_predeployed_false_annotation + kind = "MockResource" + resource = build_mock_resource + resource.expects(:predeployed?).returns(false) + # ResourceDeployer only creates a ResourceWatcher if one or more resources + # are deployed. See test_predeploy_priority_resources_respects_empty_pre_deploy_list + # for counter example + Krane::ResourceWatcher.expects(:new).never + priority_list = { kind => { group: "core" } } + resource_deployer(kubectl_times: 0).predeploy_priority_resources([resource], priority_list) + end + def test_predeploy_priority_resources_respects_empty_pre_deploy_list resource = build_mock_resource priority_list = [] @@ -98,4 +118,4 @@ def resource_deployer(kubectl_times: 1, prune_allowlist: []) def build_mock_resource(final_status: "success", hits_to_complete: 0, name: "web-pod") MockResource.new(name, hits_to_complete, final_status) end -end +end \ No newline at end of file