From 271ab63213c149098fc897652c2750d437e72b66 Mon Sep 17 00:00:00 2001 From: Konstantin Khlebnikov Date: Fri, 1 Nov 2024 12:53:54 +0100 Subject: [PATCH] test: refactor e2e tests --- pkg/testutil/spec_builders.go | 9 +- test/e2e/suite_test.go | 39 +- test/e2e/ytsaurus_controller_test.go | 762 +++++++++------------------ 3 files changed, 297 insertions(+), 513 deletions(-) diff --git a/pkg/testutil/spec_builders.go b/pkg/testutil/spec_builders.go index 3fa1f131..3aacb448 100644 --- a/pkg/testutil/spec_builders.go +++ b/pkg/testutil/spec_builders.go @@ -110,8 +110,7 @@ func CreateMinimalYtsaurusResource(namespace string) *ytv1.Ytsaurus { } } -func CreateBaseYtsaurusResource(namespace string) *ytv1.Ytsaurus { - ytsaurus := CreateMinimalYtsaurusResource(namespace) +func WithBaseYtsaurusComponents(ytsaurus *ytv1.Ytsaurus) *ytv1.Ytsaurus { ytsaurus = WithBootstrap(ytsaurus) ytsaurus = WithScheduler(ytsaurus) ytsaurus = WithControllerAgents(ytsaurus) @@ -121,6 +120,12 @@ func CreateBaseYtsaurusResource(namespace string) *ytv1.Ytsaurus { return ytsaurus } +func CreateBaseYtsaurusResource(namespace string) *ytv1.Ytsaurus { + ytsaurus := CreateMinimalYtsaurusResource(namespace) + ytsaurus = WithBaseYtsaurusComponents(ytsaurus) + return ytsaurus +} + // TODO (l0kix2): merge with ytconfig build spec helpers. func WithDataNodes(ytsaurus *ytv1.Ytsaurus) *ytv1.Ytsaurus { return WithDataNodesCount(ytsaurus, 3, nil) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index a525c533..8fcf31d0 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -27,6 +27,7 @@ import ( "k8s.io/utils/ptr" . "github.com/onsi/ginkgo/v2" + gtypes "github.com/onsi/ginkgo/v2/types" . "github.com/onsi/gomega" otypes "github.com/onsi/gomega/types" @@ -34,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + clientgoretry "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" @@ -106,6 +108,14 @@ var _ = SynchronizedBeforeSuite(func(ctx context.Context) []byte { Expect(k8sClient).NotTo(BeNil()) }) +func ShouldPreserveArtifacts() bool { + suiteConfig, _ := GinkgoConfiguration() + if !suiteConfig.FailFast { + return false + } + return CurrentSpecReport().State.Is(gtypes.SpecStateFailed | gtypes.SpecStateTimedout) +} + var _ = BeforeEach(func() { var cancel context.CancelFunc ctx, cancel = context.WithCancel(context.TODO()) @@ -181,21 +191,34 @@ func NewYtsaurusStatusTracker() func(*ytv1.Ytsaurus) bool { } } -func ExpectYtsaurus(ctx context.Context, name types.NamespacedName) Assertion { - var ytsaurus ytv1.Ytsaurus - Expect(k8sClient.Get(ctx, name, &ytsaurus)).Should(Succeed()) +func UpdateYtsaurus(ctx context.Context, ytsaurus *ytv1.Ytsaurus) { + var current ytv1.Ytsaurus + name := client.ObjectKeyFromObject(ytsaurus) + err := clientgoretry.RetryOnConflict(clientgoretry.DefaultRetry, func() error { + Expect(k8sClient.Get(ctx, name, ¤t)).To(Succeed()) + // Fetch current resource version: any status update changes it too. + ytsaurus.SetResourceVersion(current.GetResourceVersion()) + return k8sClient.Update(ctx, ytsaurus) + }) + Expect(err).ToNot(HaveOccurred()) +} + +func CheckYtsaurus(ctx context.Context, ytsaurus *ytv1.Ytsaurus) Assertion { + name := client.ObjectKeyFromObject(ytsaurus) + err := k8sClient.Get(ctx, name, ytsaurus) + Expect(err).ToNot(HaveOccurred()) return Expect(ytsaurus) } -func EventuallyYtsaurus(ctx context.Context, name types.NamespacedName, timeout time.Duration) AsyncAssertion { - var ytsaurus ytv1.Ytsaurus +func EventuallyYtsaurus(ctx context.Context, ytsaurus *ytv1.Ytsaurus, timeout time.Duration) AsyncAssertion { trackStatus := NewYtsaurusStatusTracker() + name := client.ObjectKeyFromObject(ytsaurus) return Eventually(ctx, func(ctx context.Context) (*ytv1.Ytsaurus, error) { - err := k8sClient.Get(ctx, name, &ytsaurus) + err := k8sClient.Get(ctx, name, ytsaurus) if err == nil { - trackStatus(&ytsaurus) + trackStatus(ytsaurus) } - return &ytsaurus, err + return ytsaurus, err }, timeout, pollInterval) } diff --git a/test/e2e/ytsaurus_controller_test.go b/test/e2e/ytsaurus_controller_test.go index 8f228ac0..490308c2 100644 --- a/test/e2e/ytsaurus_controller_test.go +++ b/test/e2e/ytsaurus_controller_test.go @@ -9,11 +9,14 @@ import ( "net/http" "os" "sort" + "strings" "time" "golang.org/x/exp/maps" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -30,7 +33,6 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ytv1 "github.com/ytsaurus/ytsaurus-k8s-operator/api/v1" "github.com/ytsaurus/ytsaurus-k8s-operator/pkg/components" @@ -89,13 +91,13 @@ func getRPCProxyAddress(g *ytconfig.Generator, namespace string) string { func getServiceAddress(svcName string, namespace string) string { svc := corev1.Service{} Expect(k8sClient.Get(ctx, - types.NamespacedName{Name: svcName, Namespace: namespace}, + client.ObjectKey{Name: svcName, Namespace: namespace}, &svc), ).Should(Succeed()) k8sNode := corev1.Node{} Expect(k8sClient.Get(ctx, - types.NamespacedName{Name: getKindControlPlaneName(), Namespace: namespace}, + client.ObjectKey{Name: getKindControlPlaneName(), Namespace: namespace}, &k8sNode), ).Should(Succeed()) @@ -124,239 +126,183 @@ func getKindControlPlaneName() string { func getMasterPod(name, namespace string) corev1.Pod { msPod := corev1.Pod{} - err := k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, &msPod) + err := k8sClient.Get(ctx, client.ObjectKey{Name: name, Namespace: namespace}, &msPod) Expect(err).Should(Succeed()) return msPod } -func deleteYtsaurus(ctx context.Context, ytsaurus *ytv1.Ytsaurus) { - if err := k8sClient.Delete(ctx, ytsaurus); err != nil { - log.Error(err, "Deleting ytsaurus failed") - } - - pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "master-data-ms-0", - Namespace: ytsaurus.Namespace, - }, - } - - if err := k8sClient.Delete(ctx, pvc); err != nil { - log.Error(err, "Deleting ytsaurus pvc failed") - } - - if err := k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ytsaurus.Namespace}}); err != nil { - log.Error(err, "Deleting namespace failed") - } -} - -func runYtsaurus(ytsaurus *ytv1.Ytsaurus) { - Expect(k8sClient.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ytsaurus.Namespace}})).Should(Succeed()) - - Expect(k8sClient.Create(ctx, ytsaurus)).Should(Succeed()) - - ytsaurusLookupKey := types.NamespacedName{Name: ytsaurus.Name, Namespace: ytsaurus.Namespace} - - Eventually(func() bool { - createdYtsaurus := &ytv1.Ytsaurus{} - err := k8sClient.Get(ctx, ytsaurusLookupKey, createdYtsaurus) - return err == nil - }, reactionTimeout, pollInterval).Should(BeTrue()) - - pods := []string{"ms-0", "hp-0"} - if ytsaurus.Spec.Discovery.InstanceCount != 0 { - pods = append(pods, "ds-0") - } - for _, dataNodeGroup := range ytsaurus.Spec.DataNodes { - pods = append(pods, fmt.Sprintf("%v-0", consts.FormatComponentStringWithDefault("dnd", dataNodeGroup.Name))) - } - - if len(ytsaurus.Spec.ExecNodes) != 0 { - pods = append(pods, "end-0") - } - - By("Checking that ytsaurus state is equal to `Running`") - EventuallyYtsaurus(ctx, ytsaurusLookupKey, bootstrapTimeout).Should(HaveClusterStateRunning()) - - By("Check pods are running") - for _, podName := range pods { - pod := &corev1.Pod{} - err := k8sClient.Get(ctx, types.NamespacedName{Name: podName, Namespace: ytsaurus.Namespace}, pod) - Expect(err).NotTo(HaveOccurred()) - Expect(pod.Status.Phase).To(Equal(corev1.PodRunning)) - } -} - -func createRemoteYtsaurus(remoteYtsaurus *ytv1.RemoteYtsaurus) { - Expect(k8sClient.Create(ctx, remoteYtsaurus)).Should(Succeed()) - lookupKey := types.NamespacedName{Name: remoteYtsaurus.Name, Namespace: remoteYtsaurus.Namespace} - Eventually(func() bool { - createdYtsaurus := &ytv1.RemoteYtsaurus{} - err := k8sClient.Get(ctx, lookupKey, createdYtsaurus) - return err == nil - }, reactionTimeout, pollInterval).Should(BeTrue()) -} - -func deleteRemoteYtsaurus(ctx context.Context, remoteYtsaurus *ytv1.RemoteYtsaurus) { - if err := k8sClient.Delete(ctx, remoteYtsaurus); err != nil { - log.Error(err, "Deleting remote ytsaurus failed") - } -} - -func runRemoteExecNodes(remoteExecNodes *ytv1.RemoteExecNodes) { - Expect(k8sClient.Create(ctx, remoteExecNodes)).Should(Succeed()) - lookupKey := types.NamespacedName{Name: remoteExecNodes.Name, Namespace: remoteExecNodes.Namespace} - Eventually(func() bool { - createdYtsaurus := &ytv1.RemoteExecNodes{} - err := k8sClient.Get(ctx, lookupKey, createdYtsaurus) - return err == nil - }, reactionTimeout, pollInterval).Should(BeTrue()) - - By("Checking that remote exec nodes state is equal to `Running`") - Eventually( - func() (*ytv1.RemoteExecNodes, error) { - nodes := &ytv1.RemoteExecNodes{} - err := k8sClient.Get(ctx, lookupKey, nodes) - return nodes, err - }, - reactionTimeout*2, - pollInterval, - ).Should(HaveField("Status.ReleaseStatus", ytv1.RemoteNodeReleaseStatusRunning)) -} - -func deleteRemoteExecNodes(ctx context.Context, remoteExecNodes *ytv1.RemoteExecNodes) { - if err := k8sClient.Delete(ctx, remoteExecNodes); err != nil { - log.Error(err, "Deleting remote exec nodes failed") - } -} - -func runRemoteDataNodes(remoteDataNodes *ytv1.RemoteDataNodes) { - Expect(k8sClient.Create(ctx, remoteDataNodes)).Should(Succeed()) - lookupKey := types.NamespacedName{Name: remoteDataNodes.Name, Namespace: remoteDataNodes.Namespace} - Eventually(func() bool { - createdYtsaurus := &ytv1.RemoteDataNodes{} - err := k8sClient.Get(ctx, lookupKey, createdYtsaurus) - return err == nil - }, reactionTimeout, pollInterval).Should(BeTrue()) - - By("Checking that remote data nodes state is equal to `Running`") - Eventually( - func() (*ytv1.RemoteDataNodes, error) { - nodes := &ytv1.RemoteDataNodes{} - err := k8sClient.Get(ctx, lookupKey, nodes) - return nodes, err - }, - reactionTimeout*2, - pollInterval, - ).Should(HaveField("Status.ReleaseStatus", ytv1.RemoteNodeReleaseStatusRunning)) -} - -func deleteRemoteDataNodes(ctx context.Context, remoteDataNodes *ytv1.RemoteDataNodes) { - if err := k8sClient.Delete(ctx, remoteDataNodes); err != nil { - log.Error(err, "Deleting remote data nodes failed") - } -} - -func runRemoteTabletNodes(remoteTabletNodes *ytv1.RemoteTabletNodes) { - Expect(k8sClient.Create(ctx, remoteTabletNodes)).Should(Succeed()) - lookupKey := types.NamespacedName{Name: remoteTabletNodes.Name, Namespace: remoteTabletNodes.Namespace} - Eventually(func() bool { - createdYtsaurus := &ytv1.RemoteTabletNodes{} - err := k8sClient.Get(ctx, lookupKey, createdYtsaurus) - return err == nil - }, reactionTimeout, pollInterval).Should(BeTrue()) - - By("Checking that remote tablet nodes state is equal to `Running`") - Eventually( - func() (*ytv1.RemoteTabletNodes, error) { - nodes := &ytv1.RemoteTabletNodes{} - err := k8sClient.Get(ctx, lookupKey, nodes) - return nodes, err - }, - reactionTimeout*2, - pollInterval, - ).Should(HaveField("Status.ReleaseStatus", ytv1.RemoteNodeReleaseStatusRunning)) -} - -func deleteRemoteTabletNodes(ctx context.Context, remoteTabletNodes *ytv1.RemoteTabletNodes) { - if err := k8sClient.Delete(ctx, remoteTabletNodes); err != nil { - log.Error(err, "Deleting remote tablet nodes failed") - } -} - func runImpossibleUpdateAndRollback(ytsaurus *ytv1.Ytsaurus, ytClient yt.Client) { - name := types.NamespacedName{Name: ytsaurus.Name, Namespace: ytsaurus.Namespace} + name := client.ObjectKey{Name: ytsaurus.Name, Namespace: ytsaurus.Namespace} By("Run cluster impossible update") Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed()) ytsaurus.Spec.CoreImage = testutil.CoreImageSecond - Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed()) + UpdateYtsaurus(ctx, ytsaurus) - EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveClusterUpdateState(ytv1.UpdateStateImpossibleToStart)) + EventuallyYtsaurus(ctx, ytsaurus, reactionTimeout).Should(HaveClusterUpdateState(ytv1.UpdateStateImpossibleToStart)) By("Set previous core image") Eventually(func(g Gomega) { g.Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed()) ytsaurus.Spec.CoreImage = testutil.CoreImageFirst - g.Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed()) + UpdateYtsaurus(ctx, ytsaurus) }, reactionTimeout, pollInterval).Should(Succeed()) By("Wait for running") - EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterStateRunning()) + EventuallyYtsaurus(ctx, ytsaurus, upgradeTimeout).Should(HaveClusterStateRunning()) By("Check that cluster alive after update") res := make([]string, 0) Expect(ytClient.ListNode(ctx, ypath.Path("/"), &res, nil)).Should(Succeed()) } -func createConfigOverridesMap(namespace, name, key, value string) { - resource := corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Data: map[string]string{key: value}, - } - Expect(k8sClient.Create(ctx, &resource)).Should(Succeed()) -} - type testRow struct { A string `yson:"a"` } var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() { + var namespace string + var objects []client.Object + var name client.ObjectKey + var ytsaurus *ytv1.Ytsaurus + var ytClient yt.Client + var g *ytconfig.Generator + + // Executuin sequence for each test spec: + // - BeforeEach (configuration) + // - JustBeforeEach (creation) + // - It (test) + // - JustAfterEach (diagnosis) + // - AfterEach, DeferCleanup (cleanup) + // + // See: + // https://onsi.github.io/ginkgo/#separating-creation-and-configuration-justbeforeeach + // https://onsi.github.io/ginkgo/#spec-cleanup-aftereach-and-defercleanup + // https://onsi.github.io/ginkgo/#separating-diagnostics-collection-and-teardown-justaftereach + + BeforeEach(func() { + By("Creating namespace") + currentSpec := CurrentSpecReport() + namespaceObject := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-e2e-", + Labels: map[string]string{ + "app.kubernetes.io/component": "test", + "app.kubernetes.io/name": "test-" + strings.Join(currentSpec.Labels(), "-"), + }, + Annotations: map[string]string{ + "kubernetes.io/description": currentSpec.LeafNodeText, + }, + }, + } + Expect(k8sClient.Create(ctx, &namespaceObject)).Should(Succeed()) + namespace = namespaceObject.Name // Fetch unique namespace name - Context("Update scenarios", Label("update"), func() { + By("Creating minimal Ytsaurus spec") + ytsaurus = testutil.CreateMinimalYtsaurusResource(namespace) + objects = []client.Object{ytsaurus} - Context("Updating Ytsaurus image", func() { + name = client.ObjectKey{ + Name: ytsaurus.Name, + Namespace: namespace, + } + }) + + JustBeforeEach(func(ctx context.Context) { + + By("Creating resource objects") + for _, object := range objects { + By(fmt.Sprintf("Creating %v", object.GetName())) + object.SetNamespace(namespace) + Expect(k8sClient.Create(ctx, object)).Should(Succeed()) + } - It("Should run and update Ytsaurus within same major version", Label("basic"), - getSimpleUpdateScenario("test-minor-update", testutil.CoreImageSecond), - ) + By("Checking that Ytsaurus state is equal to `Running`") + EventuallyYtsaurus(ctx, ytsaurus, bootstrapTimeout).Should(HaveClusterStateRunning()) - It("Should run and update Ytsaurus to the next major version", - getSimpleUpdateScenario("test-major-update", testutil.CoreImageNextVer), - ) + g = ytconfig.NewGenerator(ytsaurus, "local") + + ytClient = createYtsaurusClient(ytsaurus, namespace) + checkClusterViability(ytClient) + }) + + AfterEach(func(ctx context.Context) { + if ShouldPreserveArtifacts() { + log.Info("Preserving artifacts", "namespace", namespace) + return + } + + By("Deleting resource objects") + for _, object := range objects { + By(fmt.Sprintf("Deleting %v", object.GetName())) + if err := k8sClient.Delete(ctx, object); err != nil { + log.Error(err, "Cannot delete", "object", object.GetName()) + } + } + + By("Deleting namespace") + namespaceObject := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + Expect(k8sClient.Delete(ctx, &namespaceObject)).Should(Succeed()) + }) + + Context("Update scenarios", Label("update"), func() { + var podsBeforeUpdate map[string]corev1.Pod + + BeforeEach(func() { + By("Adding base components") + testutil.WithBaseYtsaurusComponents(ytsaurus) }) + JustBeforeEach(func() { + By("Getting pods before actions") + podsBeforeUpdate = getComponentPods(ctx, namespace) + }) + + DescribeTableSubtree("Updating Ytsaurus image", + func(newImage string) { + BeforeEach(func() { + testutil.WithNamedDataNodes(ytsaurus, ptr.To("dn-t")) + }) + + It("Triggers cluster update", func(ctx context.Context) { + checkPodLabels(ctx, namespace) + + ytsaurus.Spec.CoreImage = newImage + UpdateYtsaurus(ctx, ytsaurus) + + EventuallyYtsaurus(ctx, ytsaurus, reactionTimeout).Should(HaveObservedGeneration()) + Expect(ytsaurus).Should(HaveClusterUpdateFlow(ytv1.UpdateFlowFull)) + + By("Waiting cluster update completes") + EventuallyYtsaurus(ctx, ytsaurus, upgradeTimeout).Should(HaveClusterStateRunning()) + checkClusterBaseViability(ytClient) + + podsAfterFullUpdate := getComponentPods(ctx, namespace) + pods := getChangedPods(podsBeforeUpdate, podsAfterFullUpdate) + Expect(pods.Created).To(BeEmpty(), "created") + Expect(pods.Deleted).To(BeEmpty(), "deleted") + Expect(pods.Updated).To(ConsistOf(maps.Keys(podsBeforeUpdate)), "updated") + + CheckYtsaurus(ctx, ytsaurus).Should(HaveObservedGeneration()) + }) + }, + Entry("When update Ytsaurus within same major version", Label("basic"), testutil.CoreImageSecond), + Entry("When update Ytsaurus to the next major version", testutil.CoreImageNextVer), + ) + Context("Test UpdateSelector", Label("selector"), func() { It("Should be updated according to UpdateSelector=Everything", func(ctx context.Context) { - namespace := "testslcteverything" - - By("Creating a Ytsaurus resource") - ytsaurus := testutil.CreateBaseYtsaurusResource(namespace) - DeferCleanup(deleteYtsaurus, ytsaurus) - name := types.NamespacedName{Name: ytsaurus.GetName(), Namespace: namespace} - deployAndCheck(ytsaurus, namespace) - podsBeforeUpdate := getComponentPods(ctx, namespace) By("Run cluster update with selector: nothing") - Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed()) ytsaurus.Spec.UpdateSelector = ytv1.UpdateSelectorNothing // We want change in all yson configs, new discovery instance will trigger that. ytsaurus.Spec.Discovery.InstanceCount += 1 - Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed()) + UpdateYtsaurus(ctx, ytsaurus) By("Ensure cluster doesn't start updating for 5 seconds") ConsistentlyYtsaurus(ctx, name, 5*time.Second).Should(HaveClusterStateRunning()) @@ -367,15 +313,15 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() ) By("Update cluster update with strategy full") - Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed()) ytsaurus.Spec.UpdateSelector = ytv1.UpdateSelectorEverything ytsaurus.Spec.Discovery.InstanceCount += 1 - Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed()) - EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveObservedGeneration()) - ExpectYtsaurus(ctx, name).Should(HaveClusterUpdatingComponents()) + UpdateYtsaurus(ctx, ytsaurus) + + EventuallyYtsaurus(ctx, ytsaurus, reactionTimeout).Should(HaveObservedGeneration()) + Expect(ytsaurus).Should(HaveClusterUpdatingComponents()) By("Wait cluster update with full update complete") - EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterStateRunning()) + EventuallyYtsaurus(ctx, ytsaurus, upgradeTimeout).Should(HaveClusterStateRunning()) podsAfterFullUpdate := getComponentPods(ctx, namespace) pods := getChangedPods(podsBeforeUpdate, podsAfterFullUpdate) @@ -385,27 +331,17 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() }) It("Should be updated according to UpdateSelector=TabletNodesOnly,ExecNodesOnly", func(ctx context.Context) { - namespace := "testslctnodes" - - By("Creating a Ytsaurus resource") - ytsaurus := testutil.CreateBaseYtsaurusResource(namespace) - DeferCleanup(deleteYtsaurus, ytsaurus) - name := types.NamespacedName{Name: ytsaurus.GetName(), Namespace: namespace} - - deployAndCheck(ytsaurus, namespace) - podsBeforeUpdate := getComponentPods(ctx, namespace) By("Run cluster update with selector:ExecNodesOnly") - Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed()) ytsaurus.Spec.UpdateSelector = ytv1.UpdateSelectorExecNodesOnly ytsaurus.Spec.Discovery.InstanceCount += 1 - Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed()) - EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveObservedGeneration()) - ExpectYtsaurus(ctx, name).Should(HaveClusterUpdatingComponents("ExecNode")) + UpdateYtsaurus(ctx, ytsaurus) + + EventuallyYtsaurus(ctx, ytsaurus, reactionTimeout).Should(HaveObservedGeneration()) + Expect(ytsaurus).Should(HaveClusterUpdatingComponents("ExecNode")) By("Wait cluster update with selector:ExecNodesOnly complete") - EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterStateRunning()) - ytClient := createYtsaurusClient(ytsaurus, namespace) + EventuallyYtsaurus(ctx, ytsaurus, upgradeTimeout).Should(HaveClusterStateRunning()) checkClusterBaseViability(ytClient) podsAfterEndUpdate := getComponentPods(ctx, namespace) @@ -415,15 +351,15 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() Expect(pods.Updated).To(ConsistOf("end-0"), "updated") By("Run cluster update with selector:TabletNodesOnly") - Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed()) ytsaurus.Spec.UpdateSelector = ytv1.UpdateSelectorTabletNodesOnly ytsaurus.Spec.Discovery.InstanceCount += 1 - Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed()) - EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveObservedGeneration()) - ExpectYtsaurus(ctx, name).Should(HaveClusterUpdatingComponents("TabletNode")) + UpdateYtsaurus(ctx, ytsaurus) + + EventuallyYtsaurus(ctx, ytsaurus, reactionTimeout).Should(HaveObservedGeneration()) + Expect(ytsaurus).Should(HaveClusterUpdatingComponents("TabletNode")) By("Wait cluster update with selector:TabletNodesOnly complete") - EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterStateRunning()) + EventuallyYtsaurus(ctx, ytsaurus, upgradeTimeout).Should(HaveClusterStateRunning()) checkClusterBaseViability(ytClient) podsAfterTndUpdate := getComponentPods(ctx, namespace) @@ -434,27 +370,17 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() }) It("Should be updated according to UpdateSelector=MasterOnly,StatelessOnly", Label("basic"), func(ctx context.Context) { - namespace := "testslctother" - - By("Creating a Ytsaurus resource") - ytsaurus := testutil.CreateBaseYtsaurusResource(namespace) - DeferCleanup(deleteYtsaurus, ytsaurus) - name := types.NamespacedName{Name: ytsaurus.GetName(), Namespace: namespace} - - deployAndCheck(ytsaurus, namespace) - podsBeforeUpdate := getComponentPods(ctx, namespace) By("Run cluster update with selector:MasterOnly") - Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed()) ytsaurus.Spec.UpdateSelector = ytv1.UpdateSelectorMasterOnly ytsaurus.Spec.Discovery.InstanceCount += 1 - Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed()) - EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveObservedGeneration()) - ExpectYtsaurus(ctx, name).Should(HaveClusterUpdatingComponents("Master")) + UpdateYtsaurus(ctx, ytsaurus) + + EventuallyYtsaurus(ctx, ytsaurus, reactionTimeout).Should(HaveObservedGeneration()) + Expect(ytsaurus).Should(HaveClusterUpdatingComponents("Master")) By("Wait cluster update with selector:MasterOnly complete") - EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterStateRunning()) - ytClient := createYtsaurusClient(ytsaurus, namespace) + EventuallyYtsaurus(ctx, ytsaurus, upgradeTimeout).Should(HaveClusterStateRunning()) checkClusterBaseViability(ytClient) podsAfterMasterUpdate := getComponentPods(ctx, namespace) pods := getChangedPods(podsBeforeUpdate, podsAfterMasterUpdate) @@ -463,20 +389,21 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() Expect(pods.Updated).To(ConsistOf("ms-0"), "updated") By("Run cluster update with selector:StatelessOnly") - Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed()) ytsaurus.Spec.UpdateSelector = ytv1.UpdateSelectorStatelessOnly ytsaurus.Spec.Discovery.InstanceCount += 1 - Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed()) - EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveObservedGeneration()) - ExpectYtsaurus(ctx, name).Should(HaveClusterUpdatingComponents( + UpdateYtsaurus(ctx, ytsaurus) + + EventuallyYtsaurus(ctx, ytsaurus, reactionTimeout).Should(HaveObservedGeneration()) + Expect(ytsaurus).Should(HaveClusterUpdatingComponents( "Discovery", "HttpProxy", "ExecNode", "Scheduler", "ControllerAgent", )) + By("Wait cluster update with selector:StatelessOnly complete") - EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterStateRunning()) + EventuallyYtsaurus(ctx, ytsaurus, upgradeTimeout).Should(HaveClusterStateRunning()) checkClusterBaseViability(ytClient) podsAfterStatelessUpdate := getComponentPods(ctx, namespace) pods = getChangedPods(podsAfterMasterUpdate, podsAfterStatelessUpdate) @@ -489,24 +416,30 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() }) // update selector Context("With config overrides", Label("overrides"), func() { + var overrides *corev1.ConfigMap - It("ConfigOverrides update shout trigger reconciliation", func(ctx context.Context) { - namespace := "test-overrides" - coName := "config-overrides" - ytsaurus := testutil.CreateMinimalYtsaurusResource(namespace) - ytsaurus.Spec.ConfigOverrides = &corev1.LocalObjectReference{Name: coName} - DeferCleanup(deleteYtsaurus, ytsaurus) + BeforeEach(func() { + overrides = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "config-overrides", + Namespace: namespace, + }, + Data: map[string]string{}, + } + // objects = append(objects, overrides) - deployAndCheck(ytsaurus, namespace) - log.Info("sleep a little, because after cluster is running some reconciliations still may happen " + - "for some time (and for some reason) and we don't want to interfere before map creation") - time.Sleep(3 * time.Second) + ytsaurus.Spec.ConfigOverrides = &corev1.LocalObjectReference{ + Name: overrides.Name, + } + }) + + It("ConfigOverrides update shout trigger reconciliation", func(ctx context.Context) { dsPodName := "ds-0" msPodName := "ms-0" getPodByName := func(name string) (*corev1.Pod, error) { - ds0Name := types.NamespacedName{Name: name, Namespace: namespace} + ds0Name := client.ObjectKey{Name: name, Namespace: namespace} dsPod := &corev1.Pod{} err := k8sClient.Get(ctx, ds0Name, dsPod) return dsPod, err @@ -520,7 +453,11 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() msPodCreatedBefore := msPod.CreationTimestamp.Time discoveryOverride := "{resource_limits = {total_memory = 123456789;};}" - createConfigOverridesMap(namespace, coName, "ytserver-discovery.yson", discoveryOverride) + overrides.Data["ytserver-discovery.yson"] = discoveryOverride + + // FIXME(khlebnikov): Tests titled "update", but actually it creates overrides. + Expect(k8sClient.Create(ctx, overrides)).Should(Succeed()) + objects = append(objects, overrides) Eventually(ctx, func() bool { pod, err := getPodByName(dsPodName) @@ -547,21 +484,6 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() // This is a test for specific regression bug when master pods are recreated during PossibilityCheck stage. It("Master shouldn't be recreated before WaitingForPodsCreation state if config changes", func(ctx context.Context) { - namespace := "test3" - ytsaurus := testutil.CreateMinimalYtsaurusResource(namespace) - ytsaurusKey := types.NamespacedName{Name: testutil.YtsaurusName, Namespace: namespace} - - By("Creating a Ytsaurus resource") - g := ytconfig.NewGenerator(ytsaurus, "local") - DeferCleanup(deleteYtsaurus, ytsaurus) - runYtsaurus(ytsaurus) - - By("Creating ytsaurus client") - ytClient := getYtClient(g, namespace) - - By("Check that cluster alive") - res := make([]string, 0) - Expect(ytClient.ListNode(ctx, ypath.Path("/"), &res, nil)).Should(Succeed()) By("Store master pod creation time") msPod := getMasterPod(g.GetMasterPodNames()[0], namespace) @@ -582,15 +504,14 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() Expect(err).Should(Succeed()) By("Run cluster update") - Expect(k8sClient.Get(ctx, ytsaurusKey, ytsaurus)).Should(Succeed()) ytsaurus.Spec.HostNetwork = true ytsaurus.Spec.PrimaryMasters.HostAddresses = []string{ getKindControlPlaneName(), } - Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed()) + UpdateYtsaurus(ctx, ytsaurus) By("Waiting PossibilityCheck") - EventuallyYtsaurus(ctx, ytsaurusKey, reactionTimeout).Should(HaveClusterUpdateState(ytv1.UpdateStatePossibilityCheck)) + EventuallyYtsaurus(ctx, ytsaurus, reactionTimeout).Should(HaveClusterUpdateState(ytv1.UpdateStatePossibilityCheck)) By("Check that master pod was NOT recreated at the PossibilityCheck stage") time.Sleep(1 * time.Second) @@ -604,29 +525,13 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() Context("With tablet nodes", Label("tablet"), func() { - It("Should run and try to update Ytsaurus with tablet cell bundle which is not in `good` health", func(ctx context.Context) { - By("Creating a Ytsaurus resource") - - namespace := "test4" - - ytsaurus := testutil.CreateMinimalYtsaurusResource(namespace) + BeforeEach(func() { ytsaurus = testutil.WithTabletNodes(ytsaurus) + }) - g := ytconfig.NewGenerator(ytsaurus, "local") - - DeferCleanup(deleteYtsaurus, ytsaurus) - runYtsaurus(ytsaurus) - - By("Creating ytsaurus client") - ytClient := getYtClient(g, namespace) - - By("Check that cluster alive") - - res := make([]string, 0) - Expect(ytClient.ListNode(ctx, ypath.Path("/"), &res, nil)).Should(Succeed()) + It("Should run and try to update Ytsaurus with tablet cell bundle which is not in `good` health", func(ctx context.Context) { By("Check that tablet cell bundles are in `good` health") - Eventually(func() bool { notGoodBundles, err := components.GetNotGoodTabletCellBundles(ctx, ytClient) if err != nil { @@ -657,26 +562,15 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() Context("With data nodes", Label("data"), func() { - It("Should run and try to update Ytsaurus with lvc", func(ctx context.Context) { - By("Creating a Ytsaurus resource") - - namespace := "test5" + BeforeEach(func() { + Expect(ytsaurus.Spec.DataNodes).ToNot(BeEmpty()) - ytsaurus := testutil.CreateMinimalYtsaurusResource(namespace) - ytsaurus = testutil.WithDataNodes(ytsaurus) - ytsaurus.Spec.TabletNodes = make([]ytv1.TabletNodesSpec, 0) - - g := ytconfig.NewGenerator(ytsaurus, "local") - - DeferCleanup(deleteYtsaurus, ytsaurus) - runYtsaurus(ytsaurus) - - By("Creating ytsaurus client") - ytClient := getYtClient(g, namespace) + // FIXME: Why? + By("Removing tablet nodes") + ytsaurus.Spec.TabletNodes = []ytv1.TabletNodesSpec{} + }) - By("Check that cluster alive") - res := make([]string, 0) - Expect(ytClient.ListNode(ctx, ypath.Path("/"), &res, nil)).Should(Succeed()) + It("Should run and try to update Ytsaurus with lvc", func(ctx context.Context) { By("Create a chunk") _, err := ytClient.CreateNode(ctx, ypath.Path("//tmp/a"), yt.NodeTable, nil) @@ -712,22 +606,11 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() Context("With query tracker", Label("query-tracker"), func() { - It("Should run with query tracker and check that query tracker rpc address set up correctly", Label("basic"), func(ctx context.Context) { - By("Creating a Ytsaurus resource") - - namespace := "querytrackeraddress" - - ytsaurus := testutil.CreateBaseYtsaurusResource(namespace) + BeforeEach(func() { ytsaurus = testutil.WithQueryTracker(ytsaurus) + }) - g := ytconfig.NewGenerator(ytsaurus, "local") - - DeferCleanup(deleteYtsaurus, ytsaurus) - runYtsaurus(ytsaurus) - - By("Creating ytsaurus client") - ytClient := getYtClient(g, namespace) - + It("Should run with query tracker and check that query tracker rpc address set up correctly", Label("basic"), func(ctx context.Context) { By("Check that query tracker channel exists in cluster_connection") Expect(ytClient.NodeExists(ctx, ypath.Path("//sys/@cluster_connection/query_tracker/stages/production/channel"), nil)).Should(BeTrue()) }) @@ -735,23 +618,12 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() }) // update query-tracker Context("With queue agent", Label("queue-agent"), func() { - - It("Should run with query tracker and check that access control objects set up correctly", Label("basic"), func(ctx context.Context) { - By("Creating a Ytsaurus resource") - - namespace := "querytrackeraco" - - ytsaurus := testutil.CreateBaseYtsaurusResource(namespace) + BeforeEach(func() { ytsaurus = testutil.WithQueryTracker(ytsaurus) ytsaurus = testutil.WithQueueAgent(ytsaurus) + }) - g := ytconfig.NewGenerator(ytsaurus, "local") - - DeferCleanup(deleteYtsaurus, ytsaurus) - runYtsaurus(ytsaurus) - - By("Creating ytsaurus client") - ytClient := getYtClient(g, namespace) + It("Should run with query tracker and check that access control objects set up correctly", Label("basic"), func(ctx context.Context) { By("Check that access control object namespace 'queries' exists") Expect(ytClient.NodeExists(ctx, ypath.Path("//sys/access_control_object_namespaces/queries"), nil)).Should(BeTrue()) @@ -825,37 +697,40 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() }) // update Context("Remote ytsaurus tests", Label("remote"), func() { + var remoteYtsaurus *ytv1.RemoteYtsaurus + + BeforeEach(func() { + remoteYtsaurus = &ytv1.RemoteYtsaurus{ + ObjectMeta: metav1.ObjectMeta{ + Name: testutil.RemoteResourceName, + Namespace: namespace, + }, + Spec: ytv1.RemoteYtsaurusSpec{ + MasterConnectionSpec: ytv1.MasterConnectionSpec{ + CellTag: ytsaurus.Spec.PrimaryMasters.CellTag, + HostAddresses: []string{ + fmt.Sprintf("ms-0.masters.%s.svc.cluster.local", namespace), + }, + }, + }, + } + objects = append(objects, remoteYtsaurus) + }) Context("With remote exec nodes", Label("exec"), func() { + var remoteNodes *ytv1.RemoteExecNodes - It("Should create ytsaurus with remote exec nodes and execute a job", func() { - By("Creating a Ytsaurus resource") - ctx := context.Background() - - namespace := "remoteexec" + BeforeEach(func() { + ytsaurus = testutil.WithBaseYtsaurusComponents(ytsaurus) - ytsaurus := testutil.CreateBaseYtsaurusResource(namespace) // Ensure that no local exec nodes exist, only remote ones (which will be created later). ytsaurus.Spec.ExecNodes = []ytv1.ExecNodesSpec{} - g := ytconfig.NewGenerator(ytsaurus, "local") - remoteYtsaurus := &ytv1.RemoteYtsaurus{ + remoteNodes = &ytv1.RemoteExecNodes{ ObjectMeta: metav1.ObjectMeta{ Name: testutil.RemoteResourceName, Namespace: namespace, }, - Spec: ytv1.RemoteYtsaurusSpec{ - MasterConnectionSpec: ytv1.MasterConnectionSpec{ - CellTag: ytsaurus.Spec.PrimaryMasters.CellTag, - HostAddresses: []string{ - "ms-0.masters.remoteexec.svc.cluster.local", - }, - }, - }, - } - - remoteNodes := &ytv1.RemoteExecNodes{ - ObjectMeta: metav1.ObjectMeta{Name: testutil.RemoteResourceName, Namespace: namespace}, Spec: ytv1.RemoteExecNodesSpec{ RemoteClusterSpec: &corev1.LocalObjectReference{ Name: testutil.RemoteResourceName, @@ -868,18 +743,10 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() }, }, } + objects = append(objects, remoteNodes) + }) - defer deleteYtsaurus(ctx, ytsaurus) - runYtsaurus(ytsaurus) - - defer deleteRemoteYtsaurus(ctx, remoteYtsaurus) - createRemoteYtsaurus(remoteYtsaurus) - - defer deleteRemoteExecNodes(ctx, remoteNodes) - runRemoteExecNodes(remoteNodes) - - By("Creating ytsaurus client") - ytClient := getYtClient(g, namespace) + It("Should create ytsaurus with remote exec nodes and execute a job", func(ctx context.Context) { By("Running sort operation to ensure exec node works") op := runAndCheckSortOperation(ytClient) @@ -909,34 +776,14 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() }) // remote exec Context("With remote data nodes", Label("data"), func() { + var remoteNodes *ytv1.RemoteDataNodes - It("Should create ytsaurus with remote data nodes and write to a table", func() { - By("Creating a Ytsaurus resource") - ctx := context.Background() - - namespace := "remotedata" - - // We have to create only the minimal YT - with master and discovery servers - ytsaurus := testutil.CreateMinimalYtsaurusResource(namespace) - g := ytconfig.NewGenerator(ytsaurus, "local") - - remoteYtsaurus := &ytv1.RemoteYtsaurus{ + BeforeEach(func() { + remoteNodes = &ytv1.RemoteDataNodes{ ObjectMeta: metav1.ObjectMeta{ Name: testutil.RemoteResourceName, Namespace: namespace, }, - Spec: ytv1.RemoteYtsaurusSpec{ - MasterConnectionSpec: ytv1.MasterConnectionSpec{ - CellTag: ytsaurus.Spec.PrimaryMasters.CellTag, - HostAddresses: []string{ - "ms-0.masters.remotedata.svc.cluster.local", - }, - }, - }, - } - - remoteNodes := &ytv1.RemoteDataNodes{ - ObjectMeta: metav1.ObjectMeta{Name: testutil.RemoteResourceName, Namespace: namespace}, Spec: ytv1.RemoteDataNodesSpec{ RemoteClusterSpec: &corev1.LocalObjectReference{ Name: testutil.RemoteResourceName, @@ -949,18 +796,10 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() }, }, } + objects = append(objects, remoteNodes) + }) - defer deleteYtsaurus(ctx, ytsaurus) - runYtsaurus(ytsaurus) - - defer deleteRemoteYtsaurus(ctx, remoteYtsaurus) - createRemoteYtsaurus(remoteYtsaurus) - - defer deleteRemoteDataNodes(ctx, remoteNodes) - runRemoteDataNodes(remoteNodes) - - By("Creating ytsaurus client") - ytClient := getYtClient(g, namespace) + It("Should create ytsaurus with remote data nodes and write to a table", func(ctx context.Context) { By("Create a chunk on a remote data node") _, errCreateNode := ytClient.CreateNode(ctx, ypath.Path("//tmp/a"), yt.NodeTable, &yt.CreateNodeOptions{}) @@ -977,46 +816,17 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() }) // remote data Context("With remote tablet nodes", Label("tablet"), func() { + var remoteNodes *ytv1.RemoteTabletNodes - It("Should create ytsaurus with remote tablet nodes and write to a table", func() { - By("Creating a Ytsaurus resource") - ctx := context.Background() - - namespace := "remotetnd" - dynamicTableTest := "//sys/test-tnd-remote" - - type newOperation struct { - ID string `yson:"id,key"` - Status string `yson:"status"` - Restarts uint64 `yson:"restarts"` - } - - newSchema, err := schema.Infer(newOperation{}) - Expect(err).Should(Succeed()) - + BeforeEach(func() { // We have to create the minimal YT - with master, discovery servers and data-nodes - ytsaurus := testutil.CreateMinimalYtsaurusResource(namespace) ytsaurus = testutil.WithDataNodes(ytsaurus) - g := ytconfig.NewGenerator(ytsaurus, "local") - - remoteYtsaurus := &ytv1.RemoteYtsaurus{ + remoteNodes = &ytv1.RemoteTabletNodes{ ObjectMeta: metav1.ObjectMeta{ Name: testutil.RemoteResourceName, Namespace: namespace, }, - Spec: ytv1.RemoteYtsaurusSpec{ - MasterConnectionSpec: ytv1.MasterConnectionSpec{ - CellTag: ytsaurus.Spec.PrimaryMasters.CellTag, - HostAddresses: []string{ - fmt.Sprintf("ms-0.masters.%s.svc.cluster.local", namespace), - }, - }, - }, - } - - remoteNodes := &ytv1.RemoteTabletNodes{ - ObjectMeta: metav1.ObjectMeta{Name: testutil.RemoteResourceName, Namespace: namespace}, Spec: ytv1.RemoteTabletNodesSpec{ RemoteClusterSpec: &corev1.LocalObjectReference{ Name: testutil.RemoteResourceName, @@ -1029,18 +839,21 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() }, }, } + objects = append(objects, remoteNodes) + }) - defer deleteYtsaurus(ctx, ytsaurus) - runYtsaurus(ytsaurus) + It("Should create ytsaurus with remote tablet nodes and write to a table", func(ctx context.Context) { - defer deleteRemoteYtsaurus(ctx, remoteYtsaurus) - createRemoteYtsaurus(remoteYtsaurus) + dynamicTableTest := "//sys/test-tnd-remote" - defer deleteRemoteTabletNodes(ctx, remoteNodes) - runRemoteTabletNodes(remoteNodes) + type newOperation struct { + ID string `yson:"id,key"` + Status string `yson:"status"` + Restarts uint64 `yson:"restarts"` + } - By("Creating ytsaurus client") - ytClient := getYtClient(g, namespace) + newSchema, err := schema.Infer(newOperation{}) + Expect(err).Should(Succeed()) By("Creating a tablet cell in the default bundle client") tabletCellID, errCreateTabletCell := ytClient.CreateObject(ctx, "tablet_cell", &yt.CreateObjectOptions{ @@ -1108,16 +921,13 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() Context("With RPC proxy", Label("rpc-proxy"), func() { - It("Rpc proxies should require authentication", func(ctx context.Context) { - namespace := "testrpc" - ytsaurus := testutil.CreateMinimalYtsaurusResource(namespace) + BeforeEach(func() { ytsaurus = testutil.WithRPCProxies(ytsaurus) - DeferCleanup(deleteYtsaurus, ytsaurus) - deployAndCheck(ytsaurus, namespace) + }) + It("Rpc proxies should require authentication", func(ctx context.Context) { // Just in case, so dirty dev environment wouldn't interfere. Expect(os.Unsetenv("YT_TOKEN")).Should(Succeed()) - g := ytconfig.NewGenerator(ytsaurus, "local") cli, err := ytrpc.NewClient(&yt.Config{ // N.B.: no credentials are passed here. RPCProxy: getRPCProxyAddress(g, namespace), @@ -1133,18 +943,12 @@ var _ = Describe("Basic e2e test for Ytsaurus controller", Label("e2e"), func() Context("With host network", Label("host-network"), func() { - It("Sensors should be annotated with host", func(ctx context.Context) { - namespace := "testsolomon" - ytsaurus := testutil.CreateMinimalYtsaurusResource(namespace) + BeforeEach(func() { ytsaurus.Spec.HostNetwork = true - DeferCleanup(deleteYtsaurus, ytsaurus) - deployAndCheck(ytsaurus, namespace) - - By("Creating ytsaurus client") - g := ytconfig.NewGenerator(ytsaurus, "local") - ytClient := getYtClient(g, namespace) + }) - primaryMasters := make([]string, 0) + It("Sensors should be annotated with host", func(ctx context.Context) { + var primaryMasters []string Expect(ytClient.ListNode(ctx, ypath.Path("//sys/primary_masters"), &primaryMasters, nil)).Should(Succeed()) Expect(primaryMasters).Should(Not(BeEmpty())) @@ -1238,7 +1042,7 @@ func checkPodLabels(ctx context.Context, namespace string) { pods := getAllPods(ctx, namespace) for _, pod := range pods { - fmt.Fprintf(GinkgoWriter, "PodName: %v, Labels: %v\n", pod.Name, pod.ObjectMeta.Labels) + fmt.Fprintf(GinkgoWriter, "PodName: %v, Labels: %v\n", pod.Name, pod.Labels) } Expect(pods).Should(ContainElement(And( @@ -1284,60 +1088,12 @@ func checkPodLabels(ctx context.Context, namespace string) { checkPodLabelCount(pods, "app.kubernetes.io/name", "yt-scheduler", 1) } -func deployAndCheck(ytsaurus *ytv1.Ytsaurus, namespace string) { - runYtsaurus(ytsaurus) - - ytClient := createYtsaurusClient(ytsaurus, namespace) - checkClusterViability(ytClient) -} - func createYtsaurusClient(ytsaurus *ytv1.Ytsaurus, namespace string) yt.Client { By("Creating ytsaurus client") g := ytconfig.NewGenerator(ytsaurus, "local") return getYtClient(g, namespace) } -func getSimpleUpdateScenario(namespace, newImage string) func(ctx context.Context) { - return func(ctx context.Context) { - By("Creating a Ytsaurus resource") - ytsaurus := testutil.CreateBaseYtsaurusResource(namespace) - testutil.WithNamedDataNodes(ytsaurus, ptr.To("dn-t")) - DeferCleanup(deleteYtsaurus, ytsaurus) - name := types.NamespacedName{Name: ytsaurus.GetName(), Namespace: namespace} - deployAndCheck(ytsaurus, namespace) - - By("Run cluster update") - podsBeforeUpdate := getComponentPods(ctx, namespace) - Expect(k8sClient.Get(ctx, name, ytsaurus)).Should(Succeed()) - - checkPodLabels(ctx, namespace) - - ytsaurus.Spec.CoreImage = newImage - Expect(k8sClient.Update(ctx, ytsaurus)).Should(Succeed()) - - EventuallyYtsaurus(ctx, name, reactionTimeout).Should(HaveObservedGeneration()) - ExpectYtsaurus(ctx, name).Should(HaveClusterUpdateFlow(ytv1.UpdateFlowFull)) - - By("Wait cluster update complete") - EventuallyYtsaurus(ctx, name, upgradeTimeout).Should(HaveClusterStateRunning()) - g := ytconfig.NewGenerator(ytsaurus, "local") - ytClient := getYtClient(g, namespace) - checkClusterBaseViability(ytClient) - - podsAfterFullUpdate := getComponentPods(ctx, namespace) - pods := getChangedPods(podsBeforeUpdate, podsAfterFullUpdate) - Expect(pods.Created).To(BeEmpty(), "created") - Expect(pods.Deleted).To(BeEmpty(), "deleted") - Expect(pods.Updated).To(ConsistOf(maps.Keys(podsBeforeUpdate)), "updated") - - newYt := ytv1.Ytsaurus{} - err := k8sClient.Get(ctx, name, &newYt) - Expect(err).Should(Succeed()) - - Expect(newYt.Status.ObservedGeneration).To(Equal(newYt.Generation)) - } -} - func runAndCheckSortOperation(ytClient yt.Client) mapreduce.Operation { testTablePathIn := ypath.Path("//tmp/testexec") testTablePathOut := ypath.Path("//tmp/testexec-out")