diff --git a/test/e2e/bootcnode_test.go b/test/e2e/bootcnode_test.go index 85f89531..a37553fa 100644 --- a/test/e2e/bootcnode_test.go +++ b/test/e2e/bootcnode_test.go @@ -858,7 +858,11 @@ func TestPullSecretAuth(t *testing.T) { // Check Booted.Image (the full ref with manifest digest) rather // than Booted.ImageDigest (the content digest) because they can // differ with remote registries. - testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute, + // + // The budget covers a full authenticated pull (observed to take up + // to ~4 minutes in CI) plus one extra stage+reboot cycle in case + // the first reboot lands back on the previous deployment. + testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 10*time.Minute, HaveField("Booted", HaveField("Image", Equal(authImageRef))), ) @@ -922,7 +926,9 @@ func TestPullSecretRotation(t *testing.T) { ) g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) - testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute, + // The budget covers a full authenticated pull plus one extra + // stage+reboot cycle; see TestPullSecretAuth. + testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 10*time.Minute, HaveField("Booted", HaveField("Image", Equal(authUpdateRef))), ) @@ -984,7 +990,9 @@ func TestPullSecretRotation(t *testing.T) { // Phase 4: The upgrade completing proves the daemon re-read the // rotated secret and authenticated with the new registry credentials. - testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute, + // The budget covers a full authenticated pull plus one extra + // stage+reboot cycle; see TestPullSecretAuth. + testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 10*time.Minute, HaveField("Booted", HaveField("Image", Equal(authUpdate2Ref))), ) diff --git a/test/e2e/pullsecret_namespace_test.go b/test/e2e/pullsecret_namespace_test.go index 04d655af..2888d2b6 100644 --- a/test/e2e/pullsecret_namespace_test.go +++ b/test/e2e/pullsecret_namespace_test.go @@ -103,7 +103,9 @@ func TestPullSecretNamespaces(t *testing.T) { g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed()) *pool = *modified - testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute, + // The budget covers a full authenticated pull plus one extra + // stage+reboot cycle; see TestPullSecretAuth. + testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 10*time.Minute, HaveField("Booted", HaveField("Image", Equal(authImageRef))), ) g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index c30bd20a..5b4a190b 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -16,6 +16,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -61,8 +62,17 @@ func TestOperatorUpgrade(t *testing.T) { g.SetDefaultEventuallyTimeout(pollTimeout) g.SetDefaultEventuallyPollingInterval(pollInterval) - env := e2eutil.New(t) ctx := context.Background() + // Registered before the Env so it runs after the Env cleanup removes + // the test node. The restored controller may be scheduled on that + // node; wait for its replacement to lead before parallel tests start. + var env *e2eutil.Env + t.Cleanup(func() { + if env != nil { + waitForOperatorLeader(t, g, ctx, env.Client) + } + }) + env = e2eutil.New(t) repoRoot := findRepoRoot(t) chartPath := filepath.Join(repoRoot, "chart", "bootc-operator") @@ -563,6 +573,44 @@ func waitForOperatorReady( "expected operator deployment to be ready") } +// waitForOperatorLeader waits until the only running controller pod holds +// the leader election lease. A ready pod does not lead until the lease of +// a terminated predecessor expires. +func waitForOperatorLeader( + t *testing.T, + g Gomega, + ctx context.Context, + c client.Client, +) { + t.Helper() + g.Eventually(func(g Gomega) { + var pods corev1.PodList + g.Expect(c.List(ctx, &pods, + client.InNamespace(testutil.OperatorNamespaceName), + client.MatchingLabels{ + "app.kubernetes.io/name": "bootc-operator", + "app.kubernetes.io/component": "controller", + }, + )).To(Succeed()) + var running []string + for _, p := range pods.Items { + if p.DeletionTimestamp == nil && p.Status.Phase == corev1.PodRunning { + running = append(running, p.Name) + } + } + g.Expect(running).To(HaveLen(1)) + + var lease coordinationv1.Lease + g.Expect(c.Get(ctx, client.ObjectKey{ + Namespace: testutil.OperatorNamespaceName, + Name: "bootc-operator.bootc.dev", + }, &lease)).To(Succeed()) + g.Expect(lease.Spec.HolderIdentity).NotTo(BeNil()) + g.Expect(*lease.Spec.HolderIdentity).To(HavePrefix(running[0] + "_")) + }).WithTimeout(3*time.Minute).Should(Succeed(), + "expected the running controller to hold the leader election lease") +} + // readOperatorConfig returns the optional administrator-owned instance. An // ambiguous configuration should fail the upgrade instead of being discarded. func readOperatorConfig(