From 408c626b260ba073918583c412156d2fc4d66e7f Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Thu, 12 Nov 2026 10:00:34 +0000 Subject: [PATCH 1/2] e2e: Wait for controller leadership after upgrade test TestOperatorUpgrade restores the operator in cleanup. The controller can land on the test's worker node, and removing that node reschedules it while parallel tests are already running. Pool creation then races the new pod's image pull and leader election, flaking TestControllerMembership. Wait for the single running controller to hold the lease after the node is removed, so parallel tests start with a working controller. Assisted-by: AI Signed-off-by: Alice Frosi --- test/e2e/upgrade_test.go | 50 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index c30bd20a..5b4a190b 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -16,6 +16,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -61,8 +62,17 @@ func TestOperatorUpgrade(t *testing.T) { g.SetDefaultEventuallyTimeout(pollTimeout) g.SetDefaultEventuallyPollingInterval(pollInterval) - env := e2eutil.New(t) ctx := context.Background() + // Registered before the Env so it runs after the Env cleanup removes + // the test node. The restored controller may be scheduled on that + // node; wait for its replacement to lead before parallel tests start. + var env *e2eutil.Env + t.Cleanup(func() { + if env != nil { + waitForOperatorLeader(t, g, ctx, env.Client) + } + }) + env = e2eutil.New(t) repoRoot := findRepoRoot(t) chartPath := filepath.Join(repoRoot, "chart", "bootc-operator") @@ -563,6 +573,44 @@ func waitForOperatorReady( "expected operator deployment to be ready") } +// waitForOperatorLeader waits until the only running controller pod holds +// the leader election lease. A ready pod does not lead until the lease of +// a terminated predecessor expires. +func waitForOperatorLeader( + t *testing.T, + g Gomega, + ctx context.Context, + c client.Client, +) { + t.Helper() + g.Eventually(func(g Gomega) { + var pods corev1.PodList + g.Expect(c.List(ctx, &pods, + client.InNamespace(testutil.OperatorNamespaceName), + client.MatchingLabels{ + "app.kubernetes.io/name": "bootc-operator", + "app.kubernetes.io/component": "controller", + }, + )).To(Succeed()) + var running []string + for _, p := range pods.Items { + if p.DeletionTimestamp == nil && p.Status.Phase == corev1.PodRunning { + running = append(running, p.Name) + } + } + g.Expect(running).To(HaveLen(1)) + + var lease coordinationv1.Lease + g.Expect(c.Get(ctx, client.ObjectKey{ + Namespace: testutil.OperatorNamespaceName, + Name: "bootc-operator.bootc.dev", + }, &lease)).To(Succeed()) + g.Expect(lease.Spec.HolderIdentity).NotTo(BeNil()) + g.Expect(*lease.Spec.HolderIdentity).To(HavePrefix(running[0] + "_")) + }).WithTimeout(3*time.Minute).Should(Succeed(), + "expected the running controller to hold the leader election lease") +} + // readOperatorConfig returns the optional administrator-owned instance. An // ambiguous configuration should fail the upgrade instead of being discarded. func readOperatorConfig( From 66e08745c50729e8cfe99de725596cae55bd201a Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Thu, 12 Nov 2026 12:17:35 +0000 Subject: [PATCH 2/2] e2e: Give pull-secret tests budget for a fallback reboot TestPullSecretAuth failed in CI (run 43199018221, e2e 1.37): the authenticated-registry stage took about 4 minutes, and the first reboot landed back on the previous deployment because the staged deployment was silently lost at shutdown when the /boot automount had idle-expired (bootc-dev/bootc#2622). The daemon correctly re-staged and issued a second reboot, which converges, but the extra cycle pushed the total to about 8.7 minutes, just past the 8 minute WaitForNodeIdle budget. Raise the four authenticated-pull waits to 10 minutes so one extra stage+reboot cycle fits. This is a workaround for the flake, not a fix: the root cause is tracked in bootc-dev/bootc#2622 (via ostreedev/ostree#3689) and the operator-side impact in #256. Assisted-by: AI Signed-off-by: Alice Frosi --- test/e2e/bootcnode_test.go | 14 +++++++++++--- test/e2e/pullsecret_namespace_test.go | 4 +++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/test/e2e/bootcnode_test.go b/test/e2e/bootcnode_test.go index 85f89531..a37553fa 100644 --- a/test/e2e/bootcnode_test.go +++ b/test/e2e/bootcnode_test.go @@ -858,7 +858,11 @@ func TestPullSecretAuth(t *testing.T) { // Check Booted.Image (the full ref with manifest digest) rather // than Booted.ImageDigest (the content digest) because they can // differ with remote registries. - testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute, + // + // The budget covers a full authenticated pull (observed to take up + // to ~4 minutes in CI) plus one extra stage+reboot cycle in case + // the first reboot lands back on the previous deployment. + testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 10*time.Minute, HaveField("Booted", HaveField("Image", Equal(authImageRef))), ) @@ -922,7 +926,9 @@ func TestPullSecretRotation(t *testing.T) { ) g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) - testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute, + // The budget covers a full authenticated pull plus one extra + // stage+reboot cycle; see TestPullSecretAuth. + testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 10*time.Minute, HaveField("Booted", HaveField("Image", Equal(authUpdateRef))), ) @@ -984,7 +990,9 @@ func TestPullSecretRotation(t *testing.T) { // Phase 4: The upgrade completing proves the daemon re-read the // rotated secret and authenticated with the new registry credentials. - testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute, + // The budget covers a full authenticated pull plus one extra + // stage+reboot cycle; see TestPullSecretAuth. + testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 10*time.Minute, HaveField("Booted", HaveField("Image", Equal(authUpdate2Ref))), ) diff --git a/test/e2e/pullsecret_namespace_test.go b/test/e2e/pullsecret_namespace_test.go index 04d655af..2888d2b6 100644 --- a/test/e2e/pullsecret_namespace_test.go +++ b/test/e2e/pullsecret_namespace_test.go @@ -103,7 +103,9 @@ func TestPullSecretNamespaces(t *testing.T) { g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed()) *pool = *modified - testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute, + // The budget covers a full authenticated pull plus one extra + // stage+reboot cycle; see TestPullSecretAuth. + testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 10*time.Minute, HaveField("Booted", HaveField("Image", Equal(authImageRef))), ) g.Eventually(fetchPoolStatus(ctx, env.Client, pool)).