e2e: Wait for controller leadership after upgrade test

test/e2e/bootcnode_test.go

@@ -858,7 +858,11 @@ func TestPullSecretAuth(t *testing.T) {
 	// Check Booted.Image (the full ref with manifest digest) rather
 	// than Booted.ImageDigest (the content digest) because they can
 	// differ with remote registries.
-	testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute,
+	//
+	// The budget covers a full authenticated pull (observed to take up
+	// to ~4 minutes in CI) plus one extra stage+reboot cycle in case
+	// the first reboot lands back on the previous deployment.
+	testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 10*time.Minute,
 		HaveField("Booted", HaveField("Image", Equal(authImageRef))),
 	)
 
@@ -922,7 +926,9 @@ func TestPullSecretRotation(t *testing.T) {
 	)
 	g.Expect(env.Client.Create(ctx, pool)).To(Succeed())
 
-	testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute,
+	// The budget covers a full authenticated pull plus one extra
+	// stage+reboot cycle; see TestPullSecretAuth.
+	testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 10*time.Minute,
 		HaveField("Booted", HaveField("Image", Equal(authUpdateRef))),
 	)
 
@@ -984,7 +990,9 @@ func TestPullSecretRotation(t *testing.T) {
 
 	// Phase 4: The upgrade completing proves the daemon re-read the
 	// rotated secret and authenticated with the new registry credentials.
-	testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute,
+	// The budget covers a full authenticated pull plus one extra
+	// stage+reboot cycle; see TestPullSecretAuth.
+	testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 10*time.Minute,
 		HaveField("Booted", HaveField("Image", Equal(authUpdate2Ref))),
 	)
 

test/e2e/pullsecret_namespace_test.go

@@ -103,7 +103,9 @@ func TestPullSecretNamespaces(t *testing.T) {
 	g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed())
 	*pool = *modified
 
-	testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 8*time.Minute,
+	// The budget covers a full authenticated pull plus one extra
+	// stage+reboot cycle; see TestPullSecretAuth.
+	testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 10*time.Minute,
 		HaveField("Booted", HaveField("Image", Equal(authImageRef))),
 	)
 	g.Eventually(fetchPoolStatus(ctx, env.Client, pool)).

test/e2e/upgrade_test.go

@@ -16,6 +16,7 @@ import (
 
 	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
+	coordinationv1 "k8s.io/api/coordination/v1"
 	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -61,8 +62,17 @@ func TestOperatorUpgrade(t *testing.T) {
 	g.SetDefaultEventuallyTimeout(pollTimeout)
 	g.SetDefaultEventuallyPollingInterval(pollInterval)
 
-	env := e2eutil.New(t)
 	ctx := context.Background()
+	// Registered before the Env so it runs after the Env cleanup removes
+	// the test node. The restored controller may be scheduled on that
+	// node; wait for its replacement to lead before parallel tests start.
+	var env *e2eutil.Env
+	t.Cleanup(func() {
+		if env != nil {
+			waitForOperatorLeader(t, g, ctx, env.Client)
+		}
+	})
+	env = e2eutil.New(t)
 
 	repoRoot := findRepoRoot(t)
 	chartPath := filepath.Join(repoRoot, "chart", "bootc-operator")
@@ -563,6 +573,44 @@ func waitForOperatorReady(
 		"expected operator deployment to be ready")
 }
 
+// waitForOperatorLeader waits until the only running controller pod holds
+// the leader election lease. A ready pod does not lead until the lease of
+// a terminated predecessor expires.
+func waitForOperatorLeader(
+	t *testing.T,
+	g Gomega,
+	ctx context.Context,
+	c client.Client,
+) {
+	t.Helper()
+	g.Eventually(func(g Gomega) {
+		var pods corev1.PodList
+		g.Expect(c.List(ctx, &pods,
+			client.InNamespace(testutil.OperatorNamespaceName),
+			client.MatchingLabels{
+				"app.kubernetes.io/name":      "bootc-operator",
+				"app.kubernetes.io/component": "controller",
+			},
+		)).To(Succeed())
+		var running []string
+		for _, p := range pods.Items {
+			if p.DeletionTimestamp == nil && p.Status.Phase == corev1.PodRunning {
+				running = append(running, p.Name)
+			}
+		}
+		g.Expect(running).To(HaveLen(1))
+
+		var lease coordinationv1.Lease
+		g.Expect(c.Get(ctx, client.ObjectKey{
+			Namespace: testutil.OperatorNamespaceName,
+			Name:      "bootc-operator.bootc.dev",
+		}, &lease)).To(Succeed())
+		g.Expect(lease.Spec.HolderIdentity).NotTo(BeNil())
+		g.Expect(*lease.Spec.HolderIdentity).To(HavePrefix(running[0] + "_"))
+	}).WithTimeout(3*time.Minute).Should(Succeed(),
+		"expected the running controller to hold the leader election lease")
+}
+
 // readOperatorConfig returns the optional administrator-owned instance. An
 // ambiguous configuration should fail the upgrade instead of being discarded.
 func readOperatorConfig(