Create an event when drain is blocked by a PBD

api/v1alpha1/bootcnodepool_types.go

@@ -28,6 +28,11 @@ const (
 
 	// PoolPaused means updates are pending but the pool is paused.
 	PoolPaused string = "Paused"
+
+	// PoolPDBBlocked means stalled drains are blocked by
+	// PodDisruptionBudgets and that is keeping the rollout from making
+	// progress.
+	PoolPDBBlocked string = "PDBBlocked"
 )
 
 // PoolDegraded condition reasons. Precedence ordering (which reason

config/rbac/role.yaml

@@ -83,6 +83,13 @@ rules:
   - bootcnodes/status
   verbs:
   - get
+- apiGroups:
+  - policy
+  resources:
+  - poddisruptionbudgets
+  verbs:
+  - get
+  - list
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

docs/design/ARCHITECTURE.md

@@ -135,6 +135,7 @@ Pool conditions and their reasons:
 | UpToDate  | True   | AllUpdated        | All nodes are running targetDigest                  |
 | UpToDate  | False  | RolloutInProgress | Nodes are actively being updated                    |
 | UpToDate  | False  | Paused            | Updates pending but pool is paused                  |
+| UpToDate  | False  | PDBBlocked        | Stalled drains blocked by PodDisruptionBudgets      |
 | Degraded  | True   | NodeConflict      | Node selector overlaps with another pool            |
 | Degraded  | True   | NodeDegraded      | At least one node has errors or isn't converging    |
 | Degraded  | True   | RolloutHalted     | 2+ unhealthy nodes in reboot slots; rollout stopped |

docs/src/operations/pool.md

@@ -97,6 +97,61 @@ kubectl get bootcnodes
 Each `BootcNode` status reflects the output of `bootc status` on that node:
 the booted image, any staged image, and the rollback entry.
 
+## Rollout stuck on PodDisruptionBudget
+
+A drain can stall when a `PodDisruptionBudget` (PDB) refuses to let the
+workloads on a node be evicted. Once a drain has been running for five
+minutes, the operator inspects the pods it is trying to evict and reports the
+blocking budget.
+
+The `BootcNode` receives a `DrainBlockedByPDB` warning event:
+
+```shell
+kubectl describe bootcnode workers-w1
+```
+
+```
+Warning  DrainBlockedByPDB  Drain  Drain blocked by PodDisruptionBudget
+default/web: 2 pod(s) not ready
+```
+
+While a drain remains stalled, the event is re-emitted only when the set of
+blocking budgets changes, so it does not hide other events.
+
+When every occupied reboot slot is held by a drain blocked this way, the pool's
+`UpToDate` condition reports `False` with reason `PDBBlocked`:
+
+```shell
+kubectl get bootcnodepool workers -o yaml
+```
+
+```yaml
+conditions:
+- type: UpToDate
+  status: "False"
+  reason: PDBBlocked
+  message: "9/10 updated; 0 pending, 0 staging, 1 staged, 0 rebooting, 0 degraded; workers-w1 blocked by PDB default/web"
+```
+
+If only some slots are blocked, the reason stays `RolloutInProgress`, but the
+message still names each affected node and budget.
+
+The message hint explains why the budget has no disruptions available:
+
+- `N pod(s) not ready`: fewer pods are healthy than the budget expects.
+  Bring the workload back to a healthy state, for example by scaling it up or
+  fixing failing pods.
+- `never allows eviction (desiredHealthy >= expectedPods)`: the budget cannot
+  permit an eviction even when every pod is healthy. Relax the budget's
+  `minAvailable` or `maxUnavailable` for the duration of the rollout.
+- `misconfiguration: pod is selected by multiple PodDisruptionBudgets`:
+  more than one budget selects the pod. Make the selectors mutually exclusive
+  so each pod is covered by a single budget.
+
+The operator keeps retrying the drain while the rollout is blocked and
+continues normally as soon as the budget allows an eviction. The rollout
+concurrency and reboot behavior are otherwise unchanged.
+
 ## Pausing and resuming
 
 To pause a rollout (nodes already staging will complete, but no new reboots

internal/controller/bootcnodepool_controller.go

@@ -48,6 +48,9 @@ type drainStatus struct {
 	cancel    context.CancelFunc // to abort on targetDigest change or node removal
 	startTime time.Time          // for stall detection
 	isStalled bool               // set after the one-shot drain stall event is emitted
+	// lastPDBBlockKey tracks the blocking PDB set already reported for this
+	// stall so a changed set re-emits the event and repeats stay quiet.
+	lastPDBBlockKey string
 }
 
 // TagResolver resolves a container image reference to a digest.
@@ -69,6 +72,10 @@ type BootcNodePoolReconciler struct {
 	TagResolver           TagResolver
 	TagResolutionInterval time.Duration
 
+	// DrainStallThreshold is how long a drain may run before the
+	// controller reports it as stalled. Zero uses the default.
+	DrainStallThreshold time.Duration
+
 	// PullSecretNamespaces is the allowlist of namespaces pools may
 	// reference pull secrets from. The manager's Secret cache must be
 	// restricted to the same namespaces.
@@ -95,6 +102,7 @@ type BootcNodePoolReconciler struct {
 // +kubebuilder:rbac:groups="",namespace=system,resources=secrets,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list
 // +kubebuilder:rbac:groups="",resources=pods/eviction,verbs=create
+// +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list
 // +kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
 // +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
@@ -505,7 +513,7 @@ func (r *BootcNodePoolReconciler) Reconcile(
 	}
 	resolveResult.RequeueAfter = earlierRequeue(
 		resolveResult.RequeueAfter,
-		r.recordDrainStalls(&pool, ownedBootcNodes),
+		r.recordDrainStalls(ctx, &pool, rs, ownedBootcNodes),
 	)
 
 	// Early-return paths above (TargetDigest empty, InvalidSpec) skip

internal/controller/events.go

@@ -5,6 +5,7 @@ package controller
 import (
 	"context"
 	"fmt"
+	"slices"
 	"strings"
 	"time"
 	"unicode/utf8"
@@ -25,6 +26,7 @@ const (
 	eventReasonRolloutCompleted     = "RolloutCompleted"
 	eventReasonDrainFailed          = "DrainFailed"
 	eventReasonDrainTakingTooLong   = "DrainTakingTooLong"
+	eventReasonDrainBlockedByPDB    = "DrainBlockedByPDB"
 
 	eventActionResolveImage = "ResolveImage"
 	eventActionRollout      = "Rollout"
@@ -33,10 +35,22 @@ const (
 	eventActionDrain        = "Drain"
 
 	drainStallThreshold = 5 * time.Minute
-	eventNoteLimit      = 1024
-	eventNoteSuffix     = "..."
+	// drainStallRequeueInterval is short enough to notice a PDB starting to
+	// allow evictions without generating excessive API traffic.
+	drainStallRequeueInterval = 30 * time.Second
+	eventNoteLimit            = 1024
+	eventNoteSuffix           = "..."
 )
 
+// effectiveDrainStallThreshold returns the configured drain stall threshold,
+// falling back to the default when the reconciler field is unset.
+func (r *BootcNodePoolReconciler) effectiveDrainStallThreshold() time.Duration {
+	if r.DrainStallThreshold > 0 {
+		return r.DrainStallThreshold
+	}
+	return drainStallThreshold
+}
+
 // EventNote renders the human-readable message of a Kubernetes Event. Each
 // implementation owns a small set of well-defined fields and is responsible for
 // producing a message that fits the events.k8s.io/v1 1 KiB note limit. Notes
@@ -153,6 +167,21 @@ func (n drainStalledNote) Note() string {
 	)
 }
 
+// drainBlockedByPDBNote lists the PDBs preventing a drain from making
+// progress. PDB names and hints are bounded by their own fields, but the note
+// caps itself defensively because it aggregates several of them.
+type drainBlockedByPDBNote struct {
+	Blocks []pdbBlock
+}
+
+func (n drainBlockedByPDBNote) Note() string {
+	parts := make([]string, 0, len(n.Blocks))
+	for _, block := range n.Blocks {
+		parts = append(parts, block.describe())
+	}
+	return capNote("Drain blocked by " + strings.Join(parts, "; "))
+}
+
 func (r *BootcNodePoolReconciler) recordPoolEvents(
 	pool *bootcv1alpha1.BootcNodePool,
 	previous *bootcv1alpha1.BootcNodePoolStatus,
@@ -360,48 +389,106 @@ func (r *BootcNodePoolReconciler) recordDrainFailedEvent(
 	)
 }
 
-// recordDrainStalls emits one warning per drain that crosses the stall
-// threshold and returns when the next active drain should be checked. The
-// existing in-memory drain state is sufficient because drains are restarted
-// after a controller restart.
+// recordDrainStalls emits a warning for each drain that crosses the stall
+// threshold, then inspects stalled drains for PodDisruptionBudget blocks. While
+// a drain remains stalled it returns a short requeue interval so PDB state is
+// re-evaluated. The existing in-memory drain state is sufficient because drains
+// are restarted after a controller restart.
 func (r *BootcNodePoolReconciler) recordDrainStalls(
+	ctx context.Context,
 	pool *bootcv1alpha1.BootcNodePool,
+	rs *rolloutState,
 	nodes map[string]*bootcv1alpha1.BootcNode,
 ) time.Duration {
+	log := logf.FromContext(ctx)
 	now := time.Now()
-	var stalledNodes []*bootcv1alpha1.BootcNode
+	threshold := r.effectiveDrainStallThreshold()
+	var newlyStalled []*bootcv1alpha1.BootcNode
+	var stalledNodes []string
 	var nextCheck time.Duration
 
 	r.drainsMu.Lock()
 	for nodeName, status := range r.drains {
-		if status.isStalled {
+		// r.drains is shared by every pool, so only consider drains for
+		// nodes owned by the pool being reconciled.
+		if _, ok := nodes[nodeName]; !ok {
 			continue
 		}
 
-		remaining := drainStallThreshold - now.Sub(status.startTime)
-		if remaining > 0 {
-			nextCheck = earlierRequeue(nextCheck, remaining)
-			continue
+		if !status.isStalled {
+			remaining := threshold - now.Sub(status.startTime)
+			if remaining > 0 {
+				nextCheck = earlierRequeue(nextCheck, remaining)
+				continue
+			}
+
+			status.isStalled = true
+			if node, ok := nodes[nodeName]; ok {
+				newlyStalled = append(newlyStalled, node)
+			}
 		}
 
-		status.isStalled = true
-		if node, ok := nodes[nodeName]; ok {
-			stalledNodes = append(stalledNodes, node)
-		}
+		stalledNodes = append(stalledNodes, nodeName)
+		nextCheck = earlierRequeue(nextCheck, drainStallRequeueInterval)
 	}
 	r.drainsMu.Unlock()
 
-	for _, node := range stalledNodes {
+	slices.Sort(stalledNodes)
+	slices.SortFunc(newlyStalled, func(a, b *bootcv1alpha1.BootcNode) int {
+		return strings.Compare(a.Name, b.Name)
+	})
+	for _, node := range newlyStalled {
 		r.recordEvent(
 			node,
 			pool,
 			corev1.EventTypeWarning,
 			eventReasonDrainTakingTooLong,
 			eventActionDrain,
-			drainStalledNote{Threshold: drainStallThreshold},
+			drainStalledNote{Threshold: threshold},
 		)
 	}
 
+	blocked := map[string][]pdbBlock{}
+	for _, nodeName := range stalledNodes {
+		node := nodes[nodeName]
+
+		blocks, err := r.findPDBBlocksForNode(ctx, pool, nodeName)
+		if err != nil {
+			log.Error(
+				err,
+				"Failed to check PodDisruptionBudgets for stalled drain",
+				"node", nodeName,
+			)
+			continue
+		}
+
+		key := pdbBlocksKey(blocks)
+		r.drainsMu.Lock()
+		status := r.drains[nodeName]
+		emit := status != nil && key != "" && status.lastPDBBlockKey != key
+		if status != nil {
+			status.lastPDBBlockKey = key
+		}
+		r.drainsMu.Unlock()
+
+		if len(blocks) == 0 {
+			continue
+		}
+
+		blocked[nodeName] = blocks
+		if emit {
+			r.recordEvent(
+				node,
+				pool,
+				corev1.EventTypeWarning,
+				eventReasonDrainBlockedByPDB,
+				eventActionDrain,
+				drainBlockedByPDBNote{Blocks: blocks},
+			)
+		}
+	}
+
+	rs.pdbBlocked = blocked
 	return nextCheck
 }
 

internal/controller/events_test.go

@@ -18,6 +18,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/fake"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1"
@@ -686,15 +687,19 @@ func TestRecordDrainStalls(t *testing.T) {
 	stalledStatus := &drainStatus{startTime: time.Now().Add(-drainStallThreshold)}
 	freshStatus := &drainStatus{startTime: time.Now()}
 	reconciler := &BootcNodePoolReconciler{
-		Recorder: recorder,
+		Recorder:   recorder,
+		KubeClient: fake.NewSimpleClientset(),
 		drains: map[string]*drainStatus{
 			stalledNode.Name: stalledStatus,
 			freshNode.Name:   freshStatus,
 		},
 	}
+	rs := &rolloutState{}
 
 	nextCheck := reconciler.recordDrainStalls(
+		context.Background(),
 		pool,
+		rs,
 		map[string]*bootcv1alpha1.BootcNode{
 			stalledNode.Name: stalledNode,
 			freshNode.Name:   freshNode,
@@ -703,8 +708,7 @@ func TestRecordDrainStalls(t *testing.T) {
 
 	g.Expect(stalledStatus.isStalled).To(BeTrue())
 	g.Expect(freshStatus.isStalled).To(BeFalse())
-	g.Expect(nextCheck).To(BeNumerically(">", drainStallThreshold-time.Second))
-	g.Expect(nextCheck).To(BeNumerically("<=", drainStallThreshold))
+	g.Expect(nextCheck).To(Equal(drainStallRequeueInterval))
 	g.Expect(recorder.events).To(Equal([]capturedEvent{{
 		regarding: stalledNode.Name,
 		related:   pool.Name,
@@ -719,7 +723,9 @@ func TestRecordDrainStalls(t *testing.T) {
 
 	// The isStalled marker suppresses duplicate events on later reconciles.
 	reconciler.recordDrainStalls(
+		context.Background(),
 		pool,
+		rs,
 		map[string]*bootcv1alpha1.BootcNode{stalledNode.Name: stalledNode},
 	)
 	g.Expect(recorder.events).To(HaveLen(1))