Skip to content

Commit bd06f2c

Browse files
Update observedGeneration in reconcileStatus flow instead of reconcileSpec flow (#996)
* Update `ObservedGeneration` in reconcileStatus flow instead of reconcileSpec flow * Compute `canReconcileSpec` only once; use `operatorContext` to pass reconciliation information to subsequent steps * Address review comments from @unmarshall: simplify `operatorContext.Data` and `GetBoolValueOrDefault()` function * Address review comments from @unmarshall: change `GetBoolValueOrDefault()` function to `GetBoolValueOrError()` to not swallow errors * Address review comments from @unmarshall: add `r.completeReconcile()` which runs `updateObservedGeneration()` and `removeOPerationAnnotation()` * revert minor change in code
1 parent 2fc82d0 commit bd06f2c

File tree

6 files changed

+315
-69
lines changed

6 files changed

+315
-69
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package etcd
2+
3+
import (
4+
druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1"
5+
"github.com/gardener/etcd-druid/internal/component"
6+
ctrlutils "github.com/gardener/etcd-druid/internal/controller/utils"
7+
8+
v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
"sigs.k8s.io/controller-runtime/pkg/client"
11+
)
12+
13+
func (r *Reconciler) completeReconcile(ctx component.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult {
14+
rLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "completeReconcile").WithValues("runID", ctx.RunID)
15+
ctx.SetLogger(rLog)
16+
17+
reconcileCompletionStepFns := []reconcileFn{
18+
r.updateObservedGeneration,
19+
r.removeOperationAnnotation,
20+
}
21+
22+
for _, fn := range reconcileCompletionStepFns {
23+
if stepResult := fn(ctx, etcdObjectKey); ctrlutils.ShortCircuitReconcileFlow(stepResult) {
24+
return r.recordIncompleteReconcileOperation(ctx, etcdObjectKey, stepResult)
25+
}
26+
}
27+
ctx.Logger.Info("Finished reconciliation completion flow")
28+
return ctrlutils.ContinueReconcile()
29+
}
30+
31+
func (r *Reconciler) updateObservedGeneration(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult {
32+
etcd := &druidv1alpha1.Etcd{}
33+
if result := ctrlutils.GetLatestEtcd(ctx, r.client, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) {
34+
return result
35+
}
36+
originalEtcd := etcd.DeepCopy()
37+
etcd.Status.ObservedGeneration = &etcd.Generation
38+
if err := r.client.Status().Patch(ctx, etcd, client.MergeFrom(originalEtcd)); err != nil {
39+
ctx.Logger.Error(err, "failed to patch status.ObservedGeneration")
40+
return ctrlutils.ReconcileWithError(err)
41+
}
42+
ctx.Logger.Info("patched status.ObservedGeneration", "ObservedGeneration", etcd.Generation)
43+
return ctrlutils.ContinueReconcile()
44+
}
45+
46+
func (r *Reconciler) removeOperationAnnotation(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult {
47+
etcdPartialObjMeta := ctrlutils.EmptyEtcdPartialObjectMetadata()
48+
if result := ctrlutils.GetLatestEtcdPartialObjectMeta(ctx, r.client, etcdObjKey, etcdPartialObjMeta); ctrlutils.ShortCircuitReconcileFlow(result) {
49+
return result
50+
}
51+
52+
if metav1.HasAnnotation(etcdPartialObjMeta.ObjectMeta, v1beta1constants.GardenerOperation) {
53+
ctx.Logger.Info("Removing operation annotation")
54+
withOpAnnotation := etcdPartialObjMeta.DeepCopy()
55+
delete(etcdPartialObjMeta.Annotations, v1beta1constants.GardenerOperation)
56+
if err := r.client.Patch(ctx, etcdPartialObjMeta, client.MergeFrom(withOpAnnotation)); err != nil {
57+
ctx.Logger.Error(err, "failed to remove operation annotation")
58+
return ctrlutils.ReconcileWithError(err)
59+
}
60+
}
61+
return ctrlutils.ContinueReconcile()
62+
}

internal/controller/etcd/reconcile_spec.go

+6-44
Original file line numberDiff line numberDiff line change
@@ -17,29 +17,23 @@ import (
1717
v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
1818
"github.com/gardener/gardener/pkg/controllerutils"
1919
corev1 "k8s.io/api/core/v1"
20-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2120
"sigs.k8s.io/controller-runtime/pkg/client"
2221
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
2322
)
2423

2524
// syncRetryInterval will be used by both sync and preSync stages for a component and should be used when there is a need to requeue for retrying after a specific interval.
2625
const syncRetryInterval = 10 * time.Second
2726

28-
func (r *Reconciler) triggerReconcileSpecFlow(ctx component.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult {
27+
func (r *Reconciler) reconcileSpec(ctx component.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult {
28+
rLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "reconcileSpec").WithValues("runID", ctx.RunID)
29+
ctx.SetLogger(rLog)
30+
2931
reconcileStepFns := []reconcileFn{
3032
r.recordReconcileStartOperation,
3133
r.ensureFinalizer,
3234
r.preSyncEtcdResources,
3335
r.syncEtcdResources,
34-
r.updateObservedGeneration,
3536
r.recordReconcileSuccessOperation,
36-
// Removing the operation annotation after last operation recording seems counter-intuitive.
37-
// If we reverse the order where we first remove the operation annotation and then record the last operation then
38-
// in case the operation annotation removal succeeds but the last operation recording fails, then the control
39-
// will never enter this flow again and the last operation will never be recorded.
40-
// Reason: there is a predicate check done in `reconciler.canReconcile` prior to entering this flow.
41-
// That check will no longer succeed once the reconcile operation annotation has been removed.
42-
r.removeOperationAnnotation,
4337
}
4438

4539
for _, fn := range reconcileStepFns {
@@ -51,23 +45,6 @@ func (r *Reconciler) triggerReconcileSpecFlow(ctx component.OperatorContext, etc
5145
return ctrlutils.ContinueReconcile()
5246
}
5347

54-
func (r *Reconciler) removeOperationAnnotation(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult {
55-
etcdPartialObjMeta := ctrlutils.EmptyEtcdPartialObjectMetadata()
56-
if result := ctrlutils.GetLatestEtcdPartialObjectMeta(ctx, r.client, etcdObjKey, etcdPartialObjMeta); ctrlutils.ShortCircuitReconcileFlow(result) {
57-
return result
58-
}
59-
if metav1.HasAnnotation(etcdPartialObjMeta.ObjectMeta, v1beta1constants.GardenerOperation) {
60-
ctx.Logger.Info("Removing operation annotation")
61-
withOpAnnotation := etcdPartialObjMeta.DeepCopy()
62-
delete(etcdPartialObjMeta.Annotations, v1beta1constants.GardenerOperation)
63-
if err := r.client.Patch(ctx, etcdPartialObjMeta, client.MergeFrom(withOpAnnotation)); err != nil {
64-
ctx.Logger.Error(err, "failed to remove operation annotation")
65-
return ctrlutils.ReconcileWithError(err)
66-
}
67-
}
68-
return ctrlutils.ContinueReconcile()
69-
}
70-
7148
func (r *Reconciler) ensureFinalizer(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult {
7249
etcdPartialObjMeta := ctrlutils.EmptyEtcdPartialObjectMetadata()
7350
if result := ctrlutils.GetLatestEtcdPartialObjectMeta(ctx, r.client, etcdObjKey, etcdPartialObjMeta); ctrlutils.ShortCircuitReconcileFlow(result) {
@@ -123,21 +100,6 @@ func (r *Reconciler) syncEtcdResources(ctx component.OperatorContext, etcdObjKey
123100
return ctrlutils.ContinueReconcile()
124101
}
125102

126-
func (r *Reconciler) updateObservedGeneration(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult {
127-
etcd := &druidv1alpha1.Etcd{}
128-
if result := ctrlutils.GetLatestEtcd(ctx, r.client, etcdObjKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) {
129-
return result
130-
}
131-
originalEtcd := etcd.DeepCopy()
132-
etcd.Status.ObservedGeneration = &etcd.Generation
133-
if err := r.client.Status().Patch(ctx, etcd, client.MergeFrom(originalEtcd)); err != nil {
134-
ctx.Logger.Error(err, "failed to patch status.ObservedGeneration")
135-
return ctrlutils.ReconcileWithError(err)
136-
}
137-
ctx.Logger.Info("patched status.ObservedGeneration", "ObservedGeneration", etcd.Generation)
138-
return ctrlutils.ContinueReconcile()
139-
}
140-
141103
func (r *Reconciler) recordReconcileStartOperation(ctx component.OperatorContext, etcdObjKey client.ObjectKey) ctrlutils.ReconcileStepResult {
142104
if err := r.lastOpErrRecorder.RecordStart(ctx, etcdObjKey, druidv1alpha1.LastOperationTypeReconcile); err != nil {
143105
ctx.Logger.Error(err, "failed to record etcd reconcile start operation")
@@ -162,14 +124,14 @@ func (r *Reconciler) recordIncompleteReconcileOperation(ctx component.OperatorCo
162124
return exitReconcileStepResult
163125
}
164126

165-
// canReconcileSpec assesses whether the Etcd spec should undergo reconciliation.
127+
// shouldReconcileSpec assesses whether the Etcd spec should undergo reconciliation.
166128
//
167129
// Reconciliation decision follows these rules:
168130
// - Skipped if 'druid.gardener.cloud/suspend-etcd-spec-reconcile' annotation is present, signaling a pause in reconciliation.
169131
// - Automatic reconciliation occurs if EnableEtcdSpecAutoReconcile is true.
170132
// - If 'gardener.cloud/operation: reconcile' annotation exists and 'druid.gardener.cloud/suspend-etcd-spec-reconcile' annotation is not set, reconciliation proceeds upon Etcd spec changes.
171133
// - Reconciliation is not initiated if EnableEtcdSpecAutoReconcile is false and none of the relevant annotations are present.
172-
func (r *Reconciler) canReconcileSpec(etcd *druidv1alpha1.Etcd) bool {
134+
func (r *Reconciler) shouldReconcileSpec(etcd *druidv1alpha1.Etcd) bool {
173135
// Check if spec reconciliation has been suspended, if yes, then record the event and return false.
174136
if suspendReconcileAnnotKey := druidv1alpha1.GetSuspendEtcdSpecReconcileAnnotationKey(etcd.ObjectMeta); suspendReconcileAnnotKey != nil {
175137
r.recordEtcdSpecReconcileSuspension(etcd, *suspendReconcileAnnotKey)

internal/controller/etcd/reconciler.go

+35-19
Original file line numberDiff line numberDiff line change
@@ -85,28 +85,57 @@ type reconcileFn func(ctx component.OperatorContext, objectKey client.ObjectKey)
8585
// Reconcile manages the reconciliation of the Etcd component to align it with its desired specifications.
8686
//
8787
// The reconciliation process involves the following steps:
88-
// 1. Deletion Handling: If the Etcd component has a deletionTimestamp, initiate the deletion workflow. On error, requeue the request.
89-
// 2. Spec Reconciliation : Determine whether the Etcd spec should be reconciled based on annotations and flags and if there is a need then reconcile spec.
90-
// 3. Status Reconciliation: Always update the status of the Etcd component to reflect its current state.
91-
// 4. Scheduled Requeue: Requeue the reconciliation request after a defined period (EtcdStatusSyncPeriod) to maintain sync.
88+
// 1. Deletion Handling: If the Etcd component has a deletionTimestamp, initiate the deletion workflow.
89+
// On error, requeue the request.
90+
// 2. Spec Reconciliation : Determine whether the Etcd spec should be reconciled based on annotations and flags,
91+
// and if there is a need then reconcile spec.
92+
// 3. Status Reconciliation: Always update the status of the Etcd component to reflect its current state,
93+
// as well as status fields derived from spec reconciliation.
94+
// 4. Remove operation-reconcile annotation if it was set and if spec reconciliation had succeeded.
95+
// 5. Scheduled Requeue: Requeue the reconciliation request after a defined period (EtcdStatusSyncPeriod) to maintain sync.
9296
func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
9397
runID := string(controller.ReconcileIDFromContext(ctx))
9498
operatorCtx := component.NewOperatorContext(ctx, r.logger, runID)
9599
if result := r.reconcileEtcdDeletion(operatorCtx, req.NamespacedName); ctrlutils.ShortCircuitReconcileFlow(result) {
96100
return result.ReconcileResult()
97101
}
102+
103+
etcd := &druidv1alpha1.Etcd{}
104+
if result := ctrlutils.GetLatestEtcd(ctx, r.client, req.NamespacedName, etcd); ctrlutils.ShortCircuitReconcileFlow(result) {
105+
return result.ReconcileResult()
106+
}
107+
shouldReconcileSpec := r.shouldReconcileSpec(etcd)
108+
98109
var reconcileSpecResult ctrlutils.ReconcileStepResult
99-
if result := r.reconcileSpec(operatorCtx, req.NamespacedName); ctrlutils.ShortCircuitReconcileFlow(result) {
100-
reconcileSpecResult = result
110+
if shouldReconcileSpec {
111+
reconcileSpecResult = r.reconcileSpec(operatorCtx, req.NamespacedName)
101112
}
102113

103114
if result := r.reconcileStatus(operatorCtx, req.NamespacedName); ctrlutils.ShortCircuitReconcileFlow(result) {
104115
r.logger.Error(result.GetCombinedError(), "Failed to reconcile status")
105116
return result.ReconcileResult()
106117
}
118+
107119
if reconcileSpecResult.NeedsRequeue() {
108120
return reconcileSpecResult.ReconcileResult()
109121
}
122+
123+
// Spec reconciliation involves some steps that must be executed after status reconciliation,
124+
// to ensure consistency of the status and to ensure that any intermittent failures result in a
125+
// requeue to re-attempt the spec reconciliation.
126+
// Specifically, status.observedGeneration must be updated after the rest of the status fields are updated,
127+
// because consumers of the Etcd status must check the observed generation to confirm that reconciliation is
128+
// in fact complete, and the status fields reflect the latest possible state of the etcd cluster after the
129+
// spec was reconciled.
130+
// Additionally, the operation annotation needs to be removed only at the end of reconciliation, to ensure that
131+
// if any failure is encountered during reconciliation, then reconciliation is re-attempted upon the next requeue.
132+
// r.completeReconcile() is executed only if the spec was reconciled, as denoted by the `shouldReconcileSpec` flag.
133+
if shouldReconcileSpec {
134+
if result := r.completeReconcile(operatorCtx, req.NamespacedName); ctrlutils.ShortCircuitReconcileFlow(result) {
135+
return result.ReconcileResult()
136+
}
137+
}
138+
110139
return ctrlutils.ReconcileAfter(r.config.EtcdStatusSyncPeriod, "Periodic Requeue").ReconcileResult()
111140
}
112141

@@ -142,16 +171,3 @@ func (r *Reconciler) reconcileEtcdDeletion(ctx component.OperatorContext, etcdOb
142171
}
143172
return ctrlutils.ContinueReconcile()
144173
}
145-
146-
func (r *Reconciler) reconcileSpec(ctx component.OperatorContext, etcdObjectKey client.ObjectKey) ctrlutils.ReconcileStepResult {
147-
etcd := &druidv1alpha1.Etcd{}
148-
if result := ctrlutils.GetLatestEtcd(ctx, r.client, etcdObjectKey, etcd); ctrlutils.ShortCircuitReconcileFlow(result) {
149-
return result
150-
}
151-
if r.canReconcileSpec(etcd) {
152-
rLog := r.logger.WithValues("etcd", etcdObjectKey, "operation", "reconcileSpec").WithValues("runID", ctx.RunID)
153-
ctx.SetLogger(rLog)
154-
return r.triggerReconcileSpecFlow(ctx, etcdObjectKey)
155-
}
156-
return ctrlutils.ContinueReconcile()
157-
}

internal/health/condition/check_all_members_updated_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ package condition_test
66

77
import (
88
"context"
9-
"k8s.io/utils/pointer"
109

1110
druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1"
1211
"github.com/gardener/etcd-druid/internal/health/condition"
@@ -15,6 +14,7 @@ import (
1514
appsv1 "k8s.io/api/apps/v1"
1615
apierrors "k8s.io/apimachinery/pkg/api/errors"
1716
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
17+
"k8s.io/utils/pointer"
1818
"sigs.k8s.io/controller-runtime/pkg/client"
1919

2020
. "github.com/onsi/ginkgo/v2"

0 commit comments

Comments
 (0)