Skip to content

Commit 5b7b6da

Browse files
committed
feat: Implement paging on list operations
1 parent 4c127cb commit 5b7b6da

File tree

1 file changed

+39
-26
lines changed

1 file changed

+39
-26
lines changed

main.go

+39-26
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
"github.com/xhit/go-str2duration/v2"
1313
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
1415
"k8s.io/apimachinery/pkg/runtime/schema"
1516
"k8s.io/client-go/dynamic"
1617
"k8s.io/client-go/kubernetes"
@@ -23,6 +24,8 @@ const (
2324
ExecutionTimeout = 20 * time.Minute // Maximum time for each reconciliation before timing out
2425
ExecutionInterval = 5 * time.Minute // Interval between each reconciliation
2526
ThrottleDuration = 50 * time.Millisecond // Duration to sleep for throttling purposes
27+
28+
ListLimit = 500 // Maximum number of items to list at once
2629
)
2730

2831
var (
@@ -109,39 +112,49 @@ func DoReconcile(dynamicClient dynamic.Interface, resources []*metav1.APIResourc
109112
}
110113
// List all items under the resource
111114
gvr.Resource = apiResource.Name
112-
list, err := dynamicClient.Resource(gvr).List(context.TODO(), metav1.ListOptions{TimeoutSeconds: &listTimeoutSeconds})
113-
if err != nil {
114-
log.Printf("Error checking %s from %s: %s", gvr.Resource, gvr.GroupVersion(), err)
115-
continue
116-
}
117-
if debug {
118-
log.Println("Checking", len(list.Items), gvr.Resource, "from", gvr.GroupVersion())
119-
}
120-
for _, item := range list.Items {
121-
ttl, exists := item.GetAnnotations()[AnnotationTTL]
122-
if !exists {
123-
continue
124-
}
125-
ttlInDuration, err := str2duration.ParseDuration(ttl)
115+
var list *unstructured.UnstructuredList
116+
var continueToken string
117+
var err error
118+
for list == nil || continueToken != "" {
119+
list, err = dynamicClient.Resource(gvr).List(context.TODO(), metav1.ListOptions{TimeoutSeconds: &listTimeoutSeconds, Continue: continueToken, Limit: ListLimit})
126120
if err != nil {
127-
log.Printf("[%s/%s] has an invalid TTL '%s': %s\n", apiResource.Name, item.GetName(), ttl, err)
121+
log.Printf("Error checking %s from %s: %s", gvr.Resource, gvr.GroupVersion(), err)
128122
continue
129123
}
130-
ttlExpired := time.Now().After(item.GetCreationTimestamp().Add(ttlInDuration))
131-
if ttlExpired {
132-
log.Printf("[%s/%s] is configured with a TTL of %s, which means it has expired %s ago", apiResource.Name, item.GetName(), ttl, time.Since(item.GetCreationTimestamp().Add(ttlInDuration)).Round(time.Second))
133-
err := dynamicClient.Resource(gvr).Namespace(item.GetNamespace()).Delete(context.TODO(), item.GetName(), metav1.DeleteOptions{})
124+
if list != nil {
125+
continueToken = list.GetContinue()
126+
}
127+
if debug {
128+
log.Println("Checking", len(list.Items), gvr.Resource, "from", gvr.GroupVersion())
129+
}
130+
for _, item := range list.Items {
131+
ttl, exists := item.GetAnnotations()[AnnotationTTL]
132+
if !exists {
133+
continue
134+
}
135+
ttlInDuration, err := str2duration.ParseDuration(ttl)
134136
if err != nil {
135-
log.Printf("[%s/%s] failed to delete: %s\n", apiResource.Name, item.GetName(), err)
136-
// XXX: Should we retry with GracePeriodSeconds set to &0 to force immediate deletion after the first attempt failed?
137+
log.Printf("[%s/%s] has an invalid TTL '%s': %s\n", apiResource.Name, item.GetName(), ttl, err)
138+
continue
139+
}
140+
ttlExpired := time.Now().After(item.GetCreationTimestamp().Add(ttlInDuration))
141+
if ttlExpired {
142+
log.Printf("[%s/%s] is configured with a TTL of %s, which means it has expired %s ago", apiResource.Name, item.GetName(), ttl, time.Since(item.GetCreationTimestamp().Add(ttlInDuration)).Round(time.Second))
143+
err := dynamicClient.Resource(gvr).Namespace(item.GetNamespace()).Delete(context.TODO(), item.GetName(), metav1.DeleteOptions{})
144+
if err != nil {
145+
log.Printf("[%s/%s] failed to delete: %s\n", apiResource.Name, item.GetName(), err)
146+
// XXX: Should we retry with GracePeriodSeconds set to &0 to force immediate deletion after the first attempt failed?
147+
} else {
148+
log.Printf("[%s/%s] deleted", apiResource.Name, item.GetName())
149+
}
150+
// Cool off a tiny bit to avoid hitting the API too often
151+
time.Sleep(ThrottleDuration)
137152
} else {
138-
log.Printf("[%s/%s] deleted", apiResource.Name, item.GetName())
153+
log.Printf("[%s/%s] is configured with a TTL of %s, which means it will expire in %s", apiResource.Name, item.GetName(), ttl, time.Until(item.GetCreationTimestamp().Add(ttlInDuration)).Round(time.Second))
139154
}
140-
// Cool off a tiny bit to avoid hitting the API too often
141-
time.Sleep(ThrottleDuration)
142-
} else {
143-
log.Printf("[%s/%s] is configured with a TTL of %s, which means it will expire in %s", apiResource.Name, item.GetName(), ttl, time.Until(item.GetCreationTimestamp().Add(ttlInDuration)).Round(time.Second))
144155
}
156+
// Cool off a tiny bit to avoid hitting the API too often
157+
time.Sleep(ThrottleDuration)
145158
}
146159
// Cool off a tiny bit to avoid hitting the API too often
147160
time.Sleep(ThrottleDuration)

0 commit comments

Comments
 (0)