Skip to content

Commit 69d8dda

Browse files
oliverbaehleroliverbaehler
authored
feat: proxy webhooks (#607)
* feat: add webhook handling and watchdog webhook Signed-off-by: Oliver Bähler <[email protected]> * feat(helm): add webhook values and test Signed-off-by: Oliver Bähler <[email protected]> * fix: disregard delete and generic events for watchdog Signed-off-by: Oliver Bähler <[email protected]> * chore: align makefile$ Signed-off-by: Oliver Bähler <[email protected]> --------- Signed-off-by: Oliver Bähler <[email protected]>
1 parent c338410 commit 69d8dda

File tree

13 files changed

+475
-35
lines changed

13 files changed

+475
-35
lines changed

‎Makefile

+23-12
Original file line numberDiff line numberDiff line change
@@ -106,19 +106,20 @@ helm-docs: docker
106106
helm-lint: docker
107107
@docker run -v "$(SRC_ROOT):/workdir" --entrypoint /bin/sh quay.io/helmpack/chart-testing:v3.3.1 -c "cd /workdir; ct lint --config .github/configs/ct.yaml --lint-conf .github/configs/lintconf.yaml --all --debug"
108108

109-
helm-test: helm-controller-version ct ko-build-all helm-create helm-install helm-destroy
109+
helm-test: helm-controller-version ct helm-create helm-install helm-destroy
110110

111-
helm-install:
112-
@kubectl apply --server-side=true -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.2/cert-manager.yaml
113-
@make install-capsule
114-
@kubectl apply --server-side=true -f https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.58.0/bundle.yaml
111+
helm-test-ct: helm-load-image
115112
@$(CT) install --config $(SRC_ROOT)/.github/configs/ct.yaml --namespace=capsule-system --all --debug
116113

114+
helm-install: install-dependencies helm-test-ct
115+
117116
helm-create: kind
118117
@$(KIND) create cluster --wait=60s --name capsule-charts
119-
@$(KIND) load docker-image --name capsule-charts $(CAPSULE_PROXY_IMG):$(VERSION)
120118
@kubectl create ns capsule-system
121119

120+
helm-load-image: kind helm-controller-version ko-build-all
121+
@$(KIND) load docker-image --name capsule-charts $(CAPSULE_PROXY_IMG):$(VERSION)
122+
122123
helm-destroy: kind
123124
@$(KIND) delete cluster --name capsule-charts
124125

@@ -142,15 +143,10 @@ e2e-build: kind
142143
@echo "Building kubernetes env using Kind $${KIND_K8S_VERSION:-v1.27.0}..."
143144
@$(KIND) create cluster --name capsule --image kindest/node:$${KIND_K8S_VERSION:-v1.27.0} --config ./e2e/kind.yaml --wait=120s \
144145
&& kubectl taint nodes capsule-worker2 key1=value1:NoSchedule
145-
@helm repo add bitnami https://charts.bitnami.com/bitnami
146-
@helm repo update
147-
@helm upgrade --install --namespace metrics-system --create-namespace metrics-server bitnami/metrics-server \
148-
--set apiService.create=true --set "extraArgs[0]=--kubelet-insecure-tls=true" --version 6.2.9
149146
@echo "Waiting for metrics-server pod to be ready for listing metrics"
150-
@kubectl --namespace metrics-system wait --for=condition=ready --timeout=320s pod -l app.kubernetes.io/instance=metrics-server
151147

152148
.PHONY: e2e-install
153-
e2e-install: install-capsule install-capsule-proxy rbac-fix
149+
e2e-install: install-capsule install-dependencies install-capsule-proxy rbac-fix
154150

155151
.PHONY: e2e-load-image
156152
e2e-load-image: kind ko-build-all
@@ -184,12 +180,14 @@ ifeq ($(CAPSULE_PROXY_MODE),http)
184180
--set "image.tag=$(VERSION)" \
185181
--set "options.enableSSL=false" \
186182
--set "options.logLevel=10" \
183+
--set "options.pprof=true" \
187184
--set "service.type=NodePort" \
188185
--set "service.nodePort=" \
189186
--set "kind=DaemonSet" \
190187
--set "daemonset.hostNetwork=true" \
191188
--set "serviceMonitor.enabled=false" \
192189
--set "options.generateCertificates=false" \
190+
--set "webhooks.enabled=true" \
193191
--set "options.extraArgs={--feature-gates=ProxyClusterScoped=true,--feature-gates=ProxyAllNamespaced=true}"
194192
else
195193
@echo "Running in HTTPS mode"
@@ -220,15 +218,28 @@ else
220218
--set "image.pullPolicy=Never" \
221219
--set "image.tag=$(VERSION)" \
222220
--set "options.logLevel=10" \
221+
--set "options.pprof=true" \
223222
--set "service.type=NodePort" \
224223
--set "service.nodePort=" \
225224
--set "kind=DaemonSet" \
226225
--set "daemonset.hostNetwork=true" \
227226
--set "serviceMonitor.enabled=false" \
227+
--set "webhooks.enabled=true" \
228228
--set "options.extraArgs={--feature-gates=ProxyClusterScoped=true,--feature-gates=ProxyAllNamespaced=true}"
229229
endif
230230
@kubectl rollout restart ds capsule-proxy -n capsule-system || true
231231

232+
install-dependencies: install-capsule
233+
@kubectl apply --server-side=true -f https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.58.0/bundle.yaml
234+
@helm repo add cert-manager https://charts.jetstack.io
235+
@helm repo add bitnami https://charts.bitnami.com/bitnami
236+
@helm repo update
237+
@helm upgrade --install cert-manager cert-manager/cert-manager --namespace cert-manager --create-namespace --version 1.16.2 --set crds.enabled=true
238+
@helm upgrade --install --namespace metrics-system --create-namespace metrics-server bitnami/metrics-server \
239+
--set apiService.create=true --set "extraArgs[0]=--kubelet-insecure-tls=true" --version 6.2.9
240+
@kubectl --namespace metrics-system wait --for=condition=ready --timeout=320s pod -l app.kubernetes.io/instance=metrics-server
241+
242+
232243
rbac-fix:
233244
@echo "RBAC customization..."
234245
@kubectl create clusterrole capsule-selfsubjectaccessreviews --verb=create --resource=selfsubjectaccessreviews.authorization.k8s.io

‎charts/capsule-proxy/README.md

+23-1
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ If you only need to make minor customizations, you can specify them on the comma
104104
| global.jobs.certs.topologySpreadConstraints | list | `[]` | Set Topology Spread Constraints |
105105
| global.jobs.certs.ttlSecondsAfterFinished | int | `60` | Sets the ttl in seconds after a finished certgen job is deleted. Set to -1 to never delete. |
106106
| global.jobs.kubectl.affinity | object | `{}` | Set affinity rules |
107-
| global.jobs.kubectl.annotations | object | `{"helm.sh/hook-delete-policy":"before-hook-creation,hook-succeeded"}` | Annotations to add to the certgen job. |
107+
| global.jobs.kubectl.annotations | object | `{}` | Annotations |
108108
| global.jobs.kubectl.image.pullPolicy | string | `"IfNotPresent"` | Set the image pull policy of the helm chart job |
109109
| global.jobs.kubectl.image.registry | string | `"docker.io"` | Set the image repository of the helm chart job |
110110
| global.jobs.kubectl.image.repository | string | `"clastix/kubectl"` | Set the image repository of the helm chart job |
@@ -184,7 +184,9 @@ If you only need to make minor customizations, you can specify them on the comma
184184
| options.listeningPort | int | `9001` | Set the listening port of the capsule-proxy |
185185
| options.logLevel | string | `"4"` | Set the log verbosity of the capsule-proxy with a value from 1 to 10 |
186186
| options.oidcUsernameClaim | string | `"preferred_username"` | Specify if capsule-proxy will use SSL |
187+
| options.pprof | bool | `false` | Enable Pprof for profiling |
187188
| options.rolebindingsResyncPeriod | string | `"10h"` | Set the role bindings reflector resync period, a local cache to store mappings between users and their namespaces. [Use a lower value in case of flaky etcd server connections.](https://github.com/projectcapsule/capsule-proxy/issues/174) |
189+
| options.webhookPort | int | `9443` | Webhook port |
188190

189191
### Cert-Manager Parameters
190192

@@ -203,6 +205,26 @@ You can manage the certificate with the help of [cert-manager](https://cert-mana
203205
| certManager.issuer.kind | string | `"Issuer"` | Set if the cert manager will generate either self-signed or CA signed SSL certificates. Its value will be either Issuer or ClusterIssuer |
204206
| certManager.issuer.name | string | `""` | Set the name of the ClusterIssuer if issuer kind is ClusterIssuer and if cert manager will generate CA signed SSL certificates |
205207

208+
### Webhook Parameters
209+
210+
| Key | Type | Default | Description |
211+
|-----|------|---------|-------------|
212+
| webhooks.certificate.dnsNames | list | `[]` | Additional DNS Names to include in certificate |
213+
| webhooks.certificate.fields | object | `{"privateKey":{"rotationPolicy":"Always"}}` | Additional fields to include in certificate |
214+
| webhooks.certificate.ipAddresses | list | `[]` | Additional IP Addresses to include in certificate |
215+
| webhooks.certificate.uris | list | `[]` | Additional URIs to include in certificate |
216+
| webhooks.enabled | bool | `false` | Enable the usage of mutating and validating webhooks |
217+
| webhooks.service.caBundle | string | `""` | CABundle for the webhook service |
218+
| webhooks.service.name | string | `""` | Custom service name for the webhook service |
219+
| webhooks.service.namespace | string | `""` | Custom service namespace for the webhook service |
220+
| webhooks.service.port | string | `nil` | Custom service port for the webhook service |
221+
| webhooks.service.url | string | `""` | The URL where the capsule webhook services are running (Overwrites cluster scoped service definition) |
222+
| webhooks.watchdog.enabled | bool | `true` | Enable Watchdog Webhook |
223+
| webhooks.watchdog.failurePolicy | string | `"Ignore"` | Ignore failures from the webhook |
224+
| webhooks.watchdog.namespaceSelector | object | `{"matchExpressions":[{"key":"capsule.clastix.io/tenant","operator":"Exists"}]}` | Selects only namespaced items which are within a tenant |
225+
| webhooks.watchdog.rules | list | `[{"apiGroups":["*"],"apiVersions":["*"],"operations":["CREATE","UPDATE"],"resources":["*"],"scope":"Namespaced"}]` | Rules for which Objects and Actions this webhook should be called |
226+
| webhooks.watchdog.timeoutSeconds | string | `"3s"` | Timeout in seconds for mutating webhooks |
227+
206228
### Service Parameters
207229

208230
| Key | Type | Default | Description |

‎charts/capsule-proxy/README.md.gotmpl

+11-1
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ If you only need to make minor customizations, you can specify them on the comma
101101
| Key | Type | Default | Description |
102102
|-----|------|---------|-------------|
103103
{{- range .Values }}
104-
{{- if not (or (hasPrefix "certManager" .Key) (hasPrefix "global" .Key) (hasPrefix "options" .Key) (hasPrefix "service." .Key) (hasPrefix "ingress" .Key) (hasPrefix "autoscaling" .Key) (hasPrefix "serviceMonitor" .Key) ) }}
104+
{{- if not (or (hasPrefix "certManager" .Key) (hasPrefix "webhooks" .Key) (hasPrefix "global" .Key) (hasPrefix "options" .Key) (hasPrefix "service." .Key) (hasPrefix "ingress" .Key) (hasPrefix "autoscaling" .Key) (hasPrefix "serviceMonitor" .Key) ) }}
105105
| {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} |
106106
{{- end }}
107107
{{- end }}
@@ -129,6 +129,16 @@ You can manage the certificate with the help of [cert-manager](https://cert-mana
129129
{{- end }}
130130
{{- end }}
131131

132+
### Webhook Parameters
133+
134+
| Key | Type | Default | Description |
135+
|-----|------|---------|-------------|
136+
{{- range .Values }}
137+
{{- if hasPrefix "webhooks." .Key }}
138+
| {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} |
139+
{{- end }}
140+
{{- end }}
141+
132142

133143
### Service Parameters
134144

‎charts/capsule-proxy/ci/webhook-values.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
webhooks:
2+
enabled: true

‎charts/capsule-proxy/templates/_helpers.tpl

+28
Original file line numberDiff line numberDiff line change
@@ -100,3 +100,31 @@ Render the CLI flag --host values for the self-signed certificate generator
100100
{{- $values = append $values $fullname -}}
101101
{{ join "," $values }}
102102
{{- end -}}
103+
104+
105+
106+
{{/*
107+
Capsule Webhook service (Called with $.Path)
108+
109+
*/}}
110+
{{- define "capsule-proxy.webhooks.service" -}}
111+
{{- include "capsule-proxy.webhooks.cabundle" $.ctx | nindent 0 }}
112+
{{- if $.ctx.Values.webhooks.service.url }}
113+
url: {{ printf "%s/%s" (trimSuffix "/" $.ctx.Values.webhooks.service.url ) (trimPrefix "/" (required "Path is required for the function" $.path)) }}
114+
{{- else }}
115+
service:
116+
name: {{ default (printf "%s-webhook-service" (include "capsule-proxy.fullname" $.ctx)) $.ctx.Values.webhooks.service.name }}
117+
namespace: {{ default $.ctx.Release.Namespace $.ctx.Values.webhooks.service.namespace }}
118+
port: {{ default 443 $.ctx.Values.webhooks.service.port }}
119+
path: {{ required "Path is required for the function" $.path }}
120+
{{- end }}
121+
{{- end }}
122+
123+
{{/*
124+
Capsule Webhook endpoint CA Bundle
125+
*/}}
126+
{{- define "capsule-proxy.webhooks.cabundle" -}}
127+
{{- if $.Values.webhooks.service.caBundle -}}
128+
caBundle: {{ $.Values.webhooks.service.caBundle -}}
129+
{{- end -}}
130+
{{- end -}}

‎charts/capsule-proxy/templates/_pod.tpl

+26
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,13 @@ spec:
3434
secretName: {{ .Values.options.certificateVolumeName | default (include "capsule-proxy.fullname" .) }}
3535
defaultMode: 420
3636
{{- end }}
37+
{{- if .Values.webhooks.enabled }}
38+
- name: webhook
39+
secret:
40+
secretName: {{ include "capsule-proxy.fullname" . }}-webhook-cert
41+
defaultMode: 420
42+
{{- end }}
43+
3744
{{- with .Values.topologySpreadConstraints }}
3845
topologySpreadConstraints: {{- toYaml . | nindent 4 }}
3946
{{- end }}
@@ -45,6 +52,7 @@ spec:
4552
imagePullPolicy: {{ .Values.image.pullPolicy }}
4653
args:
4754
- --listening-port={{ .Values.options.listeningPort }}
55+
- --webhook-port={{ .Values.options.webhookPort }}
4856
- --capsule-configuration-name={{ .Values.options.capsuleConfigurationName }}
4957
{{- range .Values.options.ignoredUserGroups }}
5058
- --ignored-user-group={{ . }}
@@ -61,6 +69,12 @@ spec:
6169
{{- end }}
6270
- --client-connection-qps={{ .Values.options.clientConnectionQPS }}
6371
- --client-connection-burst={{ .Values.options.clientConnectionBurst }}
72+
- --enable-pprof={{ .Values.options.pprof }}
73+
{{- if .Values.webhooks.enabled }}
74+
{{- if .Values.webhooks.watchdog.enabled }}
75+
- --webhooks=watchdog
76+
{{- end }}
77+
{{- end }}
6478
{{- with .Values.options.extraArgs }}
6579
{{- toYaml . | nindent 4 }}
6680
{{- end }}
@@ -83,9 +97,16 @@ spec:
8397
- name: probe
8498
containerPort: 8081
8599
protocol: TCP
100+
{{- if .Values.options.pprof }}
86101
- name: pprof
87102
containerPort: 8082
88103
protocol: TCP
104+
{{- end }}
105+
{{- if .Values.webhooks.enabled }}
106+
- name: webhook
107+
containerPort: {{ .Values.options.webhookPort }}
108+
protocol: TCP
109+
{{- end }}
89110
{{- if .Values.livenessProbe.enabled }}
90111
livenessProbe:
91112
{{- toYaml (omit .Values.livenessProbe "enabled") | nindent 6 }}
@@ -104,6 +125,11 @@ spec:
104125
- mountPath: {{ .Values.options.SSLDirectory }}
105126
name: certs
106127
{{- end }}
128+
{{- if .Values.webhooks.enabled }}
129+
- mountPath: /tmp/k8s-webhook-server/serving-certs
130+
name: webhook
131+
readOnly: true
132+
{{- end }}
107133
{{- with .Values.nodeSelector }}
108134
nodeSelector:
109135
{{- toYaml . | nindent 8 }}

‎charts/capsule-proxy/templates/webhooks/certificate.yaml

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{{- if $.Values.webhooks.enabled }}
2+
---
3+
apiVersion: cert-manager.io/v1
4+
kind: Issuer
5+
metadata:
6+
name: {{ include "capsule-proxy.fullname" . }}-webhook-issuer
7+
spec:
8+
selfSigned: {}
9+
---
10+
apiVersion: cert-manager.io/v1
11+
kind: Certificate
12+
metadata:
13+
name: {{ include "capsule-proxy.fullname" . }}-webhook-ca
14+
spec:
15+
isCA: true
16+
commonName: {{ include "capsule-proxy.fullname" . }}-webhook-ca
17+
secretName: {{ include "capsule-proxy.fullname" . }}-webhook-ca
18+
privateKey:
19+
algorithm: ECDSA
20+
size: 256
21+
issuerRef:
22+
name: {{ include "capsule-proxy.fullname" . }}-webhook-issuer
23+
kind: Issuer
24+
group: cert-manager.io
25+
---
26+
apiVersion: cert-manager.io/v1
27+
kind: Issuer
28+
metadata:
29+
name: {{ include "capsule-proxy.fullname" . }}-webhook
30+
spec:
31+
ca:
32+
secretName: {{ include "capsule-proxy.fullname" . }}-webhook-ca
33+
---
34+
apiVersion: cert-manager.io/v1
35+
kind: Certificate
36+
metadata:
37+
name: {{ include "capsule-proxy.fullname" . }}-webhook-cert
38+
spec:
39+
{{- with .Values.webhooks.certificate.fields }}
40+
{{ toYaml . | nindent 2 }}
41+
{{- end }}
42+
dnsNames:
43+
{{- range $dns := .Values.webhooks.certificate.dnsNames }}
44+
- {{ $dns | quote }}
45+
{{- end }}
46+
- {{ include "capsule-proxy.fullname" . }}-webhook-service
47+
- {{ include "capsule-proxy.fullname" . }}-webhook-service.{{ .Release.Namespace }}.svc
48+
{{- with .Values.webhooks.certificate.ipAddresses }}
49+
ipAddresses:
50+
{{- range $ip := . }}
51+
- {{ $ip }}
52+
{{- end }}
53+
{{- end }}
54+
{{- with .Values.webhooks.certificate.uris }}
55+
uris:
56+
{{- range $uri := . }}
57+
- {{ $uri }}
58+
{{- end }}
59+
{{- end }}
60+
issuerRef:
61+
kind: "Issuer"
62+
name: {{ include "capsule-proxy.fullname" . }}-webhook
63+
secretName: {{ include "capsule-proxy.fullname" . }}-webhook-cert
64+
subject:
65+
organizations:
66+
- projectcapsule.dev
67+
{{- end }}

‎charts/capsule-proxy/templates/webhooks/mutating.yaml

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{{- if $.Values.webhooks.enabled }}
2+
apiVersion: admissionregistration.k8s.io/v1
3+
kind: MutatingWebhookConfiguration
4+
metadata:
5+
name: {{ include "capsule-proxy.fullname" . }}-webhook
6+
labels:
7+
{{- include "capsule-proxy.labels" . | nindent 4 }}
8+
annotations:
9+
cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "capsule-proxy.fullname" . }}-webhook-cert
10+
webhooks:
11+
{{- with .Values.webhooks.watchdog }}
12+
{{- if .enabled }}
13+
- admissionReviewVersions:
14+
- v1
15+
clientConfig:
16+
{{- include "capsule-proxy.webhooks.service" (dict "path" "/mutate/watchdog" "ctx" $) | nindent 4 }}
17+
failurePolicy: {{ .failurePolicy }}
18+
name: watchdog.proxy.projectcapsule.dev
19+
{{- with .rules }}
20+
rules:
21+
{{- toYaml .| nindent 4}}
22+
{{- end }}
23+
{{- with .namespaceSelector }}
24+
namespaceSelector:
25+
{{- toYaml .| nindent 4}}
26+
{{- end }}
27+
sideEffects: None
28+
timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }}
29+
{{- end }}
30+
{{- end }}
31+
{{- end }}

‎charts/capsule-proxy/templates/webhooks/service.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{{- if $.Values.webhooks.enabled }}
2+
apiVersion: v1
3+
kind: Service
4+
metadata:
5+
name: {{ include "capsule-proxy.fullname" . }}-webhook-service
6+
labels:
7+
{{- include "capsule-proxy.labels" . | nindent 4 }}
8+
spec:
9+
ports:
10+
- port: 443
11+
name: https
12+
protocol: TCP
13+
targetPort: {{ .Values.options.webhookPort }}
14+
selector:
15+
{{- include "capsule-proxy.selectorLabels" . | nindent 4 }}
16+
sessionAffinity: None
17+
type: ClusterIP
18+
{{- end }}

0 commit comments

Comments
 (0)