support k8s 1.21 - 1.23

i.e., drop support for k8s 1.17 - 1.20, add support for k8s 1.22 - 1.23 (and likely future versions until something breaks)

upgrade go to 1.17 and k8s.io dependencies, controller-runtime, vk, and minor dependencies to latest package versions, luckily cross-compatible (we might have still supported k8s 1.19 - 1.20 with older versions; not worth the maintenance cost)

k8s 1.20- aren't supported anymore because schedulers watch PodDisruptionBudgets using the policy/v1 API version, available since v1.21

use the certificates.k8s.io/v1 API version for CertificateSigningRequest

use the networking.k8s.io/v1 API version for Ingress

use the coordination.k8s.io/v1 API version for Lease

upgrade cert-manager, argo, and our own CRDs and webhook configuration because 1.22 dropped support for admissionregistration.k8s.io/v1beta1 and apiextensions.k8s.io/v1beta1

move CRDs to templates/ (from crds/, which is actually meant for CRDs that have objects created in the same chart) so they're updated on helm upgrades (cert-manager did the same move)

upgrade to go 1.17

Signed-off-by: adrienjt <[email protected]>

Author: adrienjt

.github/workflows/go.yml

@@ -52,7 +52,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        k8s_version: [ "1.17", "1.18", "1.19", "1.20", "1.21" ]
+        k8s_version: [ "1.21", "1.22", "1.23" ]
         experimental: [ false ]
         # workflow succeeds even if experimental job fails,
         # but commit/PR check/status still appears as failure overall,

charts/multicluster-scheduler/crds/clustersource.yaml

@@ -1,4 +1,4 @@
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: {{ include "fullname" . }}

charts/multicluster-scheduler/crds/clustersummary.yaml

@@ -5,7 +5,7 @@ metadata:
   labels: {{ include "labels" . | nindent 4 }}
 data:
   proxy-scheduler-config: |
-    apiVersion: kubescheduler.config.k8s.io/v1alpha2
+    apiVersion: kubescheduler.config.k8s.io/v1beta3
     kind: KubeSchedulerConfiguration
     leaderElection:
       leaderElect: true
@@ -15,20 +15,11 @@ data:
     profiles:
       - schedulerName: admiralty-proxy
         plugins:
-          filter:
-            enabled:
-              - name: proxy
-          reserve:
-            enabled:
-              - name: proxy
-          preBind:
-            enabled:
-              - name: proxy
-          postBind:
+          multiPoint:
             enabled:
               - name: proxy
   candidate-scheduler-config: |
-    apiVersion: kubescheduler.config.k8s.io/v1alpha2
+    apiVersion: kubescheduler.config.k8s.io/v1beta3
     kind: KubeSchedulerConfiguration
     leaderElection:
       leaderElect: true
@@ -38,12 +29,6 @@ data:
     profiles:
       - schedulerName: admiralty-candidate
         plugins:
-          preFilter:
-            enabled:
-              - name: candidate
-          reserve:
-            enabled:
-              - name: candidate
-          preBind:
+          multiPoint:
             enabled:
               - name: candidate

charts/multicluster-scheduler/crds/clustertarget.yaml

@@ -154,6 +154,7 @@ rules:
     resources:
       - leases # only needed in kube-node-leases namespaces (TODO: split and bind in RoleBinding)
     verbs:
+      - get
       - create
       - update
       - delete

charts/multicluster-scheduler/crds/podchaperon.yaml

@@ -0,0 +1,38 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: clustersources.multicluster.admiralty.io
+  labels: {{ include "labels" . | nindent 4 }}
+spec:
+  group: multicluster.admiralty.io
+  names:
+    kind: ClusterSource
+    plural: clustersources
+    shortNames:
+      - csrc
+  scope: Cluster
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      subresources:
+        status: { }
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                userName:
+                  type: string
+                serviceAccount:
+                  type: object
+                  properties:
+                    name:
+                      type: string
+                    namespace:
+                      type: string
+            status:
+              type: object

charts/multicluster-scheduler/crds/source.yaml

@@ -0,0 +1,32 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: clustersummaries.multicluster.admiralty.io
+  labels: {{ include "labels" . | nindent 4 }}
+spec:
+  group: multicluster.admiralty.io
+  names:
+    kind: ClusterSummary
+    plural: clustersummaries
+    shortNames:
+      - mcsum
+  scope: Cluster
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      subresources:
+        status: { }
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            capacity:
+              type: object
+              additionalProperties:
+                x-kubernetes-int-or-string: true
+            allocatable:
+              type: object
+              additionalProperties:
+                x-kubernetes-int-or-string: true

charts/multicluster-scheduler/crds/target.yaml

@@ -0,0 +1,44 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: clustertargets.multicluster.admiralty.io
+  labels: {{ include "labels" . | nindent 4 }}
+spec:
+  group: multicluster.admiralty.io
+  names:
+    kind: ClusterTarget
+    plural: clustertargets
+    shortNames:
+      - ctg
+  scope: Cluster
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      subresources:
+        status: { }
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                self:
+                  type: boolean
+                kubeconfigSecret:
+                  type: object
+                  properties:
+                    name:
+                      type: string
+                    namespace:
+                      type: string
+                    key:
+                      type: string
+                    context:
+                      type: string
+                excludedLabelsRegexp:
+                  type: string
+            status:
+              type: object

charts/multicluster-scheduler/templates/cert.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: podchaperons.multicluster.admiralty.io
+  labels: {{ include "labels" . | nindent 4 }}
+spec:
+  group: multicluster.admiralty.io
+  names:
+    kind: PodChaperon
+    plural: podchaperons
+    shortNames:
+      - chap
+  scope: Namespaced
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      subresources:
+        status: { }
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              x-kubernetes-preserve-unknown-fields: true
+              # TODO generate
+            status:
+              type: object
+              x-kubernetes-preserve-unknown-fields: true
+              # TODO generate
+      additionalPrinterColumns:
+        - name: reserved
+          type: string
+          jsonPath: .metadata.annotations.multicluster\.admiralty\.io/is-reserved
+        - name: allowed
+          type: string
+          jsonPath: .metadata.annotations.multicluster\.admiralty\.io/is-allowed

charts/multicluster-scheduler/templates/cm.yaml

@@ -0,0 +1,33 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: sources.multicluster.admiralty.io
+  labels: {{ include "labels" . | nindent 4 }}
+spec:
+  group: multicluster.admiralty.io
+  names:
+    kind: Source
+    plural: sources
+    shortNames:
+      - src
+  scope: Namespaced
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      subresources:
+        status: { }
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                userName:
+                  type: string
+                serviceAccountName:
+                  type: string
+            status:
+              type: object