fix!(kubeadm): cluster-info configmap reconciliation (#715)

This commit introduces a breaking change such as the removal of
the default bootstrap token created by kubeadm on an idempotent basis.

Signed-off-by: Dario Tranchitella <[email protected]>

Author: prometherion

internal/resources/kubeadm_phases.go

@@ -15,7 +15,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	clientset "k8s.io/client-go/kubernetes"
-	bootstraptokenv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/bootstraptoken/v1"
+	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -24,7 +24,6 @@ import (
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/internal/kubeadm"
-	"github.com/clastix/kamaji/internal/resources/utils"
 )
 
 type kubeadmPhase int
@@ -53,7 +52,7 @@ func (r *KubeadmPhase) GetWatchedObject() client.Object {
 	case PhaseUploadConfigKubelet:
 		return &corev1.ConfigMap{}
 	case PhaseBootstrapToken:
-		return &corev1.Secret{}
+		return &corev1.ConfigMap{}
 	case PhaseClusterAdminRBAC:
 		return &rbacv1.ClusterRoleBinding{}
 	default:
@@ -73,9 +72,9 @@ func (r *KubeadmPhase) GetPredicateFunc() func(obj client.Object) bool {
 		}
 	case PhaseBootstrapToken:
 		return func(obj client.Object) bool {
-			secret := obj.(*corev1.Secret) //nolint:forcetypeassert
+			cm := obj.(*corev1.ConfigMap) //nolint:forcetypeassert
 
-			return secret.Type == "bootstrap.kubernetes.io/token" && secret.GetNamespace() == metav1.NamespaceSystem
+			return cm.Name == bootstrapapi.ConfigMapClusterInfo && cm.GetNamespace() == metav1.NamespacePublic
 		}
 	case PhaseClusterAdminRBAC:
 		return func(obj client.Object) bool {
@@ -130,7 +129,7 @@ func (r *KubeadmPhase) GetKubeadmFunction(ctx context.Context, tcp *kamajiv1alph
 		return kubeadm.UploadKubeletConfig, nil
 	case PhaseBootstrapToken:
 		return func(client clientset.Interface, config *kubeadm.Configuration) ([]byte, error) {
-			bootstrapTokensEnrichment(config.InitConfiguration.BootstrapTokens)
+			config.InitConfiguration.BootstrapTokens = nil
 
 			return nil, kubeadm.BootstrapToken(client, config)
 		}, nil
@@ -177,26 +176,6 @@ func (r *KubeadmPhase) GetKubeadmFunction(ctx context.Context, tcp *kamajiv1alph
 	}
 }
 
-func bootstrapTokensEnrichment(bootstrapTokens []bootstraptokenv1.BootstrapToken) {
-	var bootstrapToken bootstraptokenv1.BootstrapToken
-	if len(bootstrapTokens) > 0 {
-		bootstrapToken = bootstrapTokens[0]
-	}
-
-	enrichBootstrapToken(&bootstrapToken)
-	bootstrapTokens[0] = bootstrapToken
-}
-
-func enrichBootstrapToken(bootstrapToken *bootstraptokenv1.BootstrapToken) {
-	if bootstrapToken.Token == nil {
-		bootstrapToken.Token = &bootstraptokenv1.BootstrapTokenString{}
-	}
-
-	if bootstrapToken.Token.ID == "" {
-		bootstrapToken.Token.ID = fmt.Sprintf("%s.%s", utils.RandomString(6), utils.RandomString(16))
-	}
-}
-
 func (r *KubeadmPhase) GetClient() client.Client {
 	return r.Client
 }
@@ -240,5 +219,9 @@ func (r *KubeadmPhase) GetStatus(tenantControlPlane *kamajiv1alpha1.TenantContro
 func (r *KubeadmPhase) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
 	logger := log.FromContext(ctx, "resource", r.GetName(), "phase", r.Phase.String())
 
+	if r.Phase == PhaseBootstrapToken {
+		return KubeadmBootstrap(ctx, r, logger, tenantControlPlane)
+	}
+
 	return KubeadmPhaseCreate(ctx, r, logger, tenantControlPlane)
 }

internal/resources/kubeadm_utils.go

@@ -8,7 +8,12 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	clientset "k8s.io/client-go/kubernetes"
+	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
@@ -81,6 +86,89 @@ func GetKubeadmManifestDeps(ctx context.Context, client client.Client, tenantCon
 	return tenantClient, config, nil
 }
 
+func KubeadmBootstrap(ctx context.Context, r KubeadmPhaseResource, logger logr.Logger, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	var checksum string
+
+	tntClient, err := utilities.GetTenantClient(ctx, r.GetClient(), tenantControlPlane)
+	if err != nil {
+		logger.Error(err, "cannot generate tenant client")
+
+		return controllerutil.OperationResultNone, err
+	}
+
+	var clusterInfo corev1.ConfigMap
+	if cmErr := tntClient.Get(ctx, types.NamespacedName{Name: bootstrapapi.ConfigMapClusterInfo, Namespace: metav1.NamespacePublic}, &clusterInfo); cmErr != nil {
+		if !k8serrors.IsNotFound(cmErr) {
+			logger.Error(cmErr, "cannot retrieve cluster-info ConfigMap")
+		}
+	}
+
+	status, err := r.GetStatus(tenantControlPlane)
+	if err != nil {
+		logger.Error(err, "cannot retrieve status")
+
+		return controllerutil.OperationResultNone, err
+	}
+
+	if status != nil {
+		checksum = utilities.CalculateMapChecksum(clusterInfo.Data)
+
+		if checksum == status.GetChecksum() {
+			r.SetKubeadmConfigChecksum(checksum)
+
+			return controllerutil.OperationResultNone, nil
+		}
+	}
+
+	kubeconfig, err := utilities.GetTenantKubeconfig(ctx, r.GetClient(), tenantControlPlane)
+	if err != nil {
+		logger.Error(err, "cannot retrieve kubeconfig configuration")
+
+		return controllerutil.OperationResultNone, err
+	}
+
+	config, err := getStoredKubeadmConfiguration(ctx, r.GetClient(), r.GetTmpDirectory(), tenantControlPlane)
+	if err != nil {
+		logger.Error(err, "cannot retrieve kubeadm configuration")
+
+		return controllerutil.OperationResultNone, err
+	}
+
+	config.Kubeconfig = *kubeconfig
+
+	fun, err := r.GetKubeadmFunction(ctx, tenantControlPlane)
+	if err != nil {
+		logger.Error(err, "cannot retrieve kubeadm function")
+
+		return controllerutil.OperationResultNone, err
+	}
+
+	client, err := utilities.GetTenantClientSet(ctx, r.GetClient(), tenantControlPlane)
+	if err != nil {
+		logger.Error(err, "cannot generate tenant client")
+
+		return controllerutil.OperationResultNone, err
+	}
+
+	if _, err = fun(client, config); err != nil {
+		logger.Error(err, "kubeadm function failed")
+
+		return controllerutil.OperationResultNone, err
+	}
+
+	if status == nil {
+		return controllerutil.OperationResultNone, nil
+	}
+
+	r.SetKubeadmConfigChecksum(checksum)
+
+	if checksum == "" {
+		return controllerutil.OperationResultCreated, nil
+	}
+
+	return controllerutil.OperationResultUpdated, nil
+}
+
 func KubeadmPhaseCreate(ctx context.Context, r KubeadmPhaseResource, logger logr.Logger, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
 	config, err := getStoredKubeadmConfiguration(ctx, r.GetClient(), r.GetTmpDirectory(), tenantControlPlane)
 	if err != nil {