feat: support metrics scrape

docs/tenant_crd.adoc

@@ -1143,6 +1143,13 @@ Directs the MinIO Operator to use prometheus operator. +
 
 Tenant scrape configuration will be added to prometheus managed by the prometheus-operator.
 
+|*`prometheusOperatorScrapeMetricsPaths`* __string array__ 
+|*Optional* +
+
+
+The name of the Prometheus instance to scrape metrics from.
+If PrometheusOperator: true and PrometheusOperatorScrapeMetricsPaths is empty, will add `/minio/v2/metrics/cluster` to the list of paths to scrape as default like before.
+
 |*`serviceAccountName`* __string__ 
 |*Optional* +
 

helm/operator/templates/minio.min.io_tenants.yaml

@@ -3682,6 +3682,10 @@ spec:
                 type: string
               prometheusOperator:
                 type: boolean
+              prometheusOperatorScrapeMetricsPaths:
+                items:
+                  type: string
+                type: array
               readiness:
                 properties:
                   exec:

pkg/apis/minio.min.io/v2/constants.go

@@ -197,9 +197,6 @@ const StatefulSetPrefix = "ss"
 // StatefulSetLegacyPrefix by old operators
 const StatefulSetLegacyPrefix = "zone"
 
-// MinIOPrometheusPathCluster is the path where MinIO tenant exposes cluster Prometheus metrics
-const MinIOPrometheusPathCluster = "/minio/v2/metrics/cluster"
-
 // MinIOPrometheusScrapeInterval defines how frequently to scrape targets.
 const MinIOPrometheusScrapeInterval = 30 * time.Second
 

pkg/apis/minio.min.io/v2/helper.go

@@ -413,6 +413,23 @@ func (t *Tenant) GenBearerToken(accessKey, secretKey string) string {
 	return token
 }
 
+// GetAccessKeyFromBearerToken parses the BearerToken with secretKey to extract accessKey
+func GetAccessKeyFromBearerToken(bearerToken string, secretKey string) (string, error) {
+	claims := &jwt.StandardClaims{}
+	token, err := jwt.ParseWithClaims(bearerToken, claims, func(_ *jwt.Token) (interface{}, error) {
+		return []byte(secretKey), nil
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to parse token: %v", err)
+	}
+
+	if !token.Valid {
+		return "", errors.New("invalid token")
+	}
+
+	return claims.Subject, nil
+}
+
 // MinIOHosts returns the domain names in ellipses format created for current Tenant
 func (t *Tenant) MinIOHosts() (hosts []string) {
 	// Create the ellipses style URL

pkg/apis/minio.min.io/v2/helper_test.go

@@ -9,6 +9,23 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+func TestTenant_GetAccessKeyFromBearerToken(t *testing.T) {
+	mt := Tenant{}
+	mt.EnsureDefaults()
+	accessKey := "testAccessKey"
+	secretKey := "testSecretKey"
+	bearerToken := mt.GenBearerToken(accessKey, secretKey)
+
+	actualAccessKey, err := GetAccessKeyFromBearerToken(bearerToken, secretKey)
+	if err != nil {
+		t.Errorf("GetAccessAndSecretKeyFromBearerToken() returned an error: %v", err)
+	}
+
+	if actualAccessKey != accessKey {
+		t.Errorf("GetAccessAndSecretKeyFromBearerToken() accessKey = %v, want %v", actualAccessKey, accessKey)
+	}
+}
+
 func TestEnsureDefaults(t *testing.T) {
 	mt := Tenant{}
 	mt.EnsureDefaults()

pkg/apis/minio.min.io/v2/types.go

@@ -278,6 +278,13 @@ type TenantSpec struct {
 	PrometheusOperator bool `json:"prometheusOperator,omitempty"`
 	// *Optional* +
 	//
+	// The name of the Prometheus instance to scrape metrics from.
+	// If PrometheusOperator: true and PrometheusOperatorScrapeMetricsPaths is empty, will add `/minio/v2/metrics/cluster` to the list of paths to scrape as default like before.
+	//
+	// +optional
+	PrometheusOperatorScrapeMetricsPaths []string `json:"prometheusOperatorScrapeMetricsPaths,omitempty"`
+	// *Optional* +
+	//
 	// The https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/[Kubernetes Service Account] to use for running MinIO pods created as part of the Tenant. +
 	// +optional
 	ServiceAccountName string `json:"serviceAccountName,omitempty"`

pkg/controller/prometheus.go

@@ -17,6 +17,8 @@ package controller
 import (
 	"context"
 	"errors"
+	"reflect"
+	"strings"
 
 	promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	promv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1alpha1"
@@ -141,7 +143,7 @@ func (c *Controller) checkAndCreatePrometheusAddlConfig(ctx context.Context, ten
 
 	// If the secret is not found, create the secret
 	if k8serrors.IsNotFound(err) {
-		klog.Infof("Adding MinIO tenant Prometheus scrape config")
+		klog.Infof("Adding MinIO tenant %s/%s Prometheus scrape config", ns, tenant.Name)
 		scrapeCfgYaml, err := yaml.Marshal(&promCfg.ScrapeConfigs)
 		if err != nil {
 			return err
@@ -161,23 +163,46 @@ func (c *Controller) checkAndCreatePrometheusAddlConfig(ctx context.Context, ten
 			return err
 		}
 	} else {
-		var scrapeConfigs []configmaps.ScrapeConfig
+		var scrapeConfigs, expectedScrapeConfigs []configmaps.ScrapeConfig
 		err := yaml.Unmarshal(secret.Data[miniov2.PrometheusAddlScrapeConfigKey], &scrapeConfigs)
 		if err != nil {
 			return err
 		}
-		// Check if the scrape config is already present
-		hasScrapeConfig := false
+		// get other scrape configs
 		for _, sc := range scrapeConfigs {
-			if sc.JobName == tenant.PrometheusOperatorAddlConfigJobName() {
-				hasScrapeConfig = true
-				break
+			if !strings.HasPrefix(sc.JobName, tenant.PrometheusOperatorAddlConfigJobName()) {
+				expectedScrapeConfigs = append(expectedScrapeConfigs, sc)
 			}
 		}
-		if !hasScrapeConfig {
-			klog.Infof("Adding MinIO tenant Prometheus scrape config")
-			scrapeConfigs = append(scrapeConfigs, promCfg.ScrapeConfigs...)
-			scrapeCfgYaml, err := yaml.Marshal(scrapeConfigs)
+		ignoreScrapeConfigsIndex := len(expectedScrapeConfigs)
+		expectedScrapeConfigs = append(expectedScrapeConfigs, promCfg.ScrapeConfigs...)
+		updateScrapeConfig := false
+		if len(scrapeConfigs) != len(expectedScrapeConfigs) {
+			updateScrapeConfig = true
+		} else {
+			for i := range scrapeConfigs {
+				// can't compare that is generated by operator
+				if i < ignoreScrapeConfigsIndex {
+					continue
+				}
+				if scrapeConfigs[i].JobName != expectedScrapeConfigs[i].JobName ||
+					scrapeConfigs[i].MetricsPath != expectedScrapeConfigs[i].MetricsPath ||
+					scrapeConfigs[i].Scheme != expectedScrapeConfigs[i].Scheme ||
+					!reflect.DeepEqual(scrapeConfigs[i].TLSConfig, expectedScrapeConfigs[i].TLSConfig) ||
+					!reflect.DeepEqual(scrapeConfigs[i].StaticConfigs, expectedScrapeConfigs[i].StaticConfigs) {
+					updateScrapeConfig = true
+					break
+				}
+				accKey, _ := miniov2.GetAccessKeyFromBearerToken(scrapeConfigs[i].BearerToken, secretKey)
+				if accKey != accessKey {
+					updateScrapeConfig = true
+					break
+				}
+			}
+		}
+		if updateScrapeConfig {
+			klog.Infof("Updating MinIO tenant %s/%s Prometheus scrape config", ns, tenant.Name)
+			scrapeCfgYaml, err := yaml.Marshal(expectedScrapeConfigs)
 			if err != nil {
 				return err
 			}
@@ -224,27 +249,20 @@ func (c *Controller) deletePrometheusAddlConfig(ctx context.Context, tenant *min
 		return err
 	}
 
-	var scrapeConfigs []configmaps.ScrapeConfig
+	var scrapeConfigs, exceptedScrapeConfigs []configmaps.ScrapeConfig
 	err = yaml.Unmarshal(secret.Data[miniov2.PrometheusAddlScrapeConfigKey], &scrapeConfigs)
 	if err != nil {
 		return err
 	}
-	// Check if the scrape config is present
-	hasScrapeConfig := false
-	scIndex := -1
-	for i, sc := range scrapeConfigs {
-		if sc.JobName == tenant.PrometheusOperatorAddlConfigJobName() {
-			hasScrapeConfig = true
-			scIndex = i
-			break
+	for _, sc := range scrapeConfigs {
+		if !strings.HasPrefix(sc.JobName, tenant.PrometheusOperatorAddlConfigJobName()) {
+			exceptedScrapeConfigs = append(exceptedScrapeConfigs, sc)
 		}
 	}
-	if hasScrapeConfig {
+	if !reflect.DeepEqual(scrapeConfigs, exceptedScrapeConfigs) {
 		klog.Infof("Deleting MinIO tenant Prometheus scrape config")
-		// Delete the config
-		newScrapeConfigs := append(scrapeConfigs[:scIndex], scrapeConfigs[scIndex+1:]...)
 		// Update the secret
-		scrapeCfgYaml, err := yaml.Marshal(newScrapeConfigs)
+		scrapeCfgYaml, err := yaml.Marshal(exceptedScrapeConfigs)
 		if err != nil {
 			return err
 		}