diff --git a/.goreleaser.yml b/.goreleaser.yml index ceec1a7..84c2005 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -8,7 +8,7 @@ before: gomod: proxy: false builds: - - main: . + - main: cmd/ binary: "{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}" env: - CGO_ENABLED=0 @@ -33,11 +33,15 @@ release: footer: | **Full Changelog**: https://github.com/projectcapsule/{{ .ProjectName }}/compare/{{ .PreviousTag }}...{{ .Tag }} + [!TIP] + [Read this documentation](https://github.com/projectcapsule/{{ .ProjectName }}/blob/{{ .Tag }}/SECURITY.md) to see how you can verify an artifacts we are releasing. + **Docker Images** - `ghcr.io/projectcapsule/{{ .ProjectName }}:{{ .Version }}` - `ghcr.io/projectcapsule/{{ .ProjectName }}:latest` **Helm Chart** + [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/cortex-proxy)](https://artifacthub.io/packages/search?repo=cortex-proxy) **Kubernetes compatibility** @@ -52,8 +56,6 @@ release: Thanks to all the contributors! 🚀 🦄 - extra_files: - - glob: ./capsule-seccomp.json checksum: name_template: 'checksums.txt' changelog: diff --git a/README.md b/README.md index 37778a4..4d025fc 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ +# Capsule ❤️ Cortex + [!IMPORTANT] This project is a permanent hard-fork of the [origin project](https://github.com/blind-oracle/cortex-tenant). -# Capsule ❤️ Cortex - ![Capsule Cortex](docs/images/logo.png)

diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..de38baa --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,52 @@ +# Release Artifacts + +[See all the available artifacts](https://github.com/orgs/projectcapsule/packages?repo_name=cortex-proxy) + +## Verifing + +To verify artifacts you need to have [cosign installed](https://github.com/sigstore/cosign#installation). This guide assumes you are using v2.x of cosign. All of the signatures are created using [keyless signing](https://docs.sigstore.dev/verifying/verify/#keyless-verification-using-openid-connect). +To verify the signature of the docker image, run the following command. Replace `` with an [available release tag](https://github.com/projectcapsule/cortex-proxy/pkgs/container/cortex-proxy). The value `release_tag` is a release but without the prefix `v` (eg. `0.1.0-alpha.3`). + + VERSION= cosign verify ghcr.io/projectcapsule/cortex-proxy:${VERSION} \ + --certificate-identity-regexp="https://github.com/projectcapsule/cortex-proxy/.github/workflows/docker-publish.yml@refs/tags/*" \ + --certificate-oidc-issuer="https://token.actions.githubusercontent.com" | jq + +To verify the signature of the helm image, run the following command. Replace `` with an [available release tag](https://github.com/projectcapsule/cortex-proxy/pkgs/container/charts%2Fcortex-proxy). The value `release_tag` is a release but without the prefix `v` (eg. `0.1.0-alpha.3`) + + VERSION= cosign verify ghcr.io/projectcapsule/charts/cortex-proxy:${VERSION} \ + --certificate-identity-regexp="https://github.com/projectcapsule/cortex-proxy/.github/workflows/helm-publish.yml@refs/tags/*" \ + --certificate-oidc-issuer="https://token.actions.githubusercontent.com" | jq + +## Verifying Provenance + +We create and attest the provenance of our builds using the [SLSA standard](https://slsa.dev/spec/v0.2/provenance) and meets the [SLSA Level 3](https://slsa.dev/spec/v0.1/levels) specification. The attested provenance may be verified using the cosign tool. + +Verify the provenance of the docker image. Replace `` with an [available release tag](https://github.com/projectcapsule/cortex-proxy/pkgs/container/cortex-proxy). The value `release_tag` is a release but without the prefix `v` (eg. `0.1.0-alpha.3`) + +```bash +cosign verify-attestation --type slsaprovenance \ + --certificate-identity-regexp="https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@refs/tags/*" \ + --certificate-oidc-issuer="https://token.actions.githubusercontent.com" \ + ghcr.io/projectcapsule/cortex-proxy: | jq .payload -r | base64 --decode | jq +``` + +Verify the provenance of the helm image. Replace `` with an [available release tag](https://github.com/projectcapsule/cortex-proxy/pkgs/container/charts%cortex-proxy). The value `release_tag` is a release but without the prefix `v` (eg. `0.1.0-alpha.3`) + +```bash +VERSION= cosign verify-attestation --type slsaprovenance \ + --certificate-identity-regexp="https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@refs/tags/*" \ + --certificate-oidc-issuer="https://token.actions.githubusercontent.com" \ + "ghcr.io/projectcapsule/charts/cortex-proxy:${VERSION}" | jq .payload -r | base64 --decode | jq +``` + +## Software Bill of Materials (SBOM) + +An SBOM (Software Bill of Materials) in CycloneDX JSON format is published for each release, including pre-releases. + +To inspect the SBOM of the docker image, run the following command. Replace `` with an [available release tag](https://github.com/projectcapsule/cortex-proxy/pkgs/container/cortex-proxy): + + COSIGN_REPOSITORY=ghcr.io/projectcapsule/cortex-proxy cosign download sbom ghcr.io/projectcapsule/cortex-proxy: + +To inspect the SBOM of the helm image, run the following command. Replace `` with an [available release tag](https://github.com/projectcapsule/cortex-proxy/pkgs/container/charts%2Fcortex-proxy): + + COSIGN_REPOSITORY=ghcr.io/projectcapsule/charts/cortex-proxy cosign download sbom ghcr.io/projectcapsule/charts/cortex-proxy: diff --git a/charts/cortex-proxy/README.md b/charts/cortex-proxy/README.md index fc78d89..e0d9298 100644 --- a/charts/cortex-proxy/README.md +++ b/charts/cortex-proxy/README.md @@ -40,7 +40,7 @@ The following Values are available for this chart. | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | Set the image pull policy. | | image.registry | string | `"ghcr.io"` | Set the image registry | -| image.repository | string | `"projectcapsule/cortex-tenant"` | Set the image repository | +| image.repository | string | `"projectcapsule/cortex-proxy"` | Set the image repository | | image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | | imagePullSecrets | list | `[]` | Configuration for `imagePullSecrets` so that you can use a private images registry. | | livenessProbe | object | `{"httpGet":{"path":"/healthz","port":10080}}` | Configure the liveness probe using Deployment probe spec | @@ -107,7 +107,7 @@ The following Values are available for this chart. | monitoring.enabled | bool | `false` | Enable Monitoring of the Operator | | monitoring.rules.annotations | object | `{}` | Assign additional Annotations | | monitoring.rules.enabled | bool | `true` | Enable deployment of PrometheusRules | -| monitoring.rules.groups | list | `[{"name":"TranslatorAlerts","rules":[{"alert":"TranslatorNotReady","annotations":{"description":"The Translator {{ $labels.name }} has been in a NotReady state for over 5 minutes.","summary":"Translator {{ $labels.name }} is not ready"},"expr":"cca_translator_condition{status=\"NotReady\"} == 1","for":"5m","labels":{"severity":"warning"}}]}]` | Prometheus Groups for the rule | +| monitoring.rules.groups | list | `[]` | Prometheus Groups for the rule | | monitoring.rules.labels | object | `{}` | Assign additional labels | | monitoring.rules.namespace | string | `""` | Install the rules into a different Namespace, as the monitoring stack one (default: the release one) | | monitoring.serviceMonitor.annotations | object | `{}` | Assign additional Annotations | diff --git a/charts/cortex-proxy/values.schema.json b/charts/cortex-proxy/values.schema.json index c564564..cb84558 100644 --- a/charts/cortex-proxy/values.schema.json +++ b/charts/cortex-proxy/values.schema.json @@ -183,50 +183,6 @@ "type": "boolean" }, "groups": { - "items": { - "properties": { - "name": { - "type": "string" - }, - "rules": { - "items": { - "properties": { - "alert": { - "type": "string" - }, - "annotations": { - "properties": { - "description": { - "type": "string" - }, - "summary": { - "type": "string" - } - }, - "type": "object" - }, - "expr": { - "type": "string" - }, - "for": { - "type": "string" - }, - "labels": { - "properties": { - "severity": { - "type": "string" - } - }, - "type": "object" - } - }, - "type": "object" - }, - "type": "array" - } - }, - "type": "object" - }, "type": "array" }, "labels": { diff --git a/charts/cortex-proxy/values.yaml b/charts/cortex-proxy/values.yaml index 16add5d..5f49d38 100644 --- a/charts/cortex-proxy/values.yaml +++ b/charts/cortex-proxy/values.yaml @@ -87,7 +87,7 @@ image: # -- Set the image registry registry: ghcr.io # -- Set the image repository - repository: projectcapsule/cortex-tenant + repository: projectcapsule/cortex-proxy # -- Set the image pull policy. pullPolicy: IfNotPresent # -- Overrides the image tag whose default is the chart appVersion. @@ -219,18 +219,23 @@ monitoring: # -- Assign additional Annotations annotations: {} # -- Prometheus Groups for the rule - groups: - - name: TranslatorAlerts - rules: - - alert: TranslatorNotReady - expr: cca_translator_condition{status="NotReady"} == 1 - for: 5m - labels: - severity: warning - annotations: - summary: "Translator {{ $labels.name }} is not ready" - description: "The Translator {{ $labels.name }} has been in a NotReady state for over 5 minutes." - + groups: [] + # - alert: CortexProxyTooMany500s + # expr: 100 * ( sum( timeseries_request_duration_seconds{code=~"5.+"} ) / sum(timeseries_request_duration_seconds) ) > 5 + # for: 5m + # labels: + # severity: warning + # annotations: + # description: Too many 5XXs + # summary: More than 5% of all requests returned 5XX, this requires your attention + # - alert: CortexProxyTooMany400s + # expr: 100 * ( sum( timeseries_request_duration_seconds{status=~"4.+"} ) / sum(timeseries_request_duration_seconds) ) > 5 + # for: 5m + # labels: + # severity: warning + # annotations: + # description: Too many 4XXs + # summary: More than 5% of all requests returned 4XX, this requires your attention # ServiceMonitor serviceMonitor: # -- Enable ServiceMonitor diff --git a/docs/README.md b/docs/README.md index 73a82d3..d509c89 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,7 +2,6 @@ See the following topics for more information on how to use this addon: -- [Installation](installation.md) - [Configuration](configuration.md) - [Monitoring](monitoring.md) - [Development](development.md) diff --git a/docs/configuration.md b/docs/configuration.md index ba94c4b..1cf6e9f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -6,107 +6,77 @@ If both are used then the env vars have precedence (i.e. they override values fr See below for config file format and corresponding env vars. ```yaml -# Where to listen for incoming write requests from Prometheus -# env: CT_LISTEN -listen: 0.0.0.0:8080 - -# Profiling API, remove to disable -# env: CT_LISTEN_PPROF -listen_pprof: 0.0.0.0:7008 - # Where to send the modified requests (Cortex/Mimir) backend: url: http://127.0.0.1:9091/receive # Authentication (optional) auth: - # Egress HTTP basic auth -> add `Authentication` header to outgoing requests - egress: - # env: CT_AUTH_EGRESS_USERNAME - username: foo - # env: CT_AUTH_EGRESS_PASSWORD - password: bar + username: foo + password: bar # Whether to enable querying for IPv6 records -# env: CT_ENABLE_IPV6 -enable_ipv6: false +ipv6: false # This parameter sets the limit for the count of outgoing concurrent connections to Cortex / Mimir. # By default it's 64 and if all of these connections are busy you will get errors when pushing from Prometheus. # If your `target` is a DNS name that resolves to several IPs then this will be a per-IP limit. -# env: CT_MAX_CONNS_PER_HOST -max_conns_per_host: 0 +maxConnectionsPerHost: 0 # HTTP request timeout -# env: CT_TIMEOUT timeout: 10s # Timeout to wait on shutdown to allow load balancers detect that we're going away. # During this period after the shutdown command the /alive endpoint will reply with HTTP 503. # Set to 0s to disable. -# env: CT_TIMEOUT_SHUTDOWN -timeout_shutdown: 10s +timeoutShutdown: 10s # Max number of parallel incoming HTTP requests to handle -# env: CT_CONCURRENCY concurrency: 10 # Whether to forward metrics metadata from Prometheus to Cortex/Mimir # Since metadata requests have no timeseries in them - we cannot divide them into tenants # So the metadata requests will be sent to the default tenant only, if one is not defined - they will be dropped -# env: CT_METADATA metadata: false -# If true response codes from metrics backend will be logged to stdout. This setting can be used to suppress errors -# which can be quite verbose like 400 code - out-of-order samples or 429 on hitting ingestion limits -# Also, those are already reported by other services like Cortex/Mimir distributors and ingesters -# env: CT_LOG_RESPONSE_ERRORS -log_response_errors: true - # Maximum duration to keep outgoing connections alive (to Cortex/Mimir) # Useful for resetting L4 load-balancer state # Use 0 to keep them indefinitely -# env: CT_MAX_CONN_DURATION -max_connection_duration: 0s - -# Address where metrics are available -# env: CT_LISTEN_METRICS_ADDRESS -listen_metrics_address: 0.0.0.0:9090 +maxConnectionDuration: 0s -# If true, then a label with the tenant’s name will be added to the metrics -# env: CT_METRICS_INCLUDE_TENANT -metrics_include_tenant: true +# Select only a subset of tenant to consider for collection +# namespaces which can not be assigned to any tenant will get the +# default value +selector: + matchLabels: + env: "prod" tenant: # List of labels examined for tenant information. - # env: CT_TENANT_LABEL_LIST - label_list: - - tenant - - other_tenant + labels: + - namespace + - target_namespace # Whether to remove the tenant label from the request - # env: CT_TENANT_LABEL_REMOVE - label_remove: true + labelRemove: true # To which header to add the tenant ID - # env: CT_TENANT_HEADER header: X-Scope-OrgID # Which tenant ID to use if the label is missing in any of the timeseries # If this is not set or empty then the write request with missing tenant label # will be rejected with HTTP code 400 - # env: CT_TENANT_DEFAULT + # Namespaces which can not be assigned to any tenant will get the + # default value default: foobar # Enable if you want all metrics from Prometheus to be accepted with a 204 HTTP code # regardless of the response from upstream. This can lose metrics if Cortex/Mimir is # throwing rejections. - # env: CT_TENANT_ACCEPT_ALL - accept_all: false + acceptAll: false # Optional prefix to be added to a tenant header before sending it to Cortex/Mimir. # Make sure to use only allowed characters: # https://grafana.com/docs/mimir/latest/configure/about-tenant-ids/ - # env: CT_TENANT_PREFIX prefix: foobar- # If true will use the tenant ID of the inbound request as the prefix of the new tenant id. @@ -115,6 +85,5 @@ tenant: # Prometheus forwards metrics with `X-Scope-OrgID: Prom-A` set in the inbound request. # This would result in the tenant prefix being set to `Prom-A-`. # https://grafana.com/docs/mimir/latest/configure/about-tenant-ids/ - # env: CT_TENANT_PREFIX_PREFER_SOURCE - prefix_prefer_source: false + prefixPreferSource: false ``` diff --git a/docs/overview.md b/docs/overview.md deleted file mode 100644 index 3961002..0000000 --- a/docs/overview.md +++ /dev/null @@ -1,19 +0,0 @@ -# Architecture - -![Architecture](architecture.svg) - -## Overview - -Cortex/Mimir tenants (separate namespaces where metrics are stored to and queried from) are identified by `X-Scope-OrgID` HTTP header on both writes and queries. - -~~Problem is that Prometheus can't be configured to send this header~~ Actually in some recent version (year 2021 onwards) this functionality was added, but the tenant is the same for all jobs. This makes it impossible to use a single Prometheus (or an HA pair) to write to multiple tenants. - -This software solves the problem using the following logic: - -- Receive Prometheus remote write -- Search each timeseries for a specific label name and extract a tenant ID from its value. - If the label wasn't found then it can fall back to a configurable default ID. - If none is configured then the write request will be rejected with HTTP code 400 -- Optionally removes this label from the timeseries -- Groups timeseries by tenant -- Issues a number of parallel per-tenant HTTP requests to Cortex/Mimir with the relevant tenant HTTP header (`X-Scope-OrgID` by default)