feat(txt-registry): deprecate legacy txt-format

.editorconfig

@@ -20,3 +20,7 @@ insert_final_newline = true
 [{Makefile,go.mod,go.sum,*.go}]
 indent_style = tab
 indent_size = 4
+
+[*.py]
+indent_style = space
+indent_size = 4

.pre-commit-config.yaml

@@ -23,5 +23,6 @@ repos:
     rev: v0.44.0
     hooks:
     - id: markdownlint
+      args: ["--fix"]
 
 minimum_pre_commit_version: !!str 3.2

docs/flags.md

@@ -161,7 +161,7 @@
 | `--txt-wildcard-replacement=""` | When using the TXT registry, a custom string that's used instead of an asterisk for TXT records corresponding to wildcard DNS records (optional) |
 | `--[no-]txt-encrypt-enabled` | When using the TXT registry, set if TXT records should be encrypted before stored (default: disabled) |
 | `--txt-encrypt-aes-key=""` | When using the TXT registry, set TXT record decryption and encryption 32 byte aes key (required when --txt-encrypt=true) |
-| `--[no-]txt-new-format-only` | When using the TXT registry, only use new format records which include record type information (e.g., prefix: 'a-'). Reduces number of TXT records (default: disabled) |
+| `--[no-]txt-new-format-only` | Was used during TXT format migration. Enabling or disabling it will not take any effect. (deprecated) |
 | `--dynamodb-region=""` | When using the DynamoDB registry, the AWS region of the DynamoDB table (optional) |
 | `--dynamodb-table="external-dns"` | When using the DynamoDB registry, the name of the DynamoDB table (default: "external-dns") |
 | `--txt-cache-interval=0s` | The interval between cache synchronizations in duration format (default: disabled) |

docs/registry/txt.md

@@ -3,8 +3,49 @@
 The TXT registry is the default registry.
 It stores DNS record metadata in TXT records, using the same provider.
 
+If you plan to manage apex domains with external-dns whilst using a txt registry, you should ensure when using --txt-prefix that you specify the record type substitution and that it ends in a period ., to ensure the record is created under the same domain as the apex record being managed, i.e. --txt-prefix=someprefix-%{record_type}.
+
 ## Record Format Options
 
+### For version `v0.16.2+`
+
+The TXT registry supports single format for storing DNS record metadata:
+
+- Creates a TXT record with record type information (e.g., 'a-' prefix for A records)
+
+The TXT registry would try to guarantee a consistency in between providers and sources, if provider supports the behaviour
+
+If you are dealing with APEX domains, example `example.com` and TXT records are failing to be created for managed record types specified by `--managed-record-types`, consider following options:
+
+1. TXT record with prefix based on requirements. Example `--txt-prefix="%{record_type}-abc-"` or `--txt-prefix="%{record_type}.abc-"`
+2. TXT record with suffix based on requirements. Example `--txt-suffix="-abc-%{record_type}"` or `--txt-suffix="-abc.%{record_type}."`
+
+Example when configured `--txt-prefix="%{record_type}-abc-"` for apex domain `example.com` the expected result is
+
+|             Name              |   TYPE   |
+|:-----------------------------:|:--------:|
+|   `a-abc-nginx-v2.ex.com.`    |  `TXT`   |
+| `nginx-v2.ex.com.`  | `CNAME`  |
+
+And when configured `--txt-suffix="-abc.%{record_type}"` for apex domain `example.com` the expected result is
+
+|             Name              |  TYPE   |
+|:-----------------------------:|:-------:|
+|   `nginx-v2-abc.a.ex.com.`    |  `TXT`  |
+| `nginx-v3.ex.com..` | `CNAME` |
+
+### Manually Cleanup Legacy TXT Records
+
+> While deleting registry TXT records won't cause downtime, a well-thought-out migration and cleanup plan is crucial.
+
+Occasionally, it may be necessary to remove outdated TXT records from your registry.
+
+An example script for AWS can be found in [scripts/cleanup-legacy-txt-records.py](../../scripts/cleanup-legacy-txt-records.py) with instructions on how to run it.
+The script performs targeted deletion of TXT records that include `ResourceRecords` matching the `heritage=external-dns,external-dns/owner=default` or similar pattern.
+In the event of unintended deletion of all TXT records managed by `external-dns`, `external-dns` will initiate a full DNS record regeneration, along with`TXT` and `non-TXT` records. Just be aware, this operation's duration is directly proportional to the DNS estate size."
+
+### For version `v0.16.0 & v0.16.1`
+
 The TXT registry supports two formats for storing DNS record metadata:
 
 - Legacy format: Creates a TXT record without record type information
@@ -31,14 +72,14 @@ The `--txt-new-format-only` flag should be used in addition to your existing ext
 
 ### Migration to New Format Only
 
+> Note: `external-dns` will not automatically remove legacy format records when switching to new-format-only mode. You'll need to clean up the old records manually if desired.
+
 When transitioning from dual-format to new-format-only records:
 
 - Ensure all your `external-dns` instances support the new format
 - Enable the `--txt-new-format-only` flag on your external-dns instances
 Manually clean up any existing legacy format TXT records from your DNS provider
 
-Note: `external-dns` will not automatically remove legacy format records when switching to new-format-only mode. You'll need to clean up the old records manually if desired.
-
 ## Prefixes and Suffixes
 
 In order to avoid having the registry TXT records collide with

pkg/apis/externaldns/types.go

@@ -615,7 +615,7 @@ func App(cfg *Config) *kingpin.Application {
 	app.Flag("txt-wildcard-replacement", "When using the TXT registry, a custom string that's used instead of an asterisk for TXT records corresponding to wildcard DNS records (optional)").Default(defaultConfig.TXTWildcardReplacement).StringVar(&cfg.TXTWildcardReplacement)
 	app.Flag("txt-encrypt-enabled", "When using the TXT registry, set if TXT records should be encrypted before stored (default: disabled)").BoolVar(&cfg.TXTEncryptEnabled)
 	app.Flag("txt-encrypt-aes-key", "When using the TXT registry, set TXT record decryption and encryption 32 byte aes key (required when --txt-encrypt=true)").Default(defaultConfig.TXTEncryptAESKey).StringVar(&cfg.TXTEncryptAESKey)
-	app.Flag("txt-new-format-only", "When using the TXT registry, only use new format records which include record type information (e.g., prefix: 'a-'). Reduces number of TXT records (default: disabled)").BoolVar(&cfg.TXTNewFormatOnly)
+	app.Flag("txt-new-format-only", "Was used during TXT format migration. Enabling or disabling it will not take any effect. (deprecated)").BoolVar(&cfg.TXTNewFormatOnly)
 	app.Flag("dynamodb-region", "When using the DynamoDB registry, the AWS region of the DynamoDB table (optional)").Default(cfg.AWSDynamoDBRegion).StringVar(&cfg.AWSDynamoDBRegion)
 	app.Flag("dynamodb-table", "When using the DynamoDB registry, the name of the DynamoDB table (default: \"external-dns\")").Default(defaultConfig.AWSDynamoDBTable).StringVar(&cfg.AWSDynamoDBTable)
 

registry/txt.go

@@ -58,8 +58,6 @@ type TXTRegistry struct {
 	// encrypt text records
 	txtEncryptEnabled bool
 	txtEncryptAESKey  []byte
-
-	newFormatOnly bool
 }
 
 // NewTXTRegistry returns a new TXTRegistry object. When newFormatOnly is true, it will only
@@ -74,6 +72,10 @@ func NewTXTRegistry(provider provider.Provider, txtPrefix, txtSuffix, ownerID st
 		return nil, errors.New("owner id cannot be empty")
 	}
 
+	if newFormatOnly {
+		log.Warn("--txt-new-format-only is left for backward compatibility, it will be removed in future releases")
+	}
+
 	if len(txtEncryptAESKey) == 0 {
 		txtEncryptAESKey = nil
 	} else if len(txtEncryptAESKey) != 32 {
@@ -103,7 +105,6 @@ func NewTXTRegistry(provider provider.Provider, txtPrefix, txtSuffix, ownerID st
 		excludeRecordTypes:  excludeRecordTypes,
 		txtEncryptEnabled:   txtEncryptEnabled,
 		txtEncryptAESKey:    txtEncryptAESKey,
-		newFormatOnly:       newFormatOnly,
 	}, nil
 }
 
@@ -236,18 +237,6 @@ func (im *TXTRegistry) Records(ctx context.Context) ([]*endpoint.Endpoint, error
 func (im *TXTRegistry) generateTXTRecord(r *endpoint.Endpoint) []*endpoint.Endpoint {
 	endpoints := make([]*endpoint.Endpoint, 0)
 
-	// Create legacy format record by default unless newFormatOnly is true
-	if !im.newFormatOnly && !im.txtEncryptEnabled && !im.mapper.recordTypeInAffix() && r.RecordType != endpoint.RecordTypeAAAA {
-		// old TXT record format
-		txt := endpoint.NewEndpoint(im.mapper.toTXTName(r.DNSName), endpoint.RecordTypeTXT, r.Labels.Serialize(true, im.txtEncryptEnabled, im.txtEncryptAESKey))
-		if txt != nil {
-			txt.WithSetIdentifier(r.SetIdentifier)
-			txt.Labels[endpoint.OwnedRecordLabelKey] = r.DNSName
-			txt.ProviderSpecific = r.ProviderSpecific
-			endpoints = append(endpoints, txt)
-		}
-	}
-
 	// Always create new format record
 	recordType := r.RecordType
 	// AWS Alias records are encoded as type "cname"