Switch derived source from field attributes to segment attribute (#2606)

Switches derived source from using field attributes to using segment
attributes to track removed vector fields.The main motivation for this
change is that: (1) it allows us to simplify mapper logic. We no longer
need to add another field attribute to an already bloated set of field
attributes. (2) At the time of setting field attributes, we do not have
access to info about nesting scope for the field. If we switch to
segment info, we can get the nested scope and avoid having to perform
complex lucene low level operations to get this info ourselves.

Along with the change, I moved certain components to the bwc package to
preserve backwards compatibility. Additionally, I added several bwc
tests.

Signed-off-by: John Mazanec <[email protected]>

Author: jmazanec15

CHANGELOG.md

@@ -15,6 +15,7 @@ Add filter function to KNNQueryBuilder with unit tests and integration tests [#2
 ### Documentation
 ### Maintenance
 ### Refactoring
+* Switch derived source from field attributes to segment attribute [#2606](https://github.com/opensearch-project/k-NN/pull/2606)
 
 ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.19...2.x)
 ### Features

qa/restart-upgrade/build.gradle

@@ -126,6 +126,31 @@ testClusters {
                 excludeTestsMatching "org.opensearch.knn.bwc.ScriptScoringIT.testNonKNNIndex_withMethodParams_withLuceneEngine"
             }
         }
+        
+        if (knn_bwc_version.startsWith("1.") ||
+                knn_bwc_version.startsWith("2.0.") ||
+                knn_bwc_version.startsWith("2.1.") ||
+                knn_bwc_version.startsWith("2.2.") ||
+                knn_bwc_version.startsWith("2.3.") ||
+                knn_bwc_version.startsWith("2.4")  ||
+                knn_bwc_version.startsWith("2.5.") ||
+                knn_bwc_version.startsWith("2.6.") ||
+                knn_bwc_version.startsWith("2.7.") ||
+                knn_bwc_version.startsWith("2.8.") ||
+                knn_bwc_version.startsWith("2.9.") ||
+                knn_bwc_version.startsWith("2.10.") ||
+                knn_bwc_version.startsWith("2.11.") ||
+                knn_bwc_version.startsWith("2.12.") ||
+                knn_bwc_version.startsWith("2.13.") ||
+                knn_bwc_version.startsWith("2.14.") ||
+                knn_bwc_version.startsWith("2.15.") ||
+                knn_bwc_version.startsWith("2.16.") ||
+                knn_bwc_version.startsWith("2.17.") ||
+                knn_bwc_version.startsWith("2.18.")) {
+            filter {
+                excludeTestsMatching "org.opensearch.knn.bwc.DerivedSourceBWCRestartIT"
+            }
+        }
 
         nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
         nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
@@ -230,6 +255,31 @@ testClusters {
             }
         }
 
+        if (knn_bwc_version.startsWith("1.") ||
+                knn_bwc_version.startsWith("2.0.") ||
+                knn_bwc_version.startsWith("2.1.") ||
+                knn_bwc_version.startsWith("2.2.") ||
+                knn_bwc_version.startsWith("2.3.") ||
+                knn_bwc_version.startsWith("2.4")  ||
+                knn_bwc_version.startsWith("2.5.") ||
+                knn_bwc_version.startsWith("2.6.") ||
+                knn_bwc_version.startsWith("2.7.") ||
+                knn_bwc_version.startsWith("2.8.") ||
+                knn_bwc_version.startsWith("2.9.") ||
+                knn_bwc_version.startsWith("2.10.") ||
+                knn_bwc_version.startsWith("2.11.") ||
+                knn_bwc_version.startsWith("2.12.") ||
+                knn_bwc_version.startsWith("2.13.") ||
+                knn_bwc_version.startsWith("2.14.") ||
+                knn_bwc_version.startsWith("2.15.") ||
+                knn_bwc_version.startsWith("2.16.") ||
+                knn_bwc_version.startsWith("2.17.") ||
+                knn_bwc_version.startsWith("2.18.")) {
+            filter {
+                excludeTestsMatching "org.opensearch.knn.bwc.DerivedSourceBWCRestartIT"
+            }
+        }
+
         nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
         nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
         systemProperty 'tests.security.manager', 'false'