diff --git a/CHANGELOG.md b/CHANGELOG.md index ec7742b8563bc..58fa06f9af83c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Added new Setting property UnmodifiableOnRestore to prevent updating settings on restore snapshot ([#16957](https://github.com/opensearch-project/OpenSearch/pull/16957)) - Introduce Template query ([#16818](https://github.com/opensearch-project/OpenSearch/pull/16818)) - Propagate the sourceIncludes and excludes fields from fetchSourceContext to FieldsVisitor. ([#17080](https://github.com/opensearch-project/OpenSearch/pull/17080)) +- Arrow Flight server bootstrap logic ([#16962](https://github.com/opensearch-project/OpenSearch/pull/16962)) ### Dependencies - Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504)) diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index a8c96f33ce51d..e63b65dc18947 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -89,3 +89,4 @@ ${error.file} # See please https://bugs.openjdk.org/browse/JDK-8341127 (openjdk/jdk#21283) 23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.setAsTypeCache 23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.asTypeUncached +24:-add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c2afc85f79078..f20d08e04fdac 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -82,7 +82,7 @@ opentelemetry = "1.46.0" opentelemetrysemconv = "1.29.0-alpha" # arrow dependencies -arrow = "17.0.0" +arrow = "18.1.0" flatbuffers = "2.0.0" [libraries] diff --git a/libs/arrow-memory-shaded/build.gradle b/libs/arrow-memory-shaded/build.gradle new file mode 100644 index 0000000000000..df77b0ff1adf2 --- /dev/null +++ b/libs/arrow-memory-shaded/build.gradle @@ -0,0 +1,82 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +plugins { + id("com.gradleup.shadow") +} + +description = '''Shading dependencies under opensearch namespace for arrow-memory-netty, netty-buffer, netty-common and other common transitive dependencies like +sl4j, jackson, flatbuffers and common-codec to prevent pollution of server classpath''' + +testingConventions.enabled = false +generatePomFileForShadowPublication.enabled = false + +dependencies { + runtimeOnly "org.apache.arrow:arrow-memory-netty:${versions.arrow}" + runtimeOnly "org.apache.arrow:arrow-memory-core:${versions.arrow}" + runtimeOnly "org.apache.arrow:arrow-memory-netty-buffer-patch:${versions.arrow}" + runtimeOnly "io.netty:netty-buffer:${versions.netty}" + runtimeOnly "io.netty:netty-common:${versions.netty}" + runtimeOnly "org.apache.arrow:arrow-vector:${versions.arrow}" + runtimeOnly "org.apache.arrow:arrow-format:${versions.arrow}" + + runtimeOnly 'org.checkerframework:checker-qual:3.44.0' + runtimeOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" + runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}" + runtimeOnly "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + runtimeOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" + runtimeOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + runtimeOnly "commons-codec:commons-codec:${versions.commonscodec}" +} + +tasks.named("shadowJar").configure { + dependencies { + include(dependency('org.apache.arrow:')) + include(dependency('io.netty:')) + include(dependency('org.checkerframework:')) + include(dependency('org.slf4j:')) + include(dependency('com.google.flatbuffers:')) + include(dependency('com.fasterxml.jackson.core:')) + include(dependency('commons-codec:')) + } + manifest { + attributes('Multi-Release': 'true') + } + exclude 'META-INF/maven/**' + relocate 'io.netty', 'org.opensearch.shaded.io.netty' + relocate 'org.checkerframework', 'org.opensearch.shaded.org.checkerframework' + relocate 'org.apache.commons.codec', 'org.opensearch.shaded.commons.codec' + relocate 'org.slf4j', 'org.opensearch.shaded.org.slf4j' + relocate 'com.google.flatbuffers', 'org.opensearch.shaded.com.google.flatbuffers' + relocate 'com.fasterxml.jackson', 'org.opensearch.shaded.com.fasterxml.jackson' + mergeServiceFiles() + archiveClassifier.set("") +} + +tasks.named('thirdPartyAudit').configure { + enabled = false +} + +tasks.register("javadocDummyFiles") { + doFirst { + def javadocDir = layout.buildDirectory.dir('docs/javadoc').get().asFile + javadocDir.mkdirs() + + new File(javadocDir, "element-list").text = "" + new File(javadocDir, "package-list").text = "" + } +} + +tasks.named('javadoc') { javadocTask -> + enabled = false + finalizedBy(tasks.named('javadocDummyFiles')) +} + +tasks.named("dependencyLicenses").configure { + mapping from: /jackson-.*/, to: 'jackson' +} diff --git a/libs/arrow-memory-shaded/licenses/arrow-format-18.1.0.jar.sha1 b/libs/arrow-memory-shaded/licenses/arrow-format-18.1.0.jar.sha1 new file mode 100644 index 0000000000000..6372bcd89eefd --- /dev/null +++ b/libs/arrow-memory-shaded/licenses/arrow-format-18.1.0.jar.sha1 @@ -0,0 +1 @@ +9d356b6f20620f5619ff85b174f97ae507df4997 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-format-LICENSE.txt b/libs/arrow-memory-shaded/licenses/arrow-format-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-format-LICENSE.txt rename to libs/arrow-memory-shaded/licenses/arrow-format-LICENSE.txt diff --git a/libs/arrow-spi/licenses/arrow-format-NOTICE.txt b/libs/arrow-memory-shaded/licenses/arrow-format-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-format-NOTICE.txt rename to libs/arrow-memory-shaded/licenses/arrow-format-NOTICE.txt diff --git a/libs/arrow-memory-shaded/licenses/arrow-memory-core-18.1.0.jar.sha1 b/libs/arrow-memory-shaded/licenses/arrow-memory-core-18.1.0.jar.sha1 new file mode 100644 index 0000000000000..1a4da42973bfe --- /dev/null +++ b/libs/arrow-memory-shaded/licenses/arrow-memory-core-18.1.0.jar.sha1 @@ -0,0 +1 @@ +35f4853d512f06759759b40b53bac850867886f8 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-memory-core-LICENSE.txt b/libs/arrow-memory-shaded/licenses/arrow-memory-core-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-memory-core-LICENSE.txt rename to libs/arrow-memory-shaded/licenses/arrow-memory-core-LICENSE.txt diff --git a/libs/arrow-spi/licenses/arrow-memory-core-NOTICE.txt b/libs/arrow-memory-shaded/licenses/arrow-memory-core-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-memory-core-NOTICE.txt rename to libs/arrow-memory-shaded/licenses/arrow-memory-core-NOTICE.txt diff --git a/libs/arrow-memory-shaded/licenses/arrow-memory-netty-18.1.0.jar.sha1 b/libs/arrow-memory-shaded/licenses/arrow-memory-netty-18.1.0.jar.sha1 new file mode 100644 index 0000000000000..291d435138e30 --- /dev/null +++ b/libs/arrow-memory-shaded/licenses/arrow-memory-netty-18.1.0.jar.sha1 @@ -0,0 +1 @@ +9e9e08d0b548d2c02c632e5daaf176e588810d22 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-LICENSE.txt b/libs/arrow-memory-shaded/licenses/arrow-memory-netty-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-memory-netty-LICENSE.txt rename to libs/arrow-memory-shaded/licenses/arrow-memory-netty-LICENSE.txt diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-NOTICE.txt b/libs/arrow-memory-shaded/licenses/arrow-memory-netty-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-memory-netty-NOTICE.txt rename to libs/arrow-memory-shaded/licenses/arrow-memory-netty-NOTICE.txt diff --git a/libs/arrow-memory-shaded/licenses/arrow-memory-netty-buffer-patch-18.1.0.jar.sha1 b/libs/arrow-memory-shaded/licenses/arrow-memory-netty-buffer-patch-18.1.0.jar.sha1 new file mode 100644 index 0000000000000..40c7b2992d715 --- /dev/null +++ b/libs/arrow-memory-shaded/licenses/arrow-memory-netty-buffer-patch-18.1.0.jar.sha1 @@ -0,0 +1 @@ +86c8fbdb6ab220603ea3a215f48a7f793ac6a08d \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-LICENSE.txt b/libs/arrow-memory-shaded/licenses/arrow-memory-netty-buffer-patch-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-LICENSE.txt rename to libs/arrow-memory-shaded/licenses/arrow-memory-netty-buffer-patch-LICENSE.txt diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-NOTICE.txt b/libs/arrow-memory-shaded/licenses/arrow-memory-netty-buffer-patch-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-NOTICE.txt rename to libs/arrow-memory-shaded/licenses/arrow-memory-netty-buffer-patch-NOTICE.txt diff --git a/libs/arrow-memory-shaded/licenses/arrow-vector-18.1.0.jar.sha1 b/libs/arrow-memory-shaded/licenses/arrow-vector-18.1.0.jar.sha1 new file mode 100644 index 0000000000000..d526f82b6f06e --- /dev/null +++ b/libs/arrow-memory-shaded/licenses/arrow-vector-18.1.0.jar.sha1 @@ -0,0 +1 @@ +b1fb77f4ef36fd52afe480ba12b7da77367eb88c \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-vector-LICENSE.txt b/libs/arrow-memory-shaded/licenses/arrow-vector-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-vector-LICENSE.txt rename to libs/arrow-memory-shaded/licenses/arrow-vector-LICENSE.txt diff --git a/libs/arrow-spi/licenses/arrow-vector-NOTICE.txt b/libs/arrow-memory-shaded/licenses/arrow-vector-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-vector-NOTICE.txt rename to libs/arrow-memory-shaded/licenses/arrow-vector-NOTICE.txt diff --git a/libs/arrow-memory-shaded/licenses/checker-qual-3.44.0.jar.sha1 b/libs/arrow-memory-shaded/licenses/checker-qual-3.44.0.jar.sha1 new file mode 100644 index 0000000000000..c3738dce281f2 --- /dev/null +++ b/libs/arrow-memory-shaded/licenses/checker-qual-3.44.0.jar.sha1 @@ -0,0 +1 @@ +e026b198319ea9dd3f221fab367d2099215079e5 \ No newline at end of file diff --git a/libs/arrow-memory-shaded/licenses/checker-qual-LICENSE.txt b/libs/arrow-memory-shaded/licenses/checker-qual-LICENSE.txt new file mode 100644 index 0000000000000..9837c6b69fdab --- /dev/null +++ b/libs/arrow-memory-shaded/licenses/checker-qual-LICENSE.txt @@ -0,0 +1,22 @@ +Checker Framework qualifiers +Copyright 2004-present by the Checker Framework developers + +MIT License: + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/libs/arrow-spi/licenses/flatbuffers-java-NOTICE.txt b/libs/arrow-memory-shaded/licenses/checker-qual-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/flatbuffers-java-NOTICE.txt rename to libs/arrow-memory-shaded/licenses/checker-qual-NOTICE.txt diff --git a/libs/arrow-spi/licenses/commons-codec-1.16.1.jar.sha1 b/libs/arrow-memory-shaded/licenses/commons-codec-1.16.1.jar.sha1 similarity index 100% rename from libs/arrow-spi/licenses/commons-codec-1.16.1.jar.sha1 rename to libs/arrow-memory-shaded/licenses/commons-codec-1.16.1.jar.sha1 diff --git a/libs/arrow-spi/licenses/commons-codec-LICENSE.txt b/libs/arrow-memory-shaded/licenses/commons-codec-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/commons-codec-LICENSE.txt rename to libs/arrow-memory-shaded/licenses/commons-codec-LICENSE.txt diff --git a/libs/arrow-spi/licenses/commons-codec-NOTICE.txt b/libs/arrow-memory-shaded/licenses/commons-codec-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/commons-codec-NOTICE.txt rename to libs/arrow-memory-shaded/licenses/commons-codec-NOTICE.txt diff --git a/libs/arrow-spi/licenses/flatbuffers-java-2.0.0.jar.sha1 b/libs/arrow-memory-shaded/licenses/flatbuffers-java-2.0.0.jar.sha1 similarity index 100% rename from libs/arrow-spi/licenses/flatbuffers-java-2.0.0.jar.sha1 rename to libs/arrow-memory-shaded/licenses/flatbuffers-java-2.0.0.jar.sha1 diff --git a/libs/arrow-spi/licenses/flatbuffers-java-LICENSE.txt b/libs/arrow-memory-shaded/licenses/flatbuffers-java-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/flatbuffers-java-LICENSE.txt rename to libs/arrow-memory-shaded/licenses/flatbuffers-java-LICENSE.txt diff --git a/libs/arrow-spi/licenses/slf4j-api-NOTICE.txt b/libs/arrow-memory-shaded/licenses/flatbuffers-java-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/slf4j-api-NOTICE.txt rename to libs/arrow-memory-shaded/licenses/flatbuffers-java-NOTICE.txt diff --git a/libs/arrow-spi/licenses/jackson-annotations-LICENSE.txt b/libs/arrow-memory-shaded/licenses/jackson-LICENSE similarity index 100% rename from libs/arrow-spi/licenses/jackson-annotations-LICENSE.txt rename to libs/arrow-memory-shaded/licenses/jackson-LICENSE diff --git a/libs/arrow-spi/licenses/jackson-annotations-NOTICE.txt b/libs/arrow-memory-shaded/licenses/jackson-NOTICE similarity index 100% rename from libs/arrow-spi/licenses/jackson-annotations-NOTICE.txt rename to libs/arrow-memory-shaded/licenses/jackson-NOTICE diff --git a/libs/arrow-spi/licenses/jackson-annotations-2.18.2.jar.sha1 b/libs/arrow-memory-shaded/licenses/jackson-annotations-2.18.2.jar.sha1 similarity index 100% rename from libs/arrow-spi/licenses/jackson-annotations-2.18.2.jar.sha1 rename to libs/arrow-memory-shaded/licenses/jackson-annotations-2.18.2.jar.sha1 diff --git a/libs/arrow-memory-shaded/licenses/jackson-core-2.18.2.jar.sha1 b/libs/arrow-memory-shaded/licenses/jackson-core-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..96350c9307ae7 --- /dev/null +++ b/libs/arrow-memory-shaded/licenses/jackson-core-2.18.2.jar.sha1 @@ -0,0 +1 @@ +fb64ccac5c27dca8819418eb4e443a9f496d9ee7 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/jackson-databind-2.18.2.jar.sha1 b/libs/arrow-memory-shaded/licenses/jackson-databind-2.18.2.jar.sha1 similarity index 100% rename from libs/arrow-spi/licenses/jackson-databind-2.18.2.jar.sha1 rename to libs/arrow-memory-shaded/licenses/jackson-databind-2.18.2.jar.sha1 diff --git a/libs/arrow-spi/licenses/netty-buffer-4.1.115.Final.jar.sha1 b/libs/arrow-memory-shaded/licenses/netty-buffer-4.1.115.Final.jar.sha1 similarity index 100% rename from libs/arrow-spi/licenses/netty-buffer-4.1.115.Final.jar.sha1 rename to libs/arrow-memory-shaded/licenses/netty-buffer-4.1.115.Final.jar.sha1 diff --git a/libs/arrow-spi/licenses/netty-buffer-LICENSE.txt b/libs/arrow-memory-shaded/licenses/netty-buffer-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/netty-buffer-LICENSE.txt rename to libs/arrow-memory-shaded/licenses/netty-buffer-LICENSE.txt diff --git a/libs/arrow-spi/licenses/netty-buffer-NOTICE.txt b/libs/arrow-memory-shaded/licenses/netty-buffer-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/netty-buffer-NOTICE.txt rename to libs/arrow-memory-shaded/licenses/netty-buffer-NOTICE.txt diff --git a/libs/arrow-spi/licenses/netty-common-4.1.115.Final.jar.sha1 b/libs/arrow-memory-shaded/licenses/netty-common-4.1.115.Final.jar.sha1 similarity index 100% rename from libs/arrow-spi/licenses/netty-common-4.1.115.Final.jar.sha1 rename to libs/arrow-memory-shaded/licenses/netty-common-4.1.115.Final.jar.sha1 diff --git a/libs/arrow-spi/licenses/netty-common-LICENSE.txt b/libs/arrow-memory-shaded/licenses/netty-common-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/netty-common-LICENSE.txt rename to libs/arrow-memory-shaded/licenses/netty-common-LICENSE.txt diff --git a/libs/arrow-spi/licenses/netty-common-NOTICE.txt b/libs/arrow-memory-shaded/licenses/netty-common-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/netty-common-NOTICE.txt rename to libs/arrow-memory-shaded/licenses/netty-common-NOTICE.txt diff --git a/libs/arrow-spi/licenses/slf4j-api-1.7.36.jar.sha1 b/libs/arrow-memory-shaded/licenses/slf4j-api-1.7.36.jar.sha1 similarity index 100% rename from libs/arrow-spi/licenses/slf4j-api-1.7.36.jar.sha1 rename to libs/arrow-memory-shaded/licenses/slf4j-api-1.7.36.jar.sha1 diff --git a/libs/arrow-spi/licenses/slf4j-api-LICENSE.txt b/libs/arrow-memory-shaded/licenses/slf4j-api-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/slf4j-api-LICENSE.txt rename to libs/arrow-memory-shaded/licenses/slf4j-api-LICENSE.txt diff --git a/libs/arrow-memory-shaded/licenses/slf4j-api-NOTICE.txt b/libs/arrow-memory-shaded/licenses/slf4j-api-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/libs/arrow-spi/build.gradle b/libs/arrow-spi/build.gradle index d14b7e88cfb8c..913527a8efb0e 100644 --- a/libs/arrow-spi/build.gradle +++ b/libs/arrow-spi/build.gradle @@ -10,22 +10,10 @@ */ testingConventions.enabled = false + dependencies { api project(':libs:opensearch-core') - api "org.apache.arrow:arrow-vector:${versions.arrow}" - api "org.apache.arrow:arrow-format:${versions.arrow}" - api "org.apache.arrow:arrow-memory-core:${versions.arrow}" - runtimeOnly "org.apache.arrow:arrow-memory-netty-buffer-patch:${versions.arrow}" - runtimeOnly "org.apache.arrow:arrow-memory-netty:${versions.arrow}" - runtimeOnly "io.netty:netty-buffer:${versions.netty}" - runtimeOnly "io.netty:netty-common:${versions.netty}" - - runtimeOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" - runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}" - runtimeOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" - api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" - - implementation "commons-codec:commons-codec:${versions.commonscodec}" + api project(path:':libs:opensearch-arrow-memory-shaded', configuration: 'shadow') } tasks.named('forbiddenApisMain').configure { @@ -34,55 +22,8 @@ tasks.named('forbiddenApisMain').configure { tasks.named('thirdPartyAudit').configure { ignoreMissingClasses( - // Logging frameworks - 'org.apache.commons.logging.Log', - 'org.apache.commons.logging.LogFactory', - 'org.apache.log4j.Level', - 'org.apache.log4j.Logger', - 'org.slf4j.impl.StaticLoggerBinder', - 'org.slf4j.impl.StaticMDCBinder', - 'org.slf4j.impl.StaticMarkerBinder', - - // Reactor BlockHound - 'reactor.blockhound.BlockHound$Builder', - 'reactor.blockhound.integration.BlockHoundIntegration' - ) - - ignoreViolations( - "io.netty.util.internal.PlatformDependent0", - "io.netty.util.internal.PlatformDependent0\$1", - "io.netty.util.internal.PlatformDependent0\$2", - "io.netty.util.internal.PlatformDependent0\$3", - "io.netty.util.internal.PlatformDependent0\$4", - "io.netty.util.internal.PlatformDependent0\$6", - "io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef", - "io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef", - "io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields", - "io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields", - "io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields", - "io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode", - "io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField", - "io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField", - "io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField", - "io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField", - "io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField", - "io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess", - "io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess", - "io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess", - "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField", - "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField", - "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField", - "org.apache.arrow.memory.ArrowBuf", - "org.apache.arrow.memory.util.ByteFunctionHelpers", - "org.apache.arrow.memory.util.MemoryUtil", - "org.apache.arrow.memory.util.MemoryUtil\$1", - "org.apache.arrow.memory.util.hash.MurmurHasher", - "org.apache.arrow.memory.util.hash.SimpleHasher", - "org.apache.arrow.vector.BaseFixedWidthVector", - "org.apache.arrow.vector.BitVectorHelper", - "org.apache.arrow.vector.Decimal256Vector", - "org.apache.arrow.vector.DecimalVector", - "org.apache.arrow.vector.util.DecimalUtility", - "org.apache.arrow.vector.util.VectorAppender" + 'org.slf4j.Logger', + 'org.slf4j.LoggerFactory', + 'org.apache.commons.codec.binary.Hex' ) } diff --git a/libs/arrow-spi/licenses/arrow-format-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-format-17.0.0.jar.sha1 deleted file mode 100644 index 34fd4704eac91..0000000000000 --- a/libs/arrow-spi/licenses/arrow-format-17.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5d052f20fd1193840eb59818515e710156c364b2 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-memory-core-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-memory-core-17.0.0.jar.sha1 deleted file mode 100644 index ea312f4f5e51a..0000000000000 --- a/libs/arrow-spi/licenses/arrow-memory-core-17.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -51c5287ef5a624656bb38da7684078905b1a88c9 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-memory-netty-17.0.0.jar.sha1 deleted file mode 100644 index f77b3d836b77b..0000000000000 --- a/libs/arrow-spi/licenses/arrow-memory-netty-17.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -de65a34dfeada4d47b161871fa39fa0a2ab4c39c \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-17.0.0.jar.sha1 deleted file mode 100644 index b21b4e8cc7d23..0000000000000 --- a/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-17.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cdfdaa1bd5135bd869515fc205392ba92dcc1509 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-vector-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-vector-17.0.0.jar.sha1 deleted file mode 100644 index 8f9fddc882396..0000000000000 --- a/libs/arrow-spi/licenses/arrow-vector-17.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -16685545e4734382c1fcdaf12ac9b0a7d1fc06c0 \ No newline at end of file diff --git a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java index c5cd6f16adfdd..4cdc59c588b80 100644 --- a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java +++ b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java @@ -11,6 +11,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.VectorSchemaRoot; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.unit.TimeValue; import org.opensearch.core.tasks.TaskId; import java.io.Closeable; @@ -86,7 +87,7 @@ public interface StreamProducer extends Closeable { * @param allocator The allocator to use for creating vectors * @return A new VectorSchemaRoot instance */ - VectorSchemaRoot createRoot(BufferAllocator allocator); + VectorSchemaRoot createRoot(BufferAllocator allocator) throws Exception; /** * Creates a job that will produce the stream data in batches. The job will populate @@ -97,6 +98,14 @@ public interface StreamProducer extends Closeable { */ BatchedJob createJob(BufferAllocator allocator); + /** + * Returns the deadline for the job execution. + * After this deadline, the job should be considered expired. + * + * @return TimeValue representing the job's deadline + */ + TimeValue getJobDeadline(); + /** * Provides an estimate of the total number of rows that will be produced. * @@ -122,7 +131,7 @@ interface BatchedJob { * @param root The VectorSchemaRoot to populate with data * @param flushSignal Signal to coordinate with consumers */ - void run(VectorSchemaRoot root, FlushSignal flushSignal); + void run(VectorSchemaRoot root, FlushSignal flushSignal) throws Exception; /** * Called to signal producer when the job is canceled. @@ -150,8 +159,8 @@ interface FlushSignal { /** * Blocks until the current batch has been consumed or timeout occurs. * - * @param timeout Maximum milliseconds to wait + * @param timeout Maximum time to wait */ - void awaitConsumption(int timeout); + void awaitConsumption(TimeValue timeout); } } diff --git a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamReader.java b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamReader.java index b258652988b96..6da24576d1936 100644 --- a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamReader.java +++ b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamReader.java @@ -43,14 +43,16 @@ public interface StreamReader extends Closeable { * Blocking request to load next batch into root. * * @return true if more data was found, false if the stream is exhausted + * @throws Exception if an error occurs while loading the next batch */ - boolean next(); + boolean next() throws Exception; /** * Returns the VectorSchemaRoot associated with this iterator. * The content of this root is updated with each successful call to next(). * * @return the VectorSchemaRoot + * @throws Exception if an error occurs while retrieving the root */ - VectorSchemaRoot getRoot(); + VectorSchemaRoot getRoot() throws Exception; } diff --git a/libs/build.gradle b/libs/build.gradle index c0fcc1ff2b977..27ba3ad2102b8 100644 --- a/libs/build.gradle +++ b/libs/build.gradle @@ -44,12 +44,12 @@ subprojects { dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep -> Project depProject = project.project(dep.path) if (depProject != null - && (false == depProject.path.equals(':libs:opensearch-core') && + && (false == depProject.path.equals(':libs:opensearch-core') && false == depProject.path.equals(':libs:opensearch-arrow-memory-shaded') && false == depProject.path.equals(':libs:opensearch-common')) && depProject.path.startsWith(':libs')) { throw new InvalidUserDataException("projects in :libs " + "may not depend on other projects libs except " - + ":libs:opensearch-core or :libs:opensearch-common but " + + ":libs:opensearch-core or :libs:opensearch-common or :libs:opensearch-arrow-memory-shaded but " + "${project.path} depends on ${depProject.path}") } } diff --git a/libs/flight-core-shaded/build.gradle b/libs/flight-core-shaded/build.gradle new file mode 100644 index 0000000000000..3b5293e115683 --- /dev/null +++ b/libs/flight-core-shaded/build.gradle @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +plugins { + id("com.gradleup.shadow") +} + +description = '''arrow-flight-core netty shaded dependencies to make it compatible with grpc-netty-shaded''' + +testingConventions.enabled = false + +// since we are not publishing it +generatePomFileForShadowPublication.enabled = false + +dependencies { + runtimeOnly "org.apache.arrow:flight-core:${versions.arrow}" +} + +shadowJar { + dependencies { + include(dependency('org.apache.arrow:')) + } + exclude 'META-INF/maven/**' + relocate 'io.grpc.netty', 'io.grpc.netty.shaded.io.grpc.netty' + relocate 'io.netty', 'io.grpc.netty.shaded.io.netty' + mergeServiceFiles() + archiveClassifier.set("") +} + +tasks.named('thirdPartyAudit').configure { + enabled = false +} + +// creating dummy javadoc to let javadoc check pass when other package use it as a dependency +tasks.register("javadocDummyFiles") { + doFirst { + def javadocDir = layout.buildDirectory.dir('docs/javadoc').get().asFile + javadocDir.mkdirs() + + new File(javadocDir, "element-list").text = "" + new File(javadocDir, "package-list").text = "" + } +} + +tasks.named('javadoc') { javadocTask -> + enabled = false + finalizedBy(tasks.named('javadocDummyFiles')) +} diff --git a/libs/flight-core-shaded/licenses/flight-core-18.1.0.jar.sha1 b/libs/flight-core-shaded/licenses/flight-core-18.1.0.jar.sha1 new file mode 100644 index 0000000000000..fc2e34539cf04 --- /dev/null +++ b/libs/flight-core-shaded/licenses/flight-core-18.1.0.jar.sha1 @@ -0,0 +1 @@ +82494895fcb0656967680442f63ce1214e532d52 \ No newline at end of file diff --git a/libs/flight-core-shaded/licenses/flight-core-LICENSE.txt b/libs/flight-core-shaded/licenses/flight-core-LICENSE.txt new file mode 100644 index 0000000000000..7bb1330a1002b --- /dev/null +++ b/libs/flight-core-shaded/licenses/flight-core-LICENSE.txt @@ -0,0 +1,2261 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +src/arrow/util (some portions): Apache 2.0, and 3-clause BSD + +Some portions of this module are derived from code in the Chromium project, +copyright (c) Google inc and (c) The Chromium Authors and licensed under the +Apache 2.0 License or the under the 3-clause BSD license: + + Copyright (c) 2013 The Chromium Authors. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from Daniel Lemire's FrameOfReference project. + +https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp +https://github.com/lemire/FrameOfReference/blob/146948b6058a976bc7767262ad3a2ce201486b93/scripts/turbopacking64.py + +Copyright: 2013 Daniel Lemire +Home page: http://lemire.me/en/ +Project page: https://github.com/lemire/FrameOfReference +License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from the TensorFlow project + +Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the NumPy project. + +https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910 + +https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c + +Copyright (c) 2005-2017, NumPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NumPy Developers nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the Boost project + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from the FlatBuffers project + +Copyright 2014 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the tslib project + +Copyright 2015 Microsoft Corporation. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the jemalloc project + +https://github.com/jemalloc/jemalloc + +Copyright (C) 2002-2017 Jason Evans . +All rights reserved. +Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. +Copyright (C) 2009-2017 Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice(s), + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice(s), + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- + +This project includes code from the Go project, BSD 3-clause license + PATENTS +weak patent termination clause +(https://github.com/golang/go/blob/master/PATENTS). + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the hs2client + +https://github.com/cloudera/hs2client + +Copyright 2016 Cloudera Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +The script ci/scripts/util_wait_for_it.sh has the following license + +Copyright (c) 2016 Giles Hall + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The script r/configure has the following license (MIT) + +Copyright (c) 2017, Jeroen Ooms and Jim Hester + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +cpp/src/arrow/util/logging.cc, cpp/src/arrow/util/logging.h and +cpp/src/arrow/util/logging-test.cc are adapted from +Ray Project (https://github.com/ray-project/ray) (Apache 2.0). + +Copyright (c) 2016 Ray Project (https://github.com/ray-project/ray) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- +The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h, +cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h, +cpp/src/arrow/vendored/datetime/ios.mm, +cpp/src/arrow/vendored/datetime/tz.cpp are adapted from +Howard Hinnant's date library (https://github.com/HowardHinnant/date) +It is licensed under MIT license. + +The MIT License (MIT) +Copyright (c) 2015, 2016, 2017 Howard Hinnant +Copyright (c) 2016 Adrian Colomitchi +Copyright (c) 2017 Florian Dang +Copyright (c) 2017 Paul Thompson +Copyright (c) 2018 Tomasz KamiƄski + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/src/arrow/util/utf8.h includes code adapted from the page + https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +with the following license (MIT) + +Copyright (c) 2008-2009 Bjoern Hoehrmann + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/xxhash/ have the following license +(BSD 2-Clause License) + +xxHash Library +Copyright (c) 2012-2014, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash homepage: http://www.xxhash.com +- xxHash source repository : https://github.com/Cyan4973/xxHash + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/double-conversion/ have the following license +(BSD 3-Clause License) + +Copyright 2006-2011, the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/uriparser/ have the following license +(BSD 3-Clause License) + +uriparser - RFC 3986 URI parsing library + +Copyright (C) 2007, Weijia Song +Copyright (C) 2007, Sebastian Pipping +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the following + disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials + provided with the distribution. + + * Neither the name of the nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files under dev/tasks/conda-recipes have the following license + +BSD 3-clause license +Copyright (c) 2015-2018, conda-forge +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/utfcpp/ have the following license + +Copyright 2006-2018 Nemanja Trifunovic + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from Apache Kudu. + + * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake + +Copyright: 2016 The Apache Software Foundation. +Home page: https://kudu.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Apache Impala (incubating), formerly +Impala. The Impala code and rights were donated to the ASF as part of the +Incubator process after the initial code imports into Apache Parquet. + +Copyright: 2012 Cloudera, Inc. +Copyright: 2016 The Apache Software Foundation. +Home page: http://impala.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Apache Aurora. + +* dev/release/{release,changelog,release-candidate} are based on the scripts from + Apache Aurora + +Copyright: 2016 The Apache Software Foundation. +Home page: https://aurora.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from the Google styleguide. + +* cpp/build-support/cpplint.py is based on the scripts from the Google styleguide. + +Copyright: 2009 Google Inc. All rights reserved. +Homepage: https://github.com/google/styleguide +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +This project includes code from Snappy. + +* cpp/cmake_modules/{SnappyCMakeLists.txt,SnappyConfig.h} are based on code + from Google's Snappy project. + +Copyright: 2009 Google Inc. All rights reserved. +Homepage: https://github.com/google/snappy +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +This project includes code from the manylinux project. + +* python/manylinux1/scripts/{build_python.sh,python-tag-abi-tag.py, + requirements.txt} are based on code from the manylinux project. + +Copyright: 2016 manylinux +Homepage: https://github.com/pypa/manylinux +License: The MIT License (MIT) + +-------------------------------------------------------------------------------- + +This project includes code from the cymove project: + +* python/pyarrow/includes/common.pxd includes code from the cymove project + +The MIT License (MIT) +Copyright (c) 2019 Omer Ozarslan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The projects includes code from the Ursabot project under the dev/archery +directory. + +License: BSD 2-Clause + +Copyright 2019 RStudio, Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project include code from mingw-w64. + +* cpp/src/arrow/util/cpu-info.cc has a polyfill for mingw-w64 < 5 + +Copyright (c) 2009 - 2013 by the mingw-w64 project +Homepage: https://mingw-w64.org +License: Zope Public License (ZPL) Version 2.1. + +--------------------------------------------------------------------------------- + +This project include code from Google's Asylo project. + +* cpp/src/arrow/result.h is based on status_or.h + +Copyright (c) Copyright 2017 Asylo authors +Homepage: https://asylo.dev/ +License: Apache 2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Google's protobuf project + +* cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN +* cpp/src/arrow/util/bit_stream_utils.h contains code from wire_format_lite.h + +Copyright 2008 Google Inc. All rights reserved. +Homepage: https://developers.google.com/protocol-buffers/ +License: + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +-------------------------------------------------------------------------------- + +3rdparty dependency LLVM is statically linked in certain binary distributions. +Additionally some sections of source code have been derived from sources in LLVM +and have been clearly labeled as such. LLVM has the following license: + +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +-------------------------------------------------------------------------------- + +3rdparty dependency gRPC is statically linked in certain binary +distributions, like the python wheels. gRPC has the following license: + +Copyright 2014 gRPC authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency Apache Thrift is statically linked in certain binary +distributions, like the python wheels. Apache Thrift has the following license: + +Apache Thrift +Copyright (C) 2006 - 2019, The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency Apache ORC is statically linked in certain binary +distributions, like the python wheels. Apache ORC has the following license: + +Apache ORC +Copyright 2013-2019 The Apache Software Foundation + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +This product includes software developed by Hewlett-Packard: +(c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency zstd is statically linked in certain binary +distributions, like the python wheels. ZSTD has the following license: + +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency lz4 is statically linked in certain binary +distributions, like the python wheels. lz4 has the following license: + +LZ4 Library +Copyright (c) 2011-2016, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency Brotli is statically linked in certain binary +distributions, like the python wheels. Brotli has the following license: + +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency rapidjson is statically linked in certain binary +distributions, like the python wheels. rapidjson and its dependencies have the +following licenses: + +Tencent is pleased to support the open source community by making RapidJSON +available. + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +All rights reserved. + +If you have downloaded a copy of the RapidJSON binary from Tencent, please note +that the RapidJSON binary is licensed under the MIT License. +If you have downloaded a copy of the RapidJSON source code from Tencent, please +note that RapidJSON source code is licensed under the MIT License, except for +the third-party components listed below which are subject to different license +terms. Your integration of RapidJSON into your own projects may require +compliance with the MIT License, as well as the other licenses applicable to +the third-party components included within RapidJSON. To avoid the problematic +JSON license in your own projects, it's sufficient to exclude the +bin/jsonchecker/ directory, as it's the only code under the JSON license. +A copy of the MIT License is included in this file. + +Other dependencies and licenses: + + Open Source Software Licensed Under the BSD License: + -------------------------------------------------------------------- + + The msinttypes r29 + Copyright (c) 2006-2013 Alexander Chemeris + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. + + Terms of the MIT License: + -------------------------------------------------------------------- + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency snappy is statically linked in certain binary +distributions, like the python wheels. snappy has the following license: + +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Google Inc. nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=== + +Some of the benchmark data in testdata/ is licensed differently: + + - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and + is licensed under the Creative Commons Attribution 3.0 license + (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ + for more information. + + - kppkn.gtb is taken from the Gaviota chess tablebase set, and + is licensed under the MIT License. See + https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 + for more information. + + - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper + “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA + Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, + which is licensed under the CC-BY license. See + http://www.ploscompbiol.org/static/license for more ifnormation. + + - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project + Gutenberg. The first three have expired copyrights and are in the public + domain; the latter does not have expired copyright, but is still in the + public domain according to the license information + (http://www.gutenberg.org/ebooks/53). + +-------------------------------------------------------------------------------- + +3rdparty dependency gflags is statically linked in certain binary +distributions, like the python wheels. gflags has the following license: + +Copyright (c) 2006, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency glog is statically linked in certain binary +distributions, like the python wheels. glog has the following license: + +Copyright (c) 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +A function gettimeofday in utilities.cc is based on + +http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd + +The license of this code is: + +Copyright (c) 2003-2008, Jouni Malinen and contributors +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name(s) of the above-listed copyright holder(s) nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency re2 is statically linked in certain binary +distributions, like the python wheels. re2 has the following license: + +Copyright (c) 2009 The RE2 Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency c-ares is statically linked in certain binary +distributions, like the python wheels. c-ares has the following license: + +# c-ares license + +Copyright (c) 2007 - 2018, Daniel Stenberg with many contributors, see AUTHORS +file. + +Copyright 1998 by the Massachusetts Institute of Technology. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, provided that +the above copyright notice appear in all copies and that both that copyright +notice and this permission notice appear in supporting documentation, and that +the name of M.I.T. not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior permission. +M.I.T. makes no representations about the suitability of this software for any +purpose. It is provided "as is" without express or implied warranty. + +-------------------------------------------------------------------------------- + +3rdparty dependency zlib is redistributed as a dynamically linked shared +library in certain binary distributions, like the python wheels. In the future +this will likely change to static linkage. zlib has the following license: + +zlib.h -- interface of the 'zlib' general purpose compression library + version 1.2.11, January 15th, 2017 + + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + +-------------------------------------------------------------------------------- + +3rdparty dependency openssl is redistributed as a dynamically linked shared +library in certain binary distributions, like the python wheels. openssl +preceding version 3 has the following license: + + LICENSE ISSUES + ============== + + The OpenSSL toolkit stays under a double license, i.e. both the conditions of + the OpenSSL License and the original SSLeay license apply to the toolkit. + See below for the actual license texts. + + OpenSSL License + --------------- + +/* ==================================================================== + * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + + Original SSLeay License + ----------------------- + +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ + +-------------------------------------------------------------------------------- + +This project includes code from the rtools-backports project. + +* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code + from the rtools-backports project. + +Copyright: Copyright (c) 2013 - 2019, АлДĐșсДĐč and Jeroen Ooms. +All rights reserved. +Homepage: https://github.com/r-windows/rtools-backports +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +Some code from pandas has been adapted for the pyarrow codebase. pandas is +available under the 3-clause BSD license, which follows: + +pandas license +============== + +Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team +All rights reserved. + +Copyright (c) 2008-2011 AQR Capital Management, LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the copyright holder nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +Some bits from DyND, in particular aspects of the build system, have been +adapted from libdynd and dynd-python under the terms of the BSD 2-clause +license + +The BSD 2-Clause License + + Copyright (C) 2011-12, Dynamic NDArray Developers + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Dynamic NDArray Developers list: + + * Mark Wiebe + * Continuum Analytics + +-------------------------------------------------------------------------------- + +Some source code from Ibis (https://github.com/cloudera/ibis) has been adapted +for PyArrow. Ibis is released under the Apache License, Version 2.0. + +-------------------------------------------------------------------------------- + +dev/tasks/homebrew-formulae/apache-arrow.rb has the following license: + +BSD 2-Clause License + +Copyright (c) 2009-present, Homebrew contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- + +cpp/src/arrow/vendored/base64.cpp has the following license + +ZLIB License + +Copyright (C) 2004-2017 RenĂ© Nyffenegger + +This source code is provided 'as-is', without any express or implied +warranty. In no event will the author be held liable for any damages arising +from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including +commercial applications, and to alter it and redistribute it freely, subject to +the following restrictions: + +1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. + +3. This notice may not be removed or altered from any source distribution. + +RenĂ© Nyffenegger rene.nyffenegger@adp-gmbh.ch + +-------------------------------------------------------------------------------- + +This project includes code from Folly. + + * cpp/src/arrow/vendored/ProducerConsumerQueue.h + +is based on Folly's + + * folly/Portability.h + * folly/lang/Align.h + * folly/ProducerConsumerQueue.h + +Copyright: Copyright (c) Facebook, Inc. and its affiliates. +Home page: https://github.com/facebook/folly +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +The file cpp/src/arrow/vendored/musl/strptime.c has the following license + +Copyright © 2005-2020 Rich Felker, et al. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/cmake_modules/BuildUtils.cmake contains code from + +https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 + +which is made available under the MIT license + +Copyright (c) 2019 Cristian Adam + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/portable-snippets/ contain code from + +https://github.com/nemequ/portable-snippets + +and have the following copyright notice: + +Each source file contains a preamble explaining the license situation +for that file, which takes priority over this file. With the +exception of some code pulled in from other repositories (such as +”nit, an MIT-licensed project which is used for testing), the code is +public domain, released using the CC0 1.0 Universal dedication (*). + +(*) https://creativecommons.org/publicdomain/zero/1.0/legalcode + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/fast_float/ contain code from + +https://github.com/lemire/fast_float + +which is made available under the Apache License 2.0. + +-------------------------------------------------------------------------------- + +The file python/pyarrow/vendored/docscrape.py contains code from + +https://github.com/numpy/numpydoc/ + +which is made available under the BSD 2-clause license. + +-------------------------------------------------------------------------------- + +The file python/pyarrow/vendored/version.py contains code from + +https://github.com/pypa/packaging/ + +which is made available under both the Apache license v2.0 and the +BSD 2-clause license. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/pcg contain code from + +https://github.com/imneme/pcg-cpp + +and have the following copyright notice: + +Copyright 2014-2019 Melissa O'Neill , + and the PCG Project contributors. + +SPDX-License-Identifier: (Apache-2.0 OR MIT) + +Licensed under the Apache License, Version 2.0 (provided in +LICENSE-APACHE.txt and at http://www.apache.org/licenses/LICENSE-2.0) +or under the MIT license (provided in LICENSE-MIT.txt and at +http://opensource.org/licenses/MIT), at your option. This file may not +be copied, modified, or distributed except according to those terms. + +Distributed on an "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, either +express or implied. See your chosen license for details. + +-------------------------------------------------------------------------------- +r/R/dplyr-count-tally.R (some portions) + +Some portions of this file are derived from code from + +https://github.com/tidyverse/dplyr/ + +which is made available under the MIT license + +Copyright (c) 2013-2019 RStudio and others. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the “Software”), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The file src/arrow/util/io_util.cc contains code from the CPython project +which is made available under the Python Software Foundation License Version 2. + +-------------------------------------------------------------------------------- + +3rdparty dependency opentelemetry-cpp is statically linked in certain binary +distributions. opentelemetry-cpp is made available under the Apache License 2.0. + +Copyright The OpenTelemetry Authors +SPDX-License-Identifier: Apache-2.0 + +-------------------------------------------------------------------------------- + +ci/conan/ is based on code from Conan Package and Dependency Manager. + +Copyright (c) 2019 Conan.io + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency UCX is redistributed as a dynamically linked shared +library in certain binary distributions. UCX has the following license: + +Copyright (c) 2014-2015 UT-Battelle, LLC. All rights reserved. +Copyright (C) 2014-2020 Mellanox Technologies Ltd. All rights reserved. +Copyright (C) 2014-2015 The University of Houston System. All rights reserved. +Copyright (C) 2015 The University of Tennessee and The University + of Tennessee Research Foundation. All rights reserved. +Copyright (C) 2016-2020 ARM Ltd. All rights reserved. +Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved. +Copyright (C) 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +Copyright (C) 2019 UChicago Argonne, LLC. All rights reserved. +Copyright (c) 2018-2020 NVIDIA CORPORATION. All rights reserved. +Copyright (C) 2020 Huawei Technologies Co., Ltd. All rights reserved. +Copyright (C) 2016-2020 Stony Brook University. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The file dev/tasks/r/github.packages.yml contains code from + +https://github.com/ursa-labs/arrow-r-nightly + +which is made available under the Apache License 2.0. + +-------------------------------------------------------------------------------- +.github/actions/sync-nightlies/action.yml (some portions) + +Some portions of this file are derived from code from + +https://github.com/JoshPiper/rsync-docker + +which is made available under the MIT license + +Copyright (c) 2020 Joshua Piper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +.github/actions/sync-nightlies/action.yml (some portions) + +Some portions of this file are derived from code from + +https://github.com/burnett01/rsync-deployments + +which is made available under the MIT license + +Copyright (c) 2019-2022 Contention +Copyright (c) 2019-2022 Burnett01 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java + +These file are derived from code from Netty, which is made available under the +Apache License 2.0. diff --git a/libs/flight-core-shaded/licenses/flight-core-NOTICE.txt b/libs/flight-core-shaded/licenses/flight-core-NOTICE.txt new file mode 100644 index 0000000000000..2089c6fb20358 --- /dev/null +++ b/libs/flight-core-shaded/licenses/flight-core-NOTICE.txt @@ -0,0 +1,84 @@ +Apache Arrow +Copyright 2016-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product includes software from the SFrame project (BSD, 3-clause). +* Copyright (C) 2015 Dato, Inc. +* Copyright (c) 2009 Carnegie Mellon University. + +This product includes software from the Feather project (Apache 2.0) +https://github.com/wesm/feather + +This product includes software from the DyND project (BSD 2-clause) +https://github.com/libdynd + +This product includes software from the LLVM project + * distributed under the University of Illinois Open Source + +This product includes software from the google-lint project + * Copyright (c) 2009 Google Inc. All rights reserved. + +This product includes software from the mman-win32 project + * Copyright https://code.google.com/p/mman-win32/ + * Licensed under the MIT License; + +This product includes software from the LevelDB project + * Copyright (c) 2011 The LevelDB Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * Moved from Kudu http://github.com/cloudera/kudu + +This product includes software from the CMake project + * Copyright 2001-2009 Kitware, Inc. + * Copyright 2012-2014 Continuum Analytics, Inc. + * All rights reserved. + +This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause) + * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved. + +This product includes software from the Ibis project (Apache 2.0) + * Copyright (c) 2015 Cloudera, Inc. + * https://github.com/cloudera/ibis + +This product includes software from Dremio (Apache 2.0) + * Copyright (C) 2017-2018 Dremio Corporation + * https://github.com/dremio/dremio-oss + +This product includes software from Google Guava (Apache 2.0) + * Copyright (C) 2007 The Guava Authors + * https://github.com/google/guava + +This product include software from CMake (BSD 3-Clause) + * CMake - Cross Platform Makefile Generator + * Copyright 2000-2019 Kitware, Inc. and Contributors + +The web site includes files generated by Jekyll. + +-------------------------------------------------------------------------------- + +This product includes code from Apache Kudu, which includes the following in +its NOTICE file: + + Apache Kudu + Copyright 2016 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + Portions of this software were developed at + Cloudera, Inc (http://www.cloudera.com/). + +-------------------------------------------------------------------------------- + +This product includes code from Apache ORC, which includes the following in +its NOTICE file: + + Apache ORC + Copyright 2013-2019 The Apache Software Foundation + + This product includes software developed by The Apache Software + Foundation (http://www.apache.org/). + + This product includes software developed by Hewlett-Packard: + (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P diff --git a/plugins/arrow-flight-rpc/build.gradle b/plugins/arrow-flight-rpc/build.gradle new file mode 100644 index 0000000000000..29c1ee970ca73 --- /dev/null +++ b/plugins/arrow-flight-rpc/build.gradle @@ -0,0 +1,230 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +apply plugin: 'opensearch.internal-cluster-test' + +opensearchplugin { + description = 'Arrow flight based Stream implementation' + classname = 'org.opensearch.arrow.flight.FlightStreamPlugin' +} + +dependencies { + implementation project(':libs:opensearch-arrow-spi') + implementation project(path:':libs:opensearch-flight-core-shaded', configuration: 'shadow') + implementation "io.grpc:grpc-netty-shaded:${versions.grpc}" + + implementation "io.grpc:grpc-api:${versions.grpc}" + runtimeOnly "io.grpc:grpc-core:${versions.grpc}" + implementation "io.grpc:grpc-stub:${versions.grpc}" + + runtimeOnly group: 'com.google.code.findbugs', name: 'jsr305', version: '3.0.2' + compileOnly 'org.immutables:value:2.10.1' + annotationProcessor 'org.immutables:value:2.10.1' + + implementation "org.slf4j:slf4j-api:${versions.slf4j}" + runtimeOnly 'io.perfmark:perfmark-api:0.27.0' + runtimeOnly 'org.apache.parquet:parquet-arrow:1.13.1' + runtimeOnly "io.grpc:grpc-protobuf-lite:${versions.grpc}" + runtimeOnly "io.grpc:grpc-protobuf:${versions.grpc}" + runtimeOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" + runtimeOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + + runtimeOnly "com.google.guava:failureaccess:1.0.1" + compileOnly "com.google.errorprone:error_prone_annotations:2.31.0" + runtimeOnly('com.google.guava:guava:33.3.1-jre') { + attributes { + attribute(Attribute.of('org.gradle.jvm.environment', String), 'standard-jvm') + } + } +} + +tasks.internalClusterTest { + jvmArgs += ["--add-opens", "java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED"] +} + +tasks.named('test').configure { + jacoco { + excludes = ['org/apache/arrow/flight/**'] + } +} + +tasks.named("dependencyLicenses").configure { + mapping from: /netty-.*/, to: 'netty' + mapping from: /grpc-.*/, to: 'grpc' + mapping from: /jackson-.*/, to: 'jackson' +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' + + excludes = [ + 'org/apache/arrow/flight/OSFlightServer$Builder.class', + 'org/apache/arrow/flight/OSFlightClient$Builder.class', + 'org/opensearch/flight/bootstrap/server/ServerConfig$Netty4Configs.class', + 'org/opensearch/flight/bootstrap/server/ServerConfig.class', + 'org/opensearch/flight/bootstrap/tls/DefaultSslContextProvider.class', + 'org/apache/arrow/flight/OpenSearchFlightClient$Builder.class' + ] +} + +tasks.named('thirdPartyAudit').configure { + ignoreMissingClasses( + 'com.google.gson.stream.JsonReader', + 'com.google.gson.stream.JsonToken', + 'org.apache.parquet.schema.GroupType', + 'com.google.rpc.Status', + 'com.google.rpc.Status$Builder', + // Parquet Schema classes + 'org.apache.parquet.schema.LogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$DateLogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$DecimalLogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$IntLogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$IntervalLogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$ListLogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$LogicalTypeAnnotationVisitor', + 'org.apache.parquet.schema.LogicalTypeAnnotation$StringLogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$TimeLogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$TimeUnit', + 'org.apache.parquet.schema.LogicalTypeAnnotation$TimestampLogicalTypeAnnotation', + 'org.apache.parquet.schema.MessageType', + 'org.apache.parquet.schema.OriginalType', + 'org.apache.parquet.schema.PrimitiveType', + 'org.apache.parquet.schema.PrimitiveType$PrimitiveTypeName', + 'org.apache.parquet.schema.PrimitiveType$PrimitiveTypeNameConverter', + 'org.apache.parquet.schema.Type', + 'org.apache.parquet.schema.Type$Repetition', + 'org.apache.parquet.schema.Types', + 'org.apache.parquet.schema.Types$BaseListBuilder', + 'org.apache.parquet.schema.Types$GroupBuilder', + 'org.apache.parquet.schema.Types$ListBuilder', + 'org.apache.parquet.schema.Types$PrimitiveBuilder', + + 'com.aayushatharva.brotli4j.Brotli4jLoader', + 'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status', + 'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper', + 'com.aayushatharva.brotli4j.encoder.BrotliEncoderChannel', + 'com.aayushatharva.brotli4j.encoder.Encoder$Mode', + 'com.aayushatharva.brotli4j.encoder.Encoder$Parameters', + // classes are missing + + // from io.netty.logging.CommonsLoggerFactory (netty) + 'org.apache.commons.logging.Log', + 'org.apache.commons.logging.LogFactory', + + 'org.slf4j.impl.StaticLoggerBinder', + 'org.slf4j.impl.StaticMDCBinder', + 'org.slf4j.impl.StaticMarkerBinder', + + // from Log4j (deliberate, Netty will fallback to Log4j 2) + 'org.apache.log4j.Level', + 'org.apache.log4j.Logger', + + // from io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (netty) + 'org.bouncycastle.cert.X509v3CertificateBuilder', + 'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter', + 'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder', + 'org.bouncycastle.openssl.PEMEncryptedKeyPair', + 'org.bouncycastle.openssl.PEMParser', + 'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter', + 'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder', + 'org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder', + 'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo', + + // from io.netty.handler.ssl.JettyNpnSslEngine (netty) + 'org.eclipse.jetty.npn.NextProtoNego$ClientProvider', + 'org.eclipse.jetty.npn.NextProtoNego$ServerProvider', + 'org.eclipse.jetty.npn.NextProtoNego', + + // from io.netty.handler.codec.marshalling.ChannelBufferByteInput (netty) + 'org.jboss.marshalling.ByteInput', + + // from io.netty.handler.codec.marshalling.ChannelBufferByteOutput (netty) + 'org.jboss.marshalling.ByteOutput', + + // from io.netty.handler.codec.marshalling.CompatibleMarshallingEncoder (netty) + 'org.jboss.marshalling.Marshaller', + + // from io.netty.handler.codec.marshalling.ContextBoundUnmarshallerProvider (netty) + 'org.jboss.marshalling.MarshallerFactory', + 'org.jboss.marshalling.MarshallingConfiguration', + 'org.jboss.marshalling.Unmarshaller', + + 'com.google.protobuf.nano.CodedOutputByteBufferNano', + 'com.google.protobuf.nano.MessageNano', + 'com.ning.compress.BufferRecycler', + 'com.ning.compress.lzf.ChunkDecoder', + 'com.ning.compress.lzf.ChunkEncoder', + 'com.ning.compress.lzf.LZFChunk', + 'com.ning.compress.lzf.LZFEncoder', + 'com.ning.compress.lzf.util.ChunkDecoderFactory', + 'com.ning.compress.lzf.util.ChunkEncoderFactory', + 'lzma.sdk.lzma.Encoder', + 'net.jpountz.lz4.LZ4Compressor', + 'net.jpountz.lz4.LZ4Factory', + 'net.jpountz.lz4.LZ4FastDecompressor', + 'net.jpountz.xxhash.XXHash32', + 'net.jpountz.xxhash.XXHashFactory', + 'org.eclipse.jetty.alpn.ALPN$ClientProvider', + 'org.eclipse.jetty.alpn.ALPN$ServerProvider', + 'org.eclipse.jetty.alpn.ALPN', + + 'org.conscrypt.AllocatedBuffer', + 'org.conscrypt.BufferAllocator', + 'org.conscrypt.Conscrypt', + 'org.conscrypt.HandshakeListener', + + 'reactor.blockhound.BlockHound$Builder', + 'reactor.blockhound.integration.BlockHoundIntegration' + ) + ignoreViolations( + // Guava internal classes + 'com.google.common.cache.Striped64', + 'com.google.common.cache.Striped64$1', + 'com.google.common.cache.Striped64$Cell', + 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray', + 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1', + 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2', + 'com.google.common.hash.Striped64', + 'com.google.common.hash.Striped64$1', + 'com.google.common.hash.Striped64$Cell', + 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', + 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1', + 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper', + 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1', + + 'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator', + 'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$1', + 'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$2', + 'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$3', + 'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$4', + 'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$5', + 'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0', + 'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$1', + 'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$2', + 'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$3', + 'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$4', + 'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$6', + 'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef', + 'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef', + 'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields', + 'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields', + 'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields', + 'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode', + 'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField', + 'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField', + 'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField', + 'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField', + 'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField', + 'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess', + 'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess', + 'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess' + ) +} diff --git a/plugins/arrow-flight-rpc/licenses/failureaccess-1.0.1.jar.sha1 b/plugins/arrow-flight-rpc/licenses/failureaccess-1.0.1.jar.sha1 new file mode 100644 index 0000000000000..4798b37e20691 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/failureaccess-1.0.1.jar.sha1 @@ -0,0 +1 @@ +1dcf1de382a0bf95a3d8b0849546c88bac1292c9 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/failureaccess-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/failureaccess-LICENSE.txt new file mode 100644 index 0000000000000..7a4a3ea2424c0 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/failureaccess-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/failureaccess-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/failureaccess-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/arrow-flight-rpc/licenses/grpc-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/grpc-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/grpc-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/arrow-flight-rpc/licenses/grpc-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/grpc-NOTICE.txt new file mode 100644 index 0000000000000..f70c5620cf75a --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/grpc-NOTICE.txt @@ -0,0 +1,62 @@ +Copyright 2014 The gRPC Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +----------------------------------------------------------------------- + +This product contains a modified portion of 'OkHttp', an open source +HTTP & SPDY client for Android and Java applications, which can be obtained +at: + + * LICENSE: + * okhttp/third_party/okhttp/LICENSE (Apache License 2.0) + * HOMEPAGE: + * https://github.com/square/okhttp + * LOCATION_IN_GRPC: + * okhttp/third_party/okhttp + +This product contains a modified portion of 'Envoy', an open source +cloud-native high-performance edge/middle/service proxy, which can be +obtained at: + + * LICENSE: + * xds/third_party/envoy/LICENSE (Apache License 2.0) + * NOTICE: + * xds/third_party/envoy/NOTICE + * HOMEPAGE: + * https://www.envoyproxy.io + * LOCATION_IN_GRPC: + * xds/third_party/envoy + +This product contains a modified portion of 'protoc-gen-validate (PGV)', +an open source protoc plugin to generate polyglot message validators, +which can be obtained at: + + * LICENSE: + * xds/third_party/protoc-gen-validate/LICENSE (Apache License 2.0) + * NOTICE: + * xds/third_party/protoc-gen-validate/NOTICE + * HOMEPAGE: + * https://github.com/envoyproxy/protoc-gen-validate + * LOCATION_IN_GRPC: + * xds/third_party/protoc-gen-validate + +This product contains a modified portion of 'udpa', +an open source universal data plane API, which can be obtained at: + + * LICENSE: + * xds/third_party/udpa/LICENSE (Apache License 2.0) + * HOMEPAGE: + * https://github.com/cncf/udpa + * LOCATION_IN_GRPC: + * xds/third_party/udpa diff --git a/plugins/arrow-flight-rpc/licenses/grpc-api-1.68.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/grpc-api-1.68.2.jar.sha1 new file mode 100644 index 0000000000000..1844172dec982 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/grpc-api-1.68.2.jar.sha1 @@ -0,0 +1 @@ +a257a5dd25dda1c97a99b56d5b9c1e56c12ae554 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/grpc-core-1.68.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/grpc-core-1.68.2.jar.sha1 new file mode 100644 index 0000000000000..e20345d29e914 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/grpc-core-1.68.2.jar.sha1 @@ -0,0 +1 @@ +b0fd51a1c029785d1c9ae2cfc80a296b60dfcfdb \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/grpc-netty-shaded-1.68.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/grpc-netty-shaded-1.68.2.jar.sha1 new file mode 100644 index 0000000000000..53fa705a66129 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/grpc-netty-shaded-1.68.2.jar.sha1 @@ -0,0 +1 @@ +8ea4186fbdcc5432664364ed53e03cf0d458c3ec \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/grpc-protobuf-1.68.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/grpc-protobuf-1.68.2.jar.sha1 new file mode 100644 index 0000000000000..e861b41837f33 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/grpc-protobuf-1.68.2.jar.sha1 @@ -0,0 +1 @@ +35b28e0d57874021cd31e76dd4a795f76a82471e \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/grpc-protobuf-lite-1.68.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/grpc-protobuf-lite-1.68.2.jar.sha1 new file mode 100644 index 0000000000000..b2401f9752829 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/grpc-protobuf-lite-1.68.2.jar.sha1 @@ -0,0 +1 @@ +a53064b896adcfefe74362a33e111492351dfc03 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/grpc-stub-1.68.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/grpc-stub-1.68.2.jar.sha1 new file mode 100644 index 0000000000000..118464f8f48ff --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/grpc-stub-1.68.2.jar.sha1 @@ -0,0 +1 @@ +d58ee1cf723b4b5536d44b67e328c163580a8d98 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/guava-33.3.1-jre.jar.sha1 b/plugins/arrow-flight-rpc/licenses/guava-33.3.1-jre.jar.sha1 new file mode 100644 index 0000000000000..ce59350c0d430 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/guava-33.3.1-jre.jar.sha1 @@ -0,0 +1 @@ +852f8b363da0111e819460021ca693cacca3e8db \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/guava-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/guava-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/guava-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/arrow-flight-rpc/licenses/guava-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/guava-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/libs/arrow-spi/licenses/jackson-databind-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/jackson-LICENSE similarity index 100% rename from libs/arrow-spi/licenses/jackson-databind-LICENSE.txt rename to plugins/arrow-flight-rpc/licenses/jackson-LICENSE diff --git a/libs/arrow-spi/licenses/jackson-databind-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/jackson-NOTICE similarity index 100% rename from libs/arrow-spi/licenses/jackson-databind-NOTICE.txt rename to plugins/arrow-flight-rpc/licenses/jackson-NOTICE diff --git a/plugins/arrow-flight-rpc/licenses/jackson-annotations-2.18.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/jackson-annotations-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..a06e1d5f28425 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/jackson-annotations-2.18.2.jar.sha1 @@ -0,0 +1 @@ +985d77751ebc7fce5db115a986bc9aa82f973f4a \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/jackson-databind-2.18.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/jackson-databind-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..eedbfff66c705 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/jackson-databind-2.18.2.jar.sha1 @@ -0,0 +1 @@ +deef8697b92141fb6caf7aa86966cff4eec9b04f \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/jsr305-3.0.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/jsr305-3.0.2.jar.sha1 new file mode 100644 index 0000000000000..c5c92d87b9d6c --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/jsr305-3.0.2.jar.sha1 @@ -0,0 +1 @@ +25ea2e8b0c338a877313bd4672d3fe056ea78f0d \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/jsr305-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/jsr305-LICENSE.txt new file mode 100644 index 0000000000000..0cb8710c4b3e5 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/jsr305-LICENSE.txt @@ -0,0 +1,29 @@ +Copyright (c) 2007-2009, JSR305 expert group +All rights reserved. + +http://www.opensource.org/licenses/bsd-license.php + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the JSR305 expert group nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + diff --git a/plugins/arrow-flight-rpc/licenses/jsr305-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/jsr305-NOTICE.txt new file mode 100644 index 0000000000000..8d1c8b69c3fce --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/jsr305-NOTICE.txt @@ -0,0 +1 @@ + diff --git a/plugins/arrow-flight-rpc/licenses/parquet-arrow-1.13.1.jar.sha1 b/plugins/arrow-flight-rpc/licenses/parquet-arrow-1.13.1.jar.sha1 new file mode 100644 index 0000000000000..a1b89891ca8e1 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/parquet-arrow-1.13.1.jar.sha1 @@ -0,0 +1 @@ +9e59add52791af8b05c1aefe2a2f8865602c9368 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/parquet-arrow-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/parquet-arrow-LICENSE.txt new file mode 100644 index 0000000000000..b0065815a5e92 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/parquet-arrow-LICENSE.txt @@ -0,0 +1,218 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +-------------------------------------------------------------------------------- + +This product includes code from Apache Avro. + +Copyright: 2014 The Apache Software Foundation. +Home page: https://avro.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Daniel Lemire's JavaFastPFOR project. The +"Lemire" bit packing source code produced by parquet-generator is derived from +the JavaFastPFOR project. + +Copyright: 2013 Daniel Lemire +Home page: http://lemire.me/en/ +Project page: https://github.com/lemire/JavaFastPFOR +License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Apache Spark. + +* dev/merge_parquet_pr.py is based on Spark's dev/merge_spark_pr.py + +Copyright: 2014 The Apache Software Foundation. +Home page: https://spark.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Twitter's ElephantBird project. + +* parquet-hadoop's UnmaterializableRecordCounter.java includes code from + ElephantBird's LzoRecordReader.java + +Copyright: 2012-2014 Twitter +Home page: https://github.com/twitter/elephant-bird +License: http://www.apache.org/licenses/LICENSE-2.0 + diff --git a/plugins/arrow-flight-rpc/licenses/parquet-arrow-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/parquet-arrow-NOTICE.txt new file mode 100644 index 0000000000000..46300d6cd98fd --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/parquet-arrow-NOTICE.txt @@ -0,0 +1,94 @@ + +Apache Parquet Java +Copyright 2014-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +-------------------------------------------------------------------------------- + +This product includes parquet-tools, initially developed at ARRIS, Inc. with +the following copyright notice: + + Copyright 2013 ARRIS, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +This product includes parquet-protobuf, initially developed by Lukas Nalezenc +with the following copyright notice: + + Copyright 2013 Lukas Nalezenec. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +This product includes code from Apache Avro, which includes the following in +its NOTICE file: + + Apache Avro + Copyright 2010-2015 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + +-------------------------------------------------------------------------------- + +This project includes code from Kite, developed at Cloudera, Inc. with +the following copyright notice: + +| Copyright 2013 Cloudera Inc. +| +| Licensed under the Apache License, Version 2.0 (the "License"); +| you may not use this file except in compliance with the License. +| You may obtain a copy of the License at +| +| http://www.apache.org/licenses/LICENSE-2.0 +| +| Unless required by applicable law or agreed to in writing, software +| distributed under the License is distributed on an "AS IS" BASIS, +| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +| See the License for the specific language governing permissions and +| limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from Netflix, Inc. with the following copyright +notice: + +| Copyright 2016 Netflix, Inc. +| +| Licensed under the Apache License, Version 2.0 (the "License"); +| you may not use this file except in compliance with the License. +| You may obtain a copy of the License at +| +| http://www.apache.org/licenses/LICENSE-2.0 +| +| Unless required by applicable law or agreed to in writing, software +| distributed under the License is distributed on an "AS IS" BASIS, +| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +| See the License for the specific language governing permissions and +| limitations under the License. + diff --git a/plugins/arrow-flight-rpc/licenses/perfmark-api-0.27.0.jar.sha1 b/plugins/arrow-flight-rpc/licenses/perfmark-api-0.27.0.jar.sha1 new file mode 100644 index 0000000000000..c85ee41fd9bbd --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/perfmark-api-0.27.0.jar.sha1 @@ -0,0 +1 @@ +f86f575a41b091786a4b027cd9c0c1d2e3fc1c01 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/perfmark-api-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/perfmark-api-LICENSE.txt new file mode 100644 index 0000000000000..261eeb9e9f8b2 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/perfmark-api-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/arrow-flight-rpc/licenses/perfmark-api-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/perfmark-api-NOTICE.txt new file mode 100644 index 0000000000000..04fbb4e692e51 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/perfmark-api-NOTICE.txt @@ -0,0 +1,41 @@ + +Copyright 2019 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +----------------------------------------------------------------------- + +This product contains a modified portion of 'Catapult', an open source +Trace Event viewer for Chome, Linux, and Android applications, which can +be obtained at: + + * LICENSE: + * traceviewer/src/main/resources/io/perfmark/traceviewer/third_party/catapult/LICENSE (New BSD License) + * HOMEPAGE: + * https://github.com/catapult-project/catapult + +This product contains a modified portion of 'Polymer', a library for Web +Components, which can be obtained at: + * LICENSE: + * traceviewer/src/main/resources/io/perfmark/traceviewer/third_party/polymer/LICENSE (New BSD License) + * HOMEPAGE: + * https://github.com/Polymer/polymer + + +This product contains a modified portion of 'ASM', an open source +Java Bytecode library, which can be obtained at: + + * LICENSE: + * agent/src/main/resources/io/perfmark/agent/third_party/asm/LICENSE (BSD style License) + * HOMEPAGE: + * https://asm.ow2.io/ \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/arrow-flight-rpc/licenses/slf4j-api-1.7.36.jar.sha1 new file mode 100644 index 0000000000000..77b9917528382 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/slf4j-api-1.7.36.jar.sha1 @@ -0,0 +1 @@ +6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/slf4j-api-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/slf4j-api-LICENSE.txt new file mode 100644 index 0000000000000..1a3d053237bec --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/slf4j-api-LICENSE.txt @@ -0,0 +1,24 @@ +Copyright (c) 2004-2022 QOS.ch Sarl (Switzerland) +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + diff --git a/plugins/arrow-flight-rpc/licenses/slf4j-api-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/slf4j-api-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/arrow-flight-rpc/src/internalClusterTest/java/org/opensearch/arrow/flight/ArrowFlightServerIT.java b/plugins/arrow-flight-rpc/src/internalClusterTest/java/org/opensearch/arrow/flight/ArrowFlightServerIT.java new file mode 100644 index 0000000000000..64ec1431fb808 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/internalClusterTest/java/org/opensearch/arrow/flight/ArrowFlightServerIT.java @@ -0,0 +1,306 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight; + +import org.apache.arrow.flight.CallOptions; +import org.apache.arrow.flight.FlightDescriptor; +import org.apache.arrow.flight.FlightInfo; +import org.apache.arrow.flight.FlightRuntimeException; +import org.apache.arrow.flight.OSFlightClient; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.opensearch.arrow.flight.bootstrap.FlightClientManager; +import org.opensearch.arrow.flight.bootstrap.FlightService; +import org.opensearch.arrow.spi.StreamManager; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.arrow.spi.StreamReader; +import org.opensearch.arrow.spi.StreamTicket; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.junit.BeforeClass; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 3) +public class ArrowFlightServerIT extends OpenSearchIntegTestCase { + + @BeforeClass + public static void setupFeatureFlags() { + FeatureFlagSetter.set(FeatureFlags.ARROW_STREAMS_SETTING.getKey()); + } + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(FlightStreamPlugin.class); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + ensureGreen(); + Thread.sleep(1000); + } + + public void testArrowFlightEndpoint() { + for (DiscoveryNode node : getClusterState().nodes()) { + FlightService flightService = internalCluster().getInstance(FlightService.class, node.getName()); + FlightClientManager flightClientManager = flightService.getFlightClientManager(); + OSFlightClient flightClient = flightClientManager.getFlightClient(node.getId()).get(); + assertNotNull(flightClient); + flightClient.handshake(CallOptions.timeout(5000L, TimeUnit.MILLISECONDS)); + } + } + + public void testFlightStreamReader() throws Exception { + for (DiscoveryNode node : getClusterState().nodes()) { + StreamManager streamManagerRandomNode = getStreamManagerRandomNode(); + StreamTicket ticket = streamManagerRandomNode.registerStream(getStreamProducer(), null); + StreamManager streamManagerCurrentNode = getStreamManager(node.getName()); + // reader should be accessible from any node in the cluster due to the use ProxyStreamProducer + try (StreamReader reader = streamManagerCurrentNode.getStreamReader(ticket)) { + int totalBatches = 0; + assertNotNull(reader.getRoot().getVector("docID")); + while (reader.next()) { + IntVector docIDVector = (IntVector) reader.getRoot().getVector("docID"); + assertEquals(10, docIDVector.getValueCount()); + for (int i = 0; i < 10; i++) { + assertEquals(docIDVector.toString(), i + (totalBatches * 10L), docIDVector.get(i)); + } + totalBatches++; + } + assertEquals(10, totalBatches); + } + } + } + + public void testEarlyCancel() throws Exception { + DiscoveryNode previousNode = null; + for (DiscoveryNode node : getClusterState().nodes()) { + if (previousNode == null) { + previousNode = node; + continue; + } + StreamManager streamManagerServer = getStreamManager(node.getName()); + TestStreamProducer streamProducer = getStreamProducer(); + StreamTicket ticket = streamManagerServer.registerStream(streamProducer, null); + StreamManager streamManagerClient = getStreamManager(previousNode.getName()); + + CountDownLatch readerComplete = new CountDownLatch(1); + AtomicReference readerException = new AtomicReference<>(); + AtomicReference readerRef = new AtomicReference<>(); + + // Start reader thread + Thread readerThread = new Thread(() -> { + try { + StreamReader reader = streamManagerClient.getStreamReader(ticket); + readerRef.set(reader); + assertNotNull(reader.getRoot()); + IntVector docIDVector = (IntVector) reader.getRoot().getVector("docID"); + assertNotNull(docIDVector); + + // Read first batch + reader.next(); + assertEquals(10, docIDVector.getValueCount()); + for (int i = 0; i < 10; i++) { + assertEquals(docIDVector.toString(), i, docIDVector.get(i)); + } + reader.close(); + } catch (Exception e) { + readerException.set(e); + } finally { + readerComplete.countDown(); + } + }, "flight-reader-thread"); + + readerThread.start(); + assertTrue("Reader thread did not complete in time", readerComplete.await(1, TimeUnit.SECONDS)); + + if (readerException.get() != null) { + throw readerException.get(); + } + + StreamReader reader = readerRef.get(); + + try { + reader.next(); + fail("Expected FlightRuntimeException"); + } catch (FlightRuntimeException e) { + assertEquals("CANCELLED", e.status().code().name()); + assertEquals("Stream closed before end", e.getMessage()); + reader.close(); + } + + // Wait for close to complete + // Due to https://github.com/grpc/grpc-java/issues/5882, there is a logic in FlightStream.java + // where it exhausts the stream on the server side before it is actually cancelled. + assertTrue( + "Timeout waiting for stream cancellation on server [" + node.getName() + "]", + streamProducer.waitForClose(2, TimeUnit.SECONDS) + ); + previousNode = node; + } + } + + public void testFlightStreamServerError() throws Exception { + DiscoveryNode previousNode = null; + for (DiscoveryNode node : getClusterState().nodes()) { + if (previousNode == null) { + previousNode = node; + continue; + } + StreamManager streamManagerServer = getStreamManager(node.getName()); + TestStreamProducer streamProducer = getStreamProducer(); + streamProducer.setProduceError(true); + StreamTicket ticket = streamManagerServer.registerStream(streamProducer, null); + StreamManager streamManagerClient = getStreamManager(previousNode.getName()); + try (StreamReader reader = streamManagerClient.getStreamReader(ticket)) { + int totalBatches = 0; + assertNotNull(reader.getRoot().getVector("docID")); + try { + while (reader.next()) { + IntVector docIDVector = (IntVector) reader.getRoot().getVector("docID"); + assertEquals(10, docIDVector.getValueCount()); + totalBatches++; + } + fail("Expected FlightRuntimeException"); + } catch (FlightRuntimeException e) { + assertEquals("INTERNAL", e.status().code().name()); + assertEquals("There was an error servicing your request.", e.getMessage()); + } + assertEquals(1, totalBatches); + } + previousNode = node; + } + } + + public void testFlightGetInfo() throws Exception { + StreamTicket ticket = null; + for (DiscoveryNode node : getClusterState().nodes()) { + FlightService flightService = internalCluster().getInstance(FlightService.class, node.getName()); + StreamManager streamManager = flightService.getStreamManager(); + if (ticket == null) { + ticket = streamManager.registerStream(getStreamProducer(), null); + } + FlightClientManager flightClientManager = flightService.getFlightClientManager(); + OSFlightClient flightClient = flightClientManager.getFlightClient(node.getId()).get(); + assertNotNull(flightClient); + FlightDescriptor flightDescriptor = FlightDescriptor.command(ticket.toBytes()); + FlightInfo flightInfo = flightClient.getInfo(flightDescriptor, CallOptions.timeout(5000L, TimeUnit.MILLISECONDS)); + assertNotNull(flightInfo); + assertEquals(100, flightInfo.getRecords()); + } + } + + private StreamManager getStreamManager(String nodeName) { + FlightService flightService = internalCluster().getInstance(FlightService.class, nodeName); + return flightService.getStreamManager(); + } + + private StreamManager getStreamManagerRandomNode() { + FlightService flightService = internalCluster().getInstance(FlightService.class); + return flightService.getStreamManager(); + } + + private TestStreamProducer getStreamProducer() { + return new TestStreamProducer(); + } + + private static class TestStreamProducer implements StreamProducer { + volatile boolean isClosed = false; + private final CountDownLatch closeLatch = new CountDownLatch(1); + TimeValue deadline = TimeValue.timeValueSeconds(5); + private volatile boolean produceError = false; + + public void setProduceError(boolean produceError) { + this.produceError = produceError; + } + + TestStreamProducer() {} + + VectorSchemaRoot root; + + @Override + public VectorSchemaRoot createRoot(BufferAllocator allocator) { + IntVector docIDVector = new IntVector("docID", allocator); + FieldVector[] vectors = new FieldVector[] { docIDVector }; + root = new VectorSchemaRoot(Arrays.asList(vectors)); + return root; + } + + @Override + public BatchedJob createJob(BufferAllocator allocator) { + return new BatchedJob() { + @Override + public void run(VectorSchemaRoot root, FlushSignal flushSignal) { + IntVector docIDVector = (IntVector) root.getVector("docID"); + root.setRowCount(10); + for (int i = 0; i < 100; i++) { + docIDVector.setSafe(i % 10, i); + if ((i + 1) % 10 == 0) { + flushSignal.awaitConsumption(TimeValue.timeValueMillis(1000)); + docIDVector.clear(); + root.setRowCount(10); + if (produceError) { + throw new RuntimeException("Server error while producing batch"); + } + } + } + } + + @Override + public void onCancel() { + root.close(); + isClosed = true; + } + + @Override + public boolean isCancelled() { + return isClosed; + } + }; + } + + @Override + public TimeValue getJobDeadline() { + return deadline; + } + + @Override + public int estimatedRowCount() { + return 100; + } + + @Override + public String getAction() { + return ""; + } + + @Override + public void close() { + root.close(); + closeLatch.countDown(); + isClosed = true; + } + + public boolean waitForClose(long timeout, TimeUnit unit) throws InterruptedException { + return closeLatch.await(timeout, unit); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/OSFlightClient.java b/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/OSFlightClient.java new file mode 100644 index 0000000000000..039152a430de2 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/OSFlightClient.java @@ -0,0 +1,950 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.apache.arrow.flight; + +import org.apache.arrow.flight.FlightProducer.StreamListener; +import org.apache.arrow.flight.auth.BasicClientAuthHandler; +import org.apache.arrow.flight.auth.ClientAuthHandler; +import org.apache.arrow.flight.auth.ClientAuthInterceptor; +import org.apache.arrow.flight.auth.ClientAuthWrapper; +import org.apache.arrow.flight.auth2.BasicAuthCredentialWriter; +import org.apache.arrow.flight.auth2.ClientBearerHeaderHandler; +import org.apache.arrow.flight.auth2.ClientHandshakeWrapper; +import org.apache.arrow.flight.auth2.ClientIncomingAuthHeaderMiddleware; +import org.apache.arrow.flight.grpc.ClientInterceptorAdapter; +import org.apache.arrow.flight.grpc.CredentialCallOption; +import org.apache.arrow.flight.grpc.StatusUtils; +import org.apache.arrow.flight.impl.Flight; +import org.apache.arrow.flight.impl.Flight.Empty; +import org.apache.arrow.flight.impl.FlightServiceGrpc; +import org.apache.arrow.flight.impl.FlightServiceGrpc.FlightServiceBlockingStub; +import org.apache.arrow.flight.impl.FlightServiceGrpc.FlightServiceStub; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider; + +import javax.net.ssl.SSLException; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.net.URISyntaxException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; + +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ClientInterceptor; +import io.grpc.ClientInterceptors; +import io.grpc.ManagedChannel; +import io.grpc.MethodDescriptor; +import io.grpc.StatusRuntimeException; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; +import io.grpc.netty.shaded.io.netty.channel.EventLoopGroup; +import io.grpc.netty.shaded.io.netty.channel.ServerChannel; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder; +import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import io.grpc.stub.ClientCallStreamObserver; +import io.grpc.stub.ClientCalls; +import io.grpc.stub.ClientResponseObserver; +import io.grpc.stub.StreamObserver; + +/** + * Clone of {@link FlightClient} to support setting SslContext directly. It can be discarded once + * FlightClient.Builder supports setting SslContext directly. + */ +@SuppressWarnings("forbiddenApisMain") +public class OSFlightClient implements AutoCloseable { + private static final int PENDING_REQUESTS = 5; + /** + * The maximum number of trace events to keep on the gRPC Channel. This value disables channel + * tracing. + */ + private static final int MAX_CHANNEL_TRACE_EVENTS = 0; + + private final BufferAllocator allocator; + private final ManagedChannel channel; + + private final FlightServiceBlockingStub blockingStub; + private final FlightServiceStub asyncStub; + private final ClientAuthInterceptor authInterceptor = new ClientAuthInterceptor(); + private final MethodDescriptor doGetDescriptor; + private final MethodDescriptor doPutDescriptor; + private final MethodDescriptor doExchangeDescriptor; + private final List middleware; + + /** Create a Flight client from an allocator and a gRPC channel. */ + OSFlightClient(BufferAllocator incomingAllocator, ManagedChannel channel, List middleware) { + this.allocator = incomingAllocator.newChildAllocator("flight-client", 0, Long.MAX_VALUE); + this.channel = channel; + this.middleware = middleware; + + final ClientInterceptor[] interceptors; + interceptors = new ClientInterceptor[] { authInterceptor, new ClientInterceptorAdapter(middleware) }; + + // Create a channel with interceptors pre-applied for DoGet and DoPut + Channel interceptedChannel = ClientInterceptors.intercept(channel, interceptors); + + blockingStub = FlightServiceGrpc.newBlockingStub(interceptedChannel); + asyncStub = FlightServiceGrpc.newStub(interceptedChannel); + doGetDescriptor = FlightBindingService.getDoGetDescriptor(allocator); + doPutDescriptor = FlightBindingService.getDoPutDescriptor(allocator); + doExchangeDescriptor = FlightBindingService.getDoExchangeDescriptor(allocator); + } + + /** + * Get a list of available flights. + * + * @param criteria Criteria for selecting flights + * @param options RPC-layer hints for the call. + * @return FlightInfo Iterable + */ + public Iterable listFlights(Criteria criteria, CallOption... options) { + final Iterator flights; + try { + flights = CallOptions.wrapStub(blockingStub, options).listFlights(criteria.asCriteria()); + } catch (StatusRuntimeException sre) { + throw StatusUtils.fromGrpcRuntimeException(sre); + } + return () -> StatusUtils.wrapIterator(flights, t -> { + try { + return new FlightInfo(t); + } catch (URISyntaxException e) { + // We don't expect this will happen for conforming Flight implementations. For + // instance, a Java server + // itself wouldn't be able to construct an invalid Location. + throw new RuntimeException(e); + } + }); + } + + /** + * Lists actions available on the Flight service. + * + * @param options RPC-layer hints for the call. + */ + public Iterable listActions(CallOption... options) { + final Iterator actions; + try { + actions = CallOptions.wrapStub(blockingStub, options).listActions(Empty.getDefaultInstance()); + } catch (StatusRuntimeException sre) { + throw StatusUtils.fromGrpcRuntimeException(sre); + } + return () -> StatusUtils.wrapIterator(actions, ActionType::new); + } + + /** + * Performs an action on the Flight service. + * + * @param action The action to perform. + * @param options RPC-layer hints for this call. + * @return An iterator of results. + */ + public Iterator doAction(Action action, CallOption... options) { + return StatusUtils.wrapIterator(CallOptions.wrapStub(blockingStub, options).doAction(action.toProtocol()), Result::new); + } + + /** Authenticates with a username and password. */ + public void authenticateBasic(String username, String password) { + BasicClientAuthHandler basicClient = new BasicClientAuthHandler(username, password); + authenticate(basicClient); + } + + /** + * Authenticates against the Flight service. + * + * @param options RPC-layer hints for this call. + * @param handler The auth mechanism to use. + */ + public void authenticate(ClientAuthHandler handler, CallOption... options) { + Preconditions.checkArgument(!authInterceptor.hasAuthHandler(), "Auth already completed."); + ClientAuthWrapper.doClientAuth(handler, CallOptions.wrapStub(asyncStub, options)); + authInterceptor.setAuthHandler(handler); + } + + /** + * Authenticates with a username and password. + * + * @param username the username. + * @param password the password. + * @return a CredentialCallOption containing a bearer token if the server emitted one, or empty if + * no bearer token was returned. This can be used in subsequent API calls. + */ + public Optional authenticateBasicToken(String username, String password) { + final ClientIncomingAuthHeaderMiddleware.Factory clientAuthMiddleware = new ClientIncomingAuthHeaderMiddleware.Factory( + new ClientBearerHeaderHandler() + ); + middleware.add(clientAuthMiddleware); + handshake(new CredentialCallOption(new BasicAuthCredentialWriter(username, password))); + + return Optional.ofNullable(clientAuthMiddleware.getCredentialCallOption()); + } + + /** + * Executes the handshake against the Flight service. + * + * @param options RPC-layer hints for this call. + */ + public void handshake(CallOption... options) { + ClientHandshakeWrapper.doClientHandshake(CallOptions.wrapStub(asyncStub, options)); + } + + /** + * Create or append a descriptor with another stream. + * + * @param descriptor FlightDescriptor the descriptor for the data + * @param root VectorSchemaRoot the root containing data + * @param metadataListener A handler for metadata messages from the server. This will be passed + * buffers that will be freed after {@link StreamListener#onNext(Object)} is called! + * @param options RPC-layer hints for this call. + * @return ClientStreamListener an interface to control uploading data + */ + public ClientStreamListener startPut( + FlightDescriptor descriptor, + VectorSchemaRoot root, + PutListener metadataListener, + CallOption... options + ) { + return startPut(descriptor, root, new MapDictionaryProvider(), metadataListener, options); + } + + /** + * Create or append a descriptor with another stream. + * + * @param descriptor FlightDescriptor the descriptor for the data + * @param root VectorSchemaRoot the root containing data + * @param metadataListener A handler for metadata messages from the server. + * @param options RPC-layer hints for this call. + * @return ClientStreamListener an interface to control uploading data. {@link + * ClientStreamListener#start(VectorSchemaRoot, DictionaryProvider)} will already have been + * called. + */ + public ClientStreamListener startPut( + FlightDescriptor descriptor, + VectorSchemaRoot root, + DictionaryProvider provider, + PutListener metadataListener, + CallOption... options + ) { + Preconditions.checkNotNull(root, "root must not be null"); + Preconditions.checkNotNull(provider, "provider must not be null"); + final ClientStreamListener writer = startPut(descriptor, metadataListener, options); + writer.start(root, provider); + return writer; + } + + /** + * Create or append a descriptor with another stream. + * + * @param descriptor FlightDescriptor the descriptor for the data + * @param metadataListener A handler for metadata messages from the server. + * @param options RPC-layer hints for this call. + * @return ClientStreamListener an interface to control uploading data. {@link + * ClientStreamListener#start(VectorSchemaRoot, DictionaryProvider)} will NOT already have + * been called. + */ + public ClientStreamListener startPut(FlightDescriptor descriptor, PutListener metadataListener, CallOption... options) { + Preconditions.checkNotNull(descriptor, "descriptor must not be null"); + Preconditions.checkNotNull(metadataListener, "metadataListener must not be null"); + + try { + final ClientCall call = asyncStubNewCall(doPutDescriptor, options); + final SetStreamObserver resultObserver = new SetStreamObserver(allocator, metadataListener); + ClientCallStreamObserver observer = (ClientCallStreamObserver) ClientCalls.asyncBidiStreamingCall( + call, + resultObserver + ); + return new PutObserver(descriptor, observer, metadataListener::isCancelled, metadataListener::getResult); + } catch (StatusRuntimeException sre) { + throw StatusUtils.fromGrpcRuntimeException(sre); + } + } + + /** + * Get info on a stream. + * + * @param descriptor The descriptor for the stream. + * @param options RPC-layer hints for this call. + */ + public FlightInfo getInfo(FlightDescriptor descriptor, CallOption... options) { + try { + return new FlightInfo(CallOptions.wrapStub(blockingStub, options).getFlightInfo(descriptor.toProtocol())); + } catch (URISyntaxException e) { + // We don't expect this will happen for conforming Flight implementations. For instance, a + // Java server + // itself wouldn't be able to construct an invalid Location. + throw new RuntimeException(e); + } catch (StatusRuntimeException sre) { + throw StatusUtils.fromGrpcRuntimeException(sre); + } + } + + /** + * Start or get info on execution of a long-running query. + * + * @param descriptor The descriptor for the stream. + * @param options RPC-layer hints for this call. + * @return Metadata about execution. + */ + public PollInfo pollInfo(FlightDescriptor descriptor, CallOption... options) { + try { + return new PollInfo(CallOptions.wrapStub(blockingStub, options).pollFlightInfo(descriptor.toProtocol())); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } catch (StatusRuntimeException sre) { + throw StatusUtils.fromGrpcRuntimeException(sre); + } + } + + /** + * Get schema for a stream. + * + * @param descriptor The descriptor for the stream. + * @param options RPC-layer hints for this call. + */ + public SchemaResult getSchema(FlightDescriptor descriptor, CallOption... options) { + try { + return SchemaResult.fromProtocol(CallOptions.wrapStub(blockingStub, options).getSchema(descriptor.toProtocol())); + } catch (StatusRuntimeException sre) { + throw StatusUtils.fromGrpcRuntimeException(sre); + } + } + + /** + * Retrieve a stream from the server. + * + * @param ticket The ticket granting access to the data stream. + * @param options RPC-layer hints for this call. + */ + public FlightStream getStream(Ticket ticket, CallOption... options) { + final ClientCall call = asyncStubNewCall(doGetDescriptor, options); + FlightStream stream = new FlightStream( + allocator, + PENDING_REQUESTS, + (String message, Throwable cause) -> call.cancel(message, cause), + (count) -> call.request(count) + ); + + final StreamObserver delegate = stream.asObserver(); + ClientResponseObserver clientResponseObserver = new ClientResponseObserver< + Flight.Ticket, + ArrowMessage>() { + + @Override + public void beforeStart(ClientCallStreamObserver requestStream) { + requestStream.disableAutoInboundFlowControl(); + } + + @Override + public void onNext(ArrowMessage value) { + delegate.onNext(value); + } + + @Override + public void onError(Throwable t) { + delegate.onError(StatusUtils.toGrpcException(t)); + } + + @Override + public void onCompleted() { + delegate.onCompleted(); + } + }; + + ClientCalls.asyncServerStreamingCall(call, ticket.toProtocol(), clientResponseObserver); + return stream; + } + + /** + * Initiate a bidirectional data exchange with the server. + * + * @param descriptor A descriptor for the data stream. + * @param options RPC call options. + * @return A pair of a readable stream and a writable stream. + */ + public ExchangeReaderWriter doExchange(FlightDescriptor descriptor, CallOption... options) { + Preconditions.checkNotNull(descriptor, "descriptor must not be null"); + + try { + final ClientCall call = asyncStubNewCall(doExchangeDescriptor, options); + final FlightStream stream = new FlightStream(allocator, PENDING_REQUESTS, call::cancel, call::request); + final ClientCallStreamObserver observer = (ClientCallStreamObserver) ClientCalls + .asyncBidiStreamingCall(call, stream.asObserver()); + final ClientStreamListener writer = new PutObserver(descriptor, observer, stream.cancelled::isDone, () -> { + try { + stream.completed.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw CallStatus.INTERNAL.withDescription("Client error: interrupted while completing call") + .withCause(e) + .toRuntimeException(); + } catch (ExecutionException e) { + throw CallStatus.INTERNAL.withDescription("Client error: internal while completing call") + .withCause(e) + .toRuntimeException(); + } + }); + // Send the descriptor to start. + try (final ArrowMessage message = new ArrowMessage(descriptor.toProtocol())) { + observer.onNext(message); + } catch (Exception e) { + throw CallStatus.INTERNAL.withCause(e).withDescription("Could not write descriptor " + descriptor).toRuntimeException(); + } + return new ExchangeReaderWriter(stream, writer); + } catch (StatusRuntimeException sre) { + throw StatusUtils.fromGrpcRuntimeException(sre); + } + } + + /** A pair of a reader and a writer for a DoExchange call. */ + public static class ExchangeReaderWriter implements AutoCloseable { + private final FlightStream reader; + private final ClientStreamListener writer; + + ExchangeReaderWriter(FlightStream reader, ClientStreamListener writer) { + this.reader = reader; + this.writer = writer; + } + + /** Get the reader for the call. */ + public FlightStream getReader() { + return reader; + } + + /** Get the writer for the call. */ + public ClientStreamListener getWriter() { + return writer; + } + + /** + * Make sure stream is drained. You must call this to be notified of any errors that may have + * happened after the exchange is complete. This should be called after + * `getWriter().completed()` and instead of `getWriter().getResult()`. + */ + public void getResult() { + // After exchange is complete, make sure stream is drained to propagate errors through reader + while (reader.next()) { + } + } + + /** Shut down the streams in this call. */ + @Override + public void close() throws Exception { + reader.close(); + } + } + + /** A stream observer for Flight.PutResult. */ + private static class SetStreamObserver implements StreamObserver { + private final BufferAllocator allocator; + private final StreamListener listener; + + SetStreamObserver(BufferAllocator allocator, StreamListener listener) { + super(); + this.allocator = allocator; + this.listener = listener == null ? NoOpStreamListener.getInstance() : listener; + } + + @Override + public void onNext(Flight.PutResult value) { + try (final PutResult message = PutResult.fromProtocol(allocator, value)) { + listener.onNext(message); + } + } + + @Override + public void onError(Throwable t) { + listener.onError(StatusUtils.fromThrowable(t)); + } + + @Override + public void onCompleted() { + listener.onCompleted(); + } + } + + /** The implementation of a {@link ClientStreamListener} for writing data to a Flight server. */ + static class PutObserver extends OutboundStreamListenerImpl implements ClientStreamListener { + private final BooleanSupplier isCancelled; + private final Runnable getResult; + + /** + * Create a new client stream listener. + * + * @param descriptor The descriptor for the stream. + * @param observer The write-side gRPC StreamObserver. + * @param isCancelled A flag to check if the call has been cancelled. + * @param getResult A flag that blocks until the overall call completes. + */ + PutObserver( + FlightDescriptor descriptor, + ClientCallStreamObserver observer, + BooleanSupplier isCancelled, + Runnable getResult + ) { + super(descriptor, observer); + Preconditions.checkNotNull(descriptor, "descriptor must be provided"); + Preconditions.checkNotNull(isCancelled, "isCancelled must be provided"); + Preconditions.checkNotNull(getResult, "getResult must be provided"); + this.isCancelled = isCancelled; + this.getResult = getResult; + this.unloader = null; + } + + @Override + protected void waitUntilStreamReady() { + // Check isCancelled as well to avoid inadvertently blocking forever + // (so long as PutListener properly implements it) + while (!responseObserver.isReady() && !isCancelled.getAsBoolean()) { + /* busy wait */ + } + } + + @Override + public void getResult() { + getResult.run(); + } + } + + /** + * Cancel execution of a distributed query. + * + * @param request The query to cancel. + * @param options Call options. + * @return The server response. + */ + public CancelFlightInfoResult cancelFlightInfo(CancelFlightInfoRequest request, CallOption... options) { + Action action = new Action(FlightConstants.CANCEL_FLIGHT_INFO.getType(), request.serialize().array()); + Iterator results = doAction(action, options); + if (!results.hasNext()) { + throw CallStatus.INTERNAL.withDescription("Server did not return a response").toRuntimeException(); + } + + CancelFlightInfoResult result; + try { + result = CancelFlightInfoResult.deserialize(ByteBuffer.wrap(results.next().getBody())); + } catch (IOException e) { + throw CallStatus.INTERNAL.withDescription("Failed to parse server response: " + e).withCause(e).toRuntimeException(); + } + results.forEachRemaining((ignored) -> {}); + return result; + } + + /** + * Request the server to extend the lifetime of a query result set. + * + * @param request The result set partition. + * @param options Call options. + * @return The new endpoint with an updated expiration time. + */ + public FlightEndpoint renewFlightEndpoint(RenewFlightEndpointRequest request, CallOption... options) { + Action action = new Action(FlightConstants.RENEW_FLIGHT_ENDPOINT.getType(), request.serialize().array()); + Iterator results = doAction(action, options); + if (!results.hasNext()) { + throw CallStatus.INTERNAL.withDescription("Server did not return a response").toRuntimeException(); + } + + FlightEndpoint result; + try { + result = FlightEndpoint.deserialize(ByteBuffer.wrap(results.next().getBody())); + } catch (IOException | URISyntaxException e) { + throw CallStatus.INTERNAL.withDescription("Failed to parse server response: " + e).withCause(e).toRuntimeException(); + } + results.forEachRemaining((ignored) -> {}); + return result; + } + + /** + * Set server session option(s) by name/value. + * + *

Sessions are generally persisted via HTTP cookies. + * + * @param request The session options to set on the server. + * @param options Call options. + * @return The result containing per-value error statuses, if any. + */ + public SetSessionOptionsResult setSessionOptions(SetSessionOptionsRequest request, CallOption... options) { + Action action = new Action(FlightConstants.SET_SESSION_OPTIONS.getType(), request.serialize().array()); + Iterator results = doAction(action, options); + if (!results.hasNext()) { + throw CallStatus.INTERNAL.withDescription("Server did not return a response").toRuntimeException(); + } + + SetSessionOptionsResult result; + try { + result = SetSessionOptionsResult.deserialize(ByteBuffer.wrap(results.next().getBody())); + } catch (IOException e) { + throw CallStatus.INTERNAL.withDescription("Failed to parse server response: " + e).withCause(e).toRuntimeException(); + } + results.forEachRemaining((ignored) -> {}); + return result; + } + + /** + * Get the current server session options. + * + *

The session is generally accessed via an HTTP cookie. + * + * @param request The (empty) GetSessionOptionsRequest. + * @param options Call options. + * @return The result containing the set of session options configured on the server. + */ + public GetSessionOptionsResult getSessionOptions(GetSessionOptionsRequest request, CallOption... options) { + Action action = new Action(FlightConstants.GET_SESSION_OPTIONS.getType(), request.serialize().array()); + Iterator results = doAction(action, options); + if (!results.hasNext()) { + throw CallStatus.INTERNAL.withDescription("Server did not return a response").toRuntimeException(); + } + + GetSessionOptionsResult result; + try { + result = GetSessionOptionsResult.deserialize(ByteBuffer.wrap(results.next().getBody())); + } catch (IOException e) { + throw CallStatus.INTERNAL.withDescription("Failed to parse server response: " + e).withCause(e).toRuntimeException(); + } + results.forEachRemaining((ignored) -> {}); + return result; + } + + /** + * Close/invalidate the current server session. + * + *

The session is generally accessed via an HTTP cookie. + * + * @param request The (empty) CloseSessionRequest. + * @param options Call options. + * @return The result containing the status of the close operation. + */ + public CloseSessionResult closeSession(CloseSessionRequest request, CallOption... options) { + Action action = new Action(FlightConstants.CLOSE_SESSION.getType(), request.serialize().array()); + Iterator results = doAction(action, options); + if (!results.hasNext()) { + throw CallStatus.INTERNAL.withDescription("Server did not return a response").toRuntimeException(); + } + + CloseSessionResult result; + try { + result = CloseSessionResult.deserialize(ByteBuffer.wrap(results.next().getBody())); + } catch (IOException e) { + throw CallStatus.INTERNAL.withDescription("Failed to parse server response: " + e).withCause(e).toRuntimeException(); + } + results.forEachRemaining((ignored) -> {}); + return result; + } + + /** Interface for writers to an Arrow data stream. */ + public interface ClientStreamListener extends OutboundStreamListener { + + /** + * Wait for the stream to finish on the server side. You must call this to be notified of any + * errors that may have happened during the upload. + */ + void getResult(); + } + + /** + * A handler for server-sent application metadata messages during a Flight DoPut operation. + * + *

Generally, instead of implementing this yourself, you should use {@link AsyncPutListener} or + * {@link SyncPutListener}. + */ + public interface PutListener extends StreamListener { + + /** + * Wait for the stream to finish on the server side. You must call this to be notified of any + * errors that may have happened during the upload. + */ + void getResult(); + + /** + * Called when a message from the server is received. + * + * @param val The application metadata. This buffer will be reclaimed once onNext returns; you + * must retain a reference to use it outside this method. + */ + @Override + void onNext(PutResult val); + + /** + * Check if the call has been cancelled. + * + *

By default, this always returns false. Implementations should provide an appropriate + * implementation, as otherwise, a DoPut operation may inadvertently block forever. + */ + default boolean isCancelled() { + return false; + } + } + + /** Shut down this client. */ + @Override + public void close() throws InterruptedException { + channel.shutdown().awaitTermination(5, TimeUnit.SECONDS); + allocator.close(); + } + + /** Create a builder for a Flight client. */ + public static Builder builder() { + return new Builder(); + } + + /** + * Create a builder for a Flight client. + * + * @param allocator The allocator to use for the client. + * @param location The location to connect to. + */ + public static Builder builder(BufferAllocator allocator, Location location) { + return new Builder(allocator, location); + } + + public static Builder builder( + BufferAllocator allocator, + Location location, + Class channelType, + ExecutorService executorService, + EventLoopGroup workerELG, + SslContext sslContext + ) { + Builder builder = new Builder(allocator, location); + builder.channelType(channelType); + builder.executor(executorService); + builder.eventLoopGroup(workerELG); + if (sslContext != null) { + builder.useTLS(sslContext); + } + return builder; + } + + /** A builder for Flight clients. */ + public static final class Builder { + private BufferAllocator allocator; + private Location location; + private boolean forceTls = false; + private int maxInboundMessageSize = FlightServer.MAX_GRPC_MESSAGE_SIZE; + private InputStream trustedCertificates = null; + private InputStream clientCertificate = null; + private InputStream clientKey = null; + private String overrideHostname = null; + private List middleware = new ArrayList<>(); + private boolean verifyServer = true; + private EventLoopGroup workerELG; + private ExecutorService executorService; + private Class channelType; + private SslContext sslContext; + + private Builder() {} + + private Builder(BufferAllocator allocator, Location location) { + this.allocator = Preconditions.checkNotNull(allocator); + this.location = Preconditions.checkNotNull(location); + } + + /** Force the client to connect over TLS. */ + public Builder useTls() { + this.forceTls = true; + return this; + } + + /** Override the hostname checked for TLS. Use with caution in production. */ + public Builder overrideHostname(final String hostname) { + this.overrideHostname = hostname; + return this; + } + + /** Set the maximum inbound message size. */ + public Builder maxInboundMessageSize(int maxSize) { + Preconditions.checkArgument(maxSize > 0); + this.maxInboundMessageSize = maxSize; + return this; + } + + /** Set the trusted TLS certificates. */ + public Builder trustedCertificates(final InputStream stream) { + this.trustedCertificates = Preconditions.checkNotNull(stream); + return this; + } + + /** Set the trusted TLS certificates. */ + public Builder clientCertificate(final InputStream clientCertificate, final InputStream clientKey) { + Preconditions.checkNotNull(clientKey); + this.clientCertificate = Preconditions.checkNotNull(clientCertificate); + this.clientKey = Preconditions.checkNotNull(clientKey); + return this; + } + + public Builder allocator(BufferAllocator allocator) { + this.allocator = Preconditions.checkNotNull(allocator); + return this; + } + + public Builder location(Location location) { + this.location = Preconditions.checkNotNull(location); + return this; + } + + public Builder intercept(FlightClientMiddleware.Factory factory) { + middleware.add(factory); + return this; + } + + public Builder verifyServer(boolean verifyServer) { + this.verifyServer = verifyServer; + return this; + } + + public Builder eventLoopGroup(EventLoopGroup elg) { + this.workerELG = elg; + return this; + } + + public Builder executor(ExecutorService executorService) { + this.executorService = executorService; + return this; + } + + public Builder channelType(Class channelType) { + this.channelType = channelType; + return this; + } + + public Builder useTLS(SslContext sslContext) { + this.sslContext = Objects.requireNonNull(sslContext); + return this; + } + + /** Create the client from this builder. */ + public OSFlightClient build() { + final NettyChannelBuilder builder; + + switch (location.getUri().getScheme()) { + case LocationSchemes.GRPC: + case LocationSchemes.GRPC_INSECURE: + case LocationSchemes.GRPC_TLS: { + builder = NettyChannelBuilder.forAddress(location.toSocketAddress()); + if (workerELG != null) { + builder.eventLoopGroup(workerELG); + } + if (executorService != null) { + builder.executor(executorService); + } + if (channelType != null) { + builder.channelType(channelType); + } + if (sslContext != null) { + builder.sslContext(sslContext); + } + break; + } + case LocationSchemes.GRPC_DOMAIN_SOCKET: { + // The implementation is platform-specific, so we have to find the classes at runtime + builder = NettyChannelBuilder.forAddress(location.toSocketAddress()); + try { + try { + // Linux + builder.channelType( + Class.forName("io.grpc.netty.shaded.io.netty.channel.epoll.EpollDomainSocketChannel") + .asSubclass(ServerChannel.class) + ); + final EventLoopGroup elg = Class.forName("io.grpc.netty.shaded.io.netty.channel.epoll.EpollEventLoopGroup") + .asSubclass(EventLoopGroup.class) + .getDeclaredConstructor() + .newInstance(); + builder.eventLoopGroup(elg); + } catch (ClassNotFoundException e) { + // BSD + // this might not work as io.netty.channel.kqueue classes aren't present in grpc-netty-shaded + builder.channelType( + Class.forName("io.grpc.netty.shaded.io.netty.channel.kqueue.KQueueDomainSocketChannel") + .asSubclass(ServerChannel.class) + ); + final EventLoopGroup elg = Class.forName("io.grpc.netty.shaded.io.netty.channel.kqueue.KQueueEventLoopGroup") + .asSubclass(EventLoopGroup.class) + .getDeclaredConstructor() + .newInstance(); + builder.eventLoopGroup(elg); + } + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException + | InvocationTargetException e) { + throw new UnsupportedOperationException( + "Could not find suitable Netty native transport implementation for domain socket address." + ); + } + break; + } + default: + throw new IllegalArgumentException("Scheme is not supported: " + location.getUri().getScheme()); + } + + if (this.forceTls || LocationSchemes.GRPC_TLS.equals(location.getUri().getScheme())) { + builder.useTransportSecurity(); + + final boolean hasTrustedCerts = this.trustedCertificates != null; + final boolean hasKeyCertPair = this.clientCertificate != null && this.clientKey != null; + if (!this.verifyServer && (hasTrustedCerts || hasKeyCertPair)) { + throw new IllegalArgumentException( + "FlightClient has been configured to disable server verification, " + "but certificate options have been specified." + ); + } + + final SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient(); + + if (!this.verifyServer) { + sslContextBuilder.trustManager(InsecureTrustManagerFactory.INSTANCE); + } else if (this.trustedCertificates != null || this.clientCertificate != null || this.clientKey != null) { + if (this.trustedCertificates != null) { + sslContextBuilder.trustManager(this.trustedCertificates); + } + if (this.clientCertificate != null && this.clientKey != null) { + sslContextBuilder.keyManager(this.clientCertificate, this.clientKey); + } + } + try { + builder.sslContext(sslContextBuilder.build()); + } catch (SSLException e) { + throw new RuntimeException(e); + } + + if (this.overrideHostname != null) { + builder.overrideAuthority(this.overrideHostname); + } + } else { + builder.usePlaintext(); + } + + builder.maxTraceEvents(MAX_CHANNEL_TRACE_EVENTS) + .maxInboundMessageSize(maxInboundMessageSize) + .maxInboundMetadataSize(maxInboundMessageSize); + return new OSFlightClient(allocator, builder.build(), middleware); + } + } + + /** + * Helper method to create a call from the asyncStub, method descriptor, and list of calling + * options. + */ + private ClientCall asyncStubNewCall( + MethodDescriptor descriptor, + CallOption... options + ) { + FlightServiceStub wrappedStub = CallOptions.wrapStub(asyncStub, options); + return wrappedStub.getChannel().newCall(descriptor, wrappedStub.getCallOptions()); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/OSFlightServer.java b/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/OSFlightServer.java new file mode 100644 index 0000000000000..64f0dd7d5e193 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/OSFlightServer.java @@ -0,0 +1,558 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.apache.arrow.flight; + +import org.apache.arrow.flight.auth.ServerAuthHandler; +import org.apache.arrow.flight.auth.ServerAuthInterceptor; +import org.apache.arrow.flight.auth2.Auth2Constants; +import org.apache.arrow.flight.auth2.CallHeaderAuthenticator; +import org.apache.arrow.flight.auth2.ServerCallHeaderAuthMiddleware; +import org.apache.arrow.flight.grpc.ServerBackpressureThresholdInterceptor; +import org.apache.arrow.flight.grpc.ServerInterceptorAdapter; +import org.apache.arrow.flight.grpc.ServerInterceptorAdapter.KeyFactory; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.util.VisibleForTesting; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.net.ssl.SSLException; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import io.grpc.Server; +import io.grpc.ServerInterceptors; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import io.grpc.netty.shaded.io.netty.channel.Channel; +import io.grpc.netty.shaded.io.netty.channel.EventLoopGroup; +import io.grpc.netty.shaded.io.netty.channel.ServerChannel; +import io.grpc.netty.shaded.io.netty.handler.ssl.ClientAuth; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder; + +/** + * Clone of {@link FlightServer} to support setting SslContext directly. It can be discarded once + * FlightServer.Builder supports setting SslContext directly. + */ +public class OSFlightServer implements AutoCloseable { + + private static final Logger logger = LogManager.getLogger(OSFlightServer.class); + + private final Location location; + private final Server server; + // The executor used by the gRPC server. We don't use it here, but we do need to clean it up with + // the server. + // May be null, if a user-supplied executor was provided (as we do not want to clean that up) + @VisibleForTesting + final ExecutorService grpcExecutor; + + /** The maximum size of an individual gRPC message. This effectively disables the limit. */ + static final int MAX_GRPC_MESSAGE_SIZE = Integer.MAX_VALUE; + + /** The default number of bytes that can be queued on an output stream before blocking. */ + public static final int DEFAULT_BACKPRESSURE_THRESHOLD = 10 * 1024 * 1024; // 10MB + + /** Create a new instance from a gRPC server. For internal use only. */ + private OSFlightServer(Location location, Server server, ExecutorService grpcExecutor) { + this.location = location; + this.server = server; + this.grpcExecutor = grpcExecutor; + } + + /** Start the server. */ + public OSFlightServer start() throws IOException { + server.start(); + return this; + } + + /** Get the port the server is running on (if applicable). */ + public int getPort() { + return server.getPort(); + } + + /** Get the location for this server. */ + public Location getLocation() { + if (location.getUri().getPort() == 0) { + // If the server was bound to port 0, replace the port in the location with the real port. + final URI uri = location.getUri(); + try { + return new Location( + new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), getPort(), uri.getPath(), uri.getQuery(), uri.getFragment()) + ); + } catch (URISyntaxException e) { + // We don't expect this to happen + throw new RuntimeException(e); + } + } + return location; + } + + /** Block until the server shuts down. */ + public void awaitTermination() throws InterruptedException { + server.awaitTermination(); + } + + /** Request that the server shut down. */ + public void shutdown() { + server.shutdown(); + if (grpcExecutor != null) { + grpcExecutor.shutdown(); + } + } + + /** + * Wait for the server to shut down with a timeout. + * + * @return true if the server shut down successfully. + */ + public boolean awaitTermination(final long timeout, final TimeUnit unit) throws InterruptedException { + return server.awaitTermination(timeout, unit); + } + + /** Shutdown the server, waits for up to 6 seconds for successful shutdown before returning. */ + @Override + public void close() throws InterruptedException { + shutdown(); + final boolean terminated = awaitTermination(3000, TimeUnit.MILLISECONDS); + if (terminated) { + logger.debug("Server was terminated within 3s"); + return; + } + + // get more aggressive in termination. + server.shutdownNow(); + + int count = 0; + while (!server.isTerminated() && count < 30) { + count++; + logger.debug("Waiting for termination"); + Thread.sleep(100); + } + + if (!server.isTerminated()) { + logger.warn("Couldn't shutdown server, resources likely will be leaked."); + } + } + + /** Create a builder for a Flight server. */ + public static Builder builder() { + return new Builder(); + } + + /** Create a builder for a Flight server. */ + public static Builder builder(BufferAllocator allocator, Location location, FlightProducer producer) { + return new Builder(allocator, location, producer); + } + + public static Builder builder( + BufferAllocator allocator, + Location location, + FlightProducer producer, + SslContext sslContext, + Class channelType, + EventLoopGroup bossELG, + EventLoopGroup workerELG, + ExecutorService grpcExecutor + ) { + Builder builder = new Builder(allocator, location, producer); + if (sslContext != null) { + builder.useTls(sslContext); + } + builder.transportHint("netty.channelType", channelType); + builder.transportHint("netty.bossEventLoopGroup", bossELG); + builder.transportHint("netty.workerEventLoopGroup", workerELG); + builder.executor(grpcExecutor); + return builder; + } + + /** A builder for Flight servers. */ + public static final class Builder { + private BufferAllocator allocator; + private Location location; + private FlightProducer producer; + private final Map builderOptions; + private ServerAuthHandler authHandler = ServerAuthHandler.NO_OP; + private CallHeaderAuthenticator headerAuthenticator = CallHeaderAuthenticator.NO_OP; + private ExecutorService executor = null; + private int maxInboundMessageSize = MAX_GRPC_MESSAGE_SIZE; + private int maxHeaderListSize = MAX_GRPC_MESSAGE_SIZE; + private int backpressureThreshold = DEFAULT_BACKPRESSURE_THRESHOLD; + private InputStream certChain; + private InputStream key; + private InputStream mTlsCACert; + private SslContext sslContext; + private final List> interceptors; + // Keep track of inserted interceptors + private final Set interceptorKeys; + + Builder() { + builderOptions = new HashMap<>(); + interceptors = new ArrayList<>(); + interceptorKeys = new HashSet<>(); + } + + Builder(BufferAllocator allocator, Location location, FlightProducer producer) { + this(); + this.allocator = Preconditions.checkNotNull(allocator); + this.location = Preconditions.checkNotNull(location); + this.producer = Preconditions.checkNotNull(producer); + } + + /** Create the server for this builder. */ + @SuppressWarnings("unchecked") + public OSFlightServer build() { + // Add the auth middleware if applicable. + if (headerAuthenticator != CallHeaderAuthenticator.NO_OP) { + this.middleware( + FlightServerMiddleware.Key.of(Auth2Constants.AUTHORIZATION_HEADER), + new ServerCallHeaderAuthMiddleware.Factory(headerAuthenticator) + ); + } + + this.middleware(FlightConstants.HEADER_KEY, new ServerHeaderMiddleware.Factory()); + + final NettyServerBuilder builder; + switch (location.getUri().getScheme()) { + case LocationSchemes.GRPC_DOMAIN_SOCKET: { + // The implementation is platform-specific, so we have to find the classes at runtime + builder = NettyServerBuilder.forAddress(location.toSocketAddress()); + try { + try { + // Linux + builder.channelType( + Class.forName("io.grpc.netty.shaded.io.netty.channel.epoll.EpollServerDomainSocketChannel") + .asSubclass(ServerChannel.class) + ); + final EventLoopGroup elg = Class.forName("io.grpc.netty.shaded.io.netty.channel.epoll.EpollEventLoopGroup") + .asSubclass(EventLoopGroup.class) + .getConstructor() + .newInstance(); + builder.bossEventLoopGroup(elg).workerEventLoopGroup(elg); + } catch (ClassNotFoundException e) { + // BSD + // below logic may not work as the kqueue classes aren't present in grpc-netty-shaded + builder.channelType( + Class.forName("io.grpc.netty.shaded.io.netty.channel.kqueue.KQueueServerDomainSocketChannel") + .asSubclass(ServerChannel.class) + ); + final EventLoopGroup elg = Class.forName("io.grpc.netty.shaded.io.netty.channel.kqueue.KQueueEventLoopGroup") + .asSubclass(EventLoopGroup.class) + .getConstructor() + .newInstance(); + builder.bossEventLoopGroup(elg).workerEventLoopGroup(elg); + } + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException + | InvocationTargetException e) { + throw new UnsupportedOperationException( + "Could not find suitable Netty native transport implementation for domain socket address." + ); + } + break; + } + case LocationSchemes.GRPC: + case LocationSchemes.GRPC_INSECURE: { + builder = NettyServerBuilder.forAddress(location.toSocketAddress()); + break; + } + case LocationSchemes.GRPC_TLS: { + if (certChain == null) { + throw new IllegalArgumentException("Must provide a certificate and key to serve gRPC over TLS"); + } + builder = NettyServerBuilder.forAddress(location.toSocketAddress()); + break; + } + default: + throw new IllegalArgumentException("Scheme is not supported: " + location.getUri().getScheme()); + } + + if (sslContext != null && certChain != null) { + SslContextBuilder sslContextBuilder = GrpcSslContexts.forServer(certChain, key); + + if (mTlsCACert != null) { + sslContextBuilder.clientAuth(ClientAuth.REQUIRE).trustManager(mTlsCACert); + } + try { + sslContext = sslContextBuilder.build(); + } catch (SSLException e) { + throw new RuntimeException(e); + } finally { + closeMTlsCACert(); + closeCertChain(); + closeKey(); + } + + builder.sslContext(sslContext); + } + + // Share one executor between the gRPC service, DoPut, and Handshake + final ExecutorService exec; + // We only want to have FlightServer close the gRPC executor if we created it here. We should + // not close + // user-supplied executors. + final ExecutorService grpcExecutor; + if (executor != null) { + exec = executor; + grpcExecutor = null; + } else { + throw new IllegalStateException("GRPC executor must be passed to start Flight server."); + } + + final FlightBindingService flightService = new FlightBindingService(allocator, producer, authHandler, exec); + builder.executor(exec) + .maxInboundMessageSize(maxInboundMessageSize) + .maxInboundMetadataSize(maxHeaderListSize) + .addService( + ServerInterceptors.intercept( + flightService, + new ServerBackpressureThresholdInterceptor(backpressureThreshold), + new ServerAuthInterceptor(authHandler) + ) + ); + + // Allow hooking into the gRPC builder. This is not guaranteed to be available on all Arrow + // versions or + // Flight implementations. + builderOptions.computeIfPresent("grpc.builderConsumer", (key, builderConsumer) -> { + final Consumer consumer = (Consumer) builderConsumer; + consumer.accept(builder); + return null; + }); + + // Allow explicitly setting some Netty-specific options + builderOptions.computeIfPresent("netty.channelType", (key, channelType) -> { + builder.channelType((Class) channelType); + return null; + }); + builderOptions.computeIfPresent("netty.bossEventLoopGroup", (key, elg) -> { + builder.bossEventLoopGroup((EventLoopGroup) elg); + return null; + }); + builderOptions.computeIfPresent("netty.workerEventLoopGroup", (key, elg) -> { + builder.workerEventLoopGroup((EventLoopGroup) elg); + return null; + }); + + builder.intercept(new ServerInterceptorAdapter(interceptors)); + return new OSFlightServer(location, builder.build(), grpcExecutor); + } + + public Builder setMaxHeaderListSize(int maxHeaderListSize) { + this.maxHeaderListSize = maxHeaderListSize; + return this; + } + + /** + * Set the maximum size of a message. Defaults to "unlimited", depending on the underlying + * transport. + */ + public Builder maxInboundMessageSize(int maxMessageSize) { + this.maxInboundMessageSize = maxMessageSize; + return this; + } + + /** + * Set the number of bytes that may be queued on a server output stream before writes are + * blocked. + */ + public Builder backpressureThreshold(int backpressureThreshold) { + Preconditions.checkArgument(backpressureThreshold > 0); + this.backpressureThreshold = backpressureThreshold; + return this; + } + + /** + * A small utility function to ensure that InputStream attributes. are closed if they are not + * null + * + * @param stream The InputStream to close (if it is not null). + */ + private void closeInputStreamIfNotNull(InputStream stream) { + if (stream != null) { + try { + stream.close(); + } catch (IOException expected) { + // stream closes gracefully, doesn't expect an exception. + } + } + } + + /** + * A small utility function to ensure that the certChain attribute is closed if it is not null. + * It then sets the attribute to null. + */ + private void closeCertChain() { + closeInputStreamIfNotNull(certChain); + certChain = null; + } + + /** + * A small utility function to ensure that the key attribute is closed if it is not null. It + * then sets the attribute to null. + */ + private void closeKey() { + closeInputStreamIfNotNull(key); + key = null; + } + + /** + * A small utility function to ensure that the mTlsCACert attribute is closed if it is not null. + * It then sets the attribute to null. + */ + private void closeMTlsCACert() { + closeInputStreamIfNotNull(mTlsCACert); + mTlsCACert = null; + } + + /** + * Enable TLS on the server. + * + * @param certChain The certificate chain to use. + * @param key The private key to use. + */ + public Builder useTls(final File certChain, final File key) throws IOException { + closeCertChain(); + this.certChain = new FileInputStream(certChain); + + closeKey(); + this.key = new FileInputStream(key); + + return this; + } + + /** + * Enable Client Verification via mTLS on the server. + * + * @param mTlsCACert The CA certificate to use for verifying clients. + */ + public Builder useMTlsClientVerification(final File mTlsCACert) throws IOException { + closeMTlsCACert(); + this.mTlsCACert = new FileInputStream(mTlsCACert); + return this; + } + + /** + * Enable TLS on the server. + * + * @param certChain The certificate chain to use. + * @param key The private key to use. + */ + public Builder useTls(final InputStream certChain, final InputStream key) throws IOException { + closeCertChain(); + this.certChain = certChain; + + closeKey(); + this.key = key; + + return this; + } + + /** + * Enable TLS on the server. + * @param sslContext SslContext to use. + */ + public Builder useTls(SslContext sslContext) { + this.sslContext = Objects.requireNonNull(sslContext); + return this; + } + + /** + * Enable mTLS on the server. + * + * @param mTlsCACert The CA certificate to use for verifying clients. + */ + public Builder useMTlsClientVerification(final InputStream mTlsCACert) throws IOException { + closeMTlsCACert(); + this.mTlsCACert = mTlsCACert; + return this; + } + + /** + * Set the executor used by the server. + * + *

Flight will NOT take ownership of the executor. The application must clean it up if one is + * provided. (If not provided, Flight will use a default executor which it will clean up.) + */ + public Builder executor(ExecutorService executor) { + this.executor = executor; + return this; + } + + /** Set the authentication handler. */ + public Builder authHandler(ServerAuthHandler authHandler) { + this.authHandler = authHandler; + return this; + } + + /** Set the header-based authentication mechanism. */ + public Builder headerAuthenticator(CallHeaderAuthenticator headerAuthenticator) { + this.headerAuthenticator = headerAuthenticator; + return this; + } + + /** Provide a transport-specific option. Not guaranteed to have any effect. */ + public Builder transportHint(final String key, Object option) { + builderOptions.put(key, option); + return this; + } + + /** + * Add a Flight middleware component to inspect and modify requests to this service. + * + * @param key An identifier for this middleware component. Service implementations can retrieve + * the middleware instance for the current call using {@link + * org.apache.arrow.flight.FlightProducer.CallContext}. + * @param factory A factory for the middleware. + * @param The middleware type. + * @throws IllegalArgumentException if the key already exists + */ + public Builder middleware( + final FlightServerMiddleware.Key key, + final FlightServerMiddleware.Factory factory + ) { + if (interceptorKeys.contains(key.key)) { + throw new IllegalArgumentException("Key already exists: " + key.key); + } + interceptors.add(new KeyFactory<>(key, factory)); + interceptorKeys.add(key.key); + return this; + } + + public Builder allocator(BufferAllocator allocator) { + this.allocator = Preconditions.checkNotNull(allocator); + return this; + } + + public Builder location(Location location) { + this.location = Preconditions.checkNotNull(location); + return this; + } + + public Builder producer(FlightProducer producer) { + this.producer = Preconditions.checkNotNull(producer); + return this; + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/package-info.java b/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/package-info.java new file mode 100644 index 0000000000000..789a88a2d1159 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Clone of FlightServer and FlightClient due to package private access of + * certain configurations. + */ +package org.apache.arrow.flight; diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/BaseFlightStreamPlugin.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/BaseFlightStreamPlugin.java new file mode 100644 index 0000000000000..74d562370f310 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/BaseFlightStreamPlugin.java @@ -0,0 +1,190 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight; + +import org.opensearch.arrow.spi.StreamManager; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.IndexScopedSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsFilter; +import org.opensearch.common.util.PageCacheRecycler; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.indices.breaker.CircuitBreakerService; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; +import org.opensearch.plugins.ActionPlugin; +import org.opensearch.plugins.ClusterPlugin; +import org.opensearch.plugins.NetworkPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SecureTransportSettingsProvider; +import org.opensearch.plugins.StreamManagerPlugin; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.rest.RestController; +import org.opensearch.rest.RestHandler; +import org.opensearch.script.ScriptService; +import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.threadpool.ExecutorBuilder; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.Transport; +import org.opensearch.watcher.ResourceWatcherService; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +/** + * BaseFlightStreamPlugin is a plugin that implements the StreamManagerPlugin interface. + * It provides the necessary components for handling flight streams in the OpenSearch cluster. + */ +public abstract class BaseFlightStreamPlugin extends Plugin implements StreamManagerPlugin, NetworkPlugin, ActionPlugin, ClusterPlugin { + + /** + * Constructor for BaseFlightStreamPlugin. + */ + public BaseFlightStreamPlugin() { + super(); + } + + /** + * createComponents for BaseFlightStreamPlugin + * @param client The client instance. + * @param clusterService The cluster service instance. + * @param threadPool The thread pool instance. + * @param resourceWatcherService The resource watcher service instance. + * @param scriptService The script service instance. + * @param xContentRegistry The named XContent registry. + * @param environment The environment instance. + * @param nodeEnvironment The node environment instance. + * @param namedWriteableRegistry The named writeable registry. + * @param indexNameExpressionResolver The index name expression resolver instance. + * @param repositoriesServiceSupplier The supplier for the repositories service. + * @return A collection of components. + */ + @Override + public abstract Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ); + + /** + * Used to get the transports for SSL/TLS configuration of Flight server and clients + * @param settings The settings for the plugin + * @param threadPool The thread pool instance + * @param pageCacheRecycler The page cache recycler instance + * @param circuitBreakerService The circuit breaker service instance + * @param namedWriteableRegistry The named writeable registry + * @param networkService The network service instance + * @param secureTransportSettingsProvider The secure transport settings provider + * @param tracer The tracer instance + * @return A map of secure transports + */ + @Override + public abstract Map> getSecureTransports( + Settings settings, + ThreadPool threadPool, + PageCacheRecycler pageCacheRecycler, + CircuitBreakerService circuitBreakerService, + NamedWriteableRegistry namedWriteableRegistry, + NetworkService networkService, + SecureTransportSettingsProvider secureTransportSettingsProvider, + Tracer tracer + ); + + /** + * Used to get the auxiliary transports for Flight server and clients + * @param settings The settings for the plugin + * @param threadPool The thread pool instance + * @param circuitBreakerService The circuit breaker service instance + * @param networkService The network service instance + * @param clusterSettings The cluster settings instance + * @param tracer The tracer instance + * @return A map of auxiliary transports + */ + @Override + public abstract Map> getAuxTransports( + Settings settings, + ThreadPool threadPool, + CircuitBreakerService circuitBreakerService, + NetworkService networkService, + ClusterSettings clusterSettings, + Tracer tracer + ); + + /** + * Returns the StreamManager instance for managing flight streams. + */ + @Override + public abstract Supplier getStreamManager(); + + /** + * Returns a list of ExecutorBuilder instances for building thread pools used for FlightServer + * @param settings The settings for the plugin + */ + @Override + public abstract List> getExecutorBuilders(Settings settings); + + /** + * Returns a list of settings for the Flight plugin. + */ + @Override + public abstract List> getSettings(); + + /** + * Returns the REST handlers for the Flight plugin. + * @param settings The settings for the plugin. + * @param restController The REST controller instance. + * @param clusterSettings The cluster settings instance. + * @param indexScopedSettings The index scoped settings instance. + * @param settingsFilter The settings filter instance. + * @param indexNameExpressionResolver The index name expression resolver instance. + * @param nodesInCluster The supplier for the discovery nodes. + * @return A list of REST handlers. + */ + @Override + public abstract List getRestHandlers( + Settings settings, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster + ); + + /** + * Returns the list of action handlers for the Flight plugin. + */ + @Override + public abstract List> getActions(); + + /** + * Called when node is started. DiscoveryNode argument is passed to allow referring localNode value inside plugin + * + * @param localNode local Node info + */ + public abstract void onNodeStarted(DiscoveryNode localNode); +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/FlightStreamPlugin.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/FlightStreamPlugin.java new file mode 100644 index 0000000000000..4bde127b4325f --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/FlightStreamPlugin.java @@ -0,0 +1,327 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight; + +import org.opensearch.arrow.flight.bootstrap.FlightStreamPluginImpl; +import org.opensearch.arrow.spi.StreamManager; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.IndexScopedSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsFilter; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.util.PageCacheRecycler; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.indices.breaker.CircuitBreakerService; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; +import org.opensearch.plugins.SecureTransportSettingsProvider; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.rest.RestController; +import org.opensearch.rest.RestHandler; +import org.opensearch.script.ScriptService; +import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.threadpool.ExecutorBuilder; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.Transport; +import org.opensearch.watcher.ResourceWatcherService; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import static org.opensearch.common.util.FeatureFlags.ARROW_STREAMS_SETTING; + +/** + * Delegates the plugin implementation to the {@link FlightStreamPluginImpl} if the Arrow Streams feature flag is enabled. + * Otherwise, it creates a no-op implementation. + */ +@ExperimentalApi +public class FlightStreamPlugin extends BaseFlightStreamPlugin { + + private final BaseFlightStreamPlugin delegate; + + /** + * Constructor for FlightStreamPlugin. + * @param settings The settings for the plugin. + */ + public FlightStreamPlugin(Settings settings) { + if (FeatureFlags.isEnabled(ARROW_STREAMS_SETTING)) { + this.delegate = new FlightStreamPluginImpl(settings); + } else { + this.delegate = new BaseFlightStreamPlugin() { + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + return List.of(); + } + + @Override + public Map> getSecureTransports( + Settings settings, + ThreadPool threadPool, + PageCacheRecycler pageCacheRecycler, + CircuitBreakerService circuitBreakerService, + NamedWriteableRegistry namedWriteableRegistry, + NetworkService networkService, + SecureTransportSettingsProvider secureTransportSettingsProvider, + Tracer tracer + ) { + return Map.of(); + } + + @Override + public Map> getAuxTransports( + Settings settings, + ThreadPool threadPool, + CircuitBreakerService circuitBreakerService, + NetworkService networkService, + ClusterSettings clusterSettings, + Tracer tracer + ) { + return Map.of(); + } + + @Override + public Supplier getStreamManager() { + return () -> null; + } + + @Override + public List> getExecutorBuilders(Settings settings) { + return List.of(); + } + + @Override + public List> getSettings() { + return List.of(); + } + + @Override + public List getRestHandlers( + Settings settings, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster + ) { + return List.of(); + } + + @Override + public List> getActions() { + return List.of(); + } + + @Override + public void onNodeStarted(DiscoveryNode localNode) { + + } + }; + } + } + + /** + * Creates components related to the Flight stream functionality. + * @param client The OpenSearch client + * @param clusterService The cluster service + * @param threadPool The thread pool + * @param resourceWatcherService The resource watcher service + * @param scriptService The script service + * @param xContentRegistry The named XContent registry + * @param environment The environment + * @param nodeEnvironment The node environment + * @param namedWriteableRegistry The named writeable registry + * @param indexNameExpressionResolver The index name expression resolver + * @param repositoriesServiceSupplier The supplier for the repositories service + * @return A collection of components + */ + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + return delegate.createComponents( + client, + clusterService, + threadPool, + resourceWatcherService, + scriptService, + xContentRegistry, + environment, + nodeEnvironment, + namedWriteableRegistry, + indexNameExpressionResolver, + repositoriesServiceSupplier + ); + } + + /** + * Gets the secure transports for Flight stream functionality. + * @param settings The settings for the plugin + * @param threadPool The thread pool + * @param pageCacheRecycler The page cache recycler + * @param circuitBreakerService The circuit breaker service + * @param namedWriteableRegistry The named writeable registry + * @param networkService The network service + * @param secureTransportSettingsProvider The secure transport settings provider + * @param tracer The tracer + * @return A map of secure transports + */ + @Override + public Map> getSecureTransports( + Settings settings, + ThreadPool threadPool, + PageCacheRecycler pageCacheRecycler, + CircuitBreakerService circuitBreakerService, + NamedWriteableRegistry namedWriteableRegistry, + NetworkService networkService, + SecureTransportSettingsProvider secureTransportSettingsProvider, + Tracer tracer + ) { + return delegate.getSecureTransports( + settings, + threadPool, + pageCacheRecycler, + circuitBreakerService, + namedWriteableRegistry, + networkService, + secureTransportSettingsProvider, + tracer + ); + } + + /** + * Gets the auxiliary transports for Flight stream functionality. + * @param settings The settings for the plugin + * @param threadPool The thread pool + * @param circuitBreakerService The circuit breaker service + * @param networkService The network service + * @param clusterSettings The cluster settings + * @param tracer The tracer + * @return A map of auxiliary transports + */ + @Override + public Map> getAuxTransports( + Settings settings, + ThreadPool threadPool, + CircuitBreakerService circuitBreakerService, + NetworkService networkService, + ClusterSettings clusterSettings, + Tracer tracer + ) { + return delegate.getAuxTransports(settings, threadPool, circuitBreakerService, networkService, clusterSettings, tracer); + } + + /** + * Gets the REST handlers for the Flight plugin. + * @param settings The settings for the plugin. + * @param restController The REST controller instance. + * @param clusterSettings The cluster settings instance. + * @param indexScopedSettings The index scoped settings instance. + * @param settingsFilter The settings filter instance. + * @param indexNameExpressionResolver The index name expression resolver instance. + * @param nodesInCluster The supplier for the discovery nodes. + * @return A list of REST handlers. + */ + @Override + public List getRestHandlers( + Settings settings, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster + ) { + return delegate.getRestHandlers( + settings, + restController, + clusterSettings, + indexScopedSettings, + settingsFilter, + indexNameExpressionResolver, + nodesInCluster + ); + } + + /** + * Gets the list of action handlers for the Flight plugin. + */ + @Override + public List> getActions() { + return delegate.getActions(); + } + + /** + * Called when node is started. DiscoveryNode argument is passed to allow referring localNode value inside plugin + * + * @param localNode local Node info + */ + @Override + public void onNodeStarted(DiscoveryNode localNode) { + delegate.onNodeStarted(localNode); + } + + /** + * Gets the StreamManager instance for managing flight streams. + */ + @Override + public Supplier getStreamManager() { + return delegate.getStreamManager(); + } + + /** + * Gets the list of ExecutorBuilder instances for building thread pools used for FlightServer. + * @param settings The settings for the plugin + */ + @Override + public List> getExecutorBuilders(Settings settings) { + return delegate.getExecutorBuilders(settings); + } + + /** + * Gets the list of settings for the Flight plugin. + */ + @Override + public List> getSettings() { + return delegate.getSettings(); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/FlightServerInfoAction.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/FlightServerInfoAction.java new file mode 100644 index 0000000000000..f64507ed1051e --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/FlightServerInfoAction.java @@ -0,0 +1,65 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; + +import java.util.List; + +import static org.opensearch.rest.RestRequest.Method.GET; + +/** + * It handles GET requests for retrieving Flight server information. + */ +public class FlightServerInfoAction extends BaseRestHandler { + + /** + * Constructor for FlightServerInfoAction. + */ + public FlightServerInfoAction() {} + + /** + * Returns the name of the action. + * @return The name of the action. + */ + @Override + public String getName() { + return "flight_server_info_action"; + } + + /** + * Returns the list of routes for the action. + * @return The list of routes for the action. + */ + @Override + public List routes() { + return List.of(new Route(GET, "/_flight/info"), new Route(GET, "/_flight/info/{nodeId}")); + } + + /** + * Prepares the request for the action. + * @param request The REST request. + * @param client The node client. + * @return The rest channel consumer. + */ + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + String nodeId = request.param("nodeId"); + if (nodeId != null) { + // Query specific node + NodesFlightInfoRequest nodesRequest = new NodesFlightInfoRequest(nodeId); + return channel -> client.execute(NodesFlightInfoAction.INSTANCE, nodesRequest, new RestToXContentListener<>(channel)); + } else { + NodesFlightInfoRequest nodesRequest = new NodesFlightInfoRequest(); + return channel -> client.execute(NodesFlightInfoAction.INSTANCE, nodesRequest, new RestToXContentListener<>(channel)); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodeFlightInfo.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodeFlightInfo.java new file mode 100644 index 0000000000000..23163bfac8c2e --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodeFlightInfo.java @@ -0,0 +1,99 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.action.support.nodes.BaseNodeResponse; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Represents the response for a node's flight information. + */ +public class NodeFlightInfo extends BaseNodeResponse implements ToXContentObject { + private final BoundTransportAddress boundAddress; + + /** + * Constructor for NodeFlightInfo. + * @param in The stream input to read from. + * @throws IOException If an I/O error occurs. + */ + public NodeFlightInfo(StreamInput in) throws IOException { + super(in); + boundAddress = new BoundTransportAddress(in); + } + + /** + * Constructor for NodeFlightInfo. + * @param node The discovery node. + * @param boundAddress The bound transport address. + */ + public NodeFlightInfo(DiscoveryNode node, BoundTransportAddress boundAddress) { + super(node); + this.boundAddress = boundAddress; + } + + /** + * Writes the node flight information to the stream. + * @param out The stream output to write to. + * @throws IOException If an I/O error occurs. + */ + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + boundAddress.writeTo(out); + } + + /** + * Returns the bound transport address. + * @return The bound transport address. + */ + public BoundTransportAddress getBoundAddress() { + return boundAddress; + } + + /** + * Converts the node flight information to XContent. + * @param builder The XContent builder. + * @param params The parameters for the XContent conversion. + * @return The XContent builder. + * @throws IOException If an I/O error occurs. + */ + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + builder.startObject("flight_server"); + + builder.startArray("bound_addresses"); + for (TransportAddress address : boundAddress.boundAddresses()) { + builder.startObject(); + builder.field("host", address.address().getHostString()); + builder.field("port", address.address().getPort()); + builder.endObject(); + } + builder.endArray(); + + TransportAddress publishAddress = boundAddress.publishAddress(); + builder.startObject("publish_address"); + builder.field("host", publishAddress.address().getHostString()); + builder.field("port", publishAddress.address().getPort()); + builder.endObject(); + + builder.endObject(); + builder.endObject(); + return builder; + } + +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoAction.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoAction.java new file mode 100644 index 0000000000000..3c3a9965459cb --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoAction.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.action.ActionType; + +/** + * Action to retrieve flight info from nodes + */ +public class NodesFlightInfoAction extends ActionType { + /** + * Singleton instance of NodesFlightInfoAction. + */ + public static final NodesFlightInfoAction INSTANCE = new NodesFlightInfoAction(); + /** + * Name of this action. + */ + public static final String NAME = "cluster:admin/flight/info"; + + NodesFlightInfoAction() { + super(NAME, NodesFlightInfoResponse::new); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoRequest.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoRequest.java new file mode 100644 index 0000000000000..43bf38a096b57 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoRequest.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.action.support.nodes.BaseNodesRequest; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.transport.TransportRequest; + +import java.io.IOException; + +/** + * Flight Info Request + */ +public class NodesFlightInfoRequest extends BaseNodesRequest { + + /** + * Constructor for NodesFlightInfoRequest + * @param in StreamInput + * @throws IOException If an I/O error occurs + */ + public NodesFlightInfoRequest(StreamInput in) throws IOException { + super(in); + } + + /** + * Constructor for NodesFlightInfoRequest + * @param nodesIds String array of node IDs + */ + public NodesFlightInfoRequest(String... nodesIds) { + super(nodesIds); + } + + /** + * Writes the request to the given StreamOutput + */ + public static class NodeFlightInfoRequest extends TransportRequest { + NodesFlightInfoRequest request; + + /** + * Constructor for NodeFlightInfoRequest + * @param in StreamInput to read from + * @throws IOException If an I/O error occurs + */ + public NodeFlightInfoRequest(StreamInput in) throws IOException { + super(in); + } + + /** + * Constructor for NodeFlightInfoRequest + * @param request NodesFlightInfoRequest + */ + public NodeFlightInfoRequest(NodesFlightInfoRequest request) { + this.request = request; + } + } + + /** + * Writes the request to the given StreamOutput + * @param out StreamOutput to write to + * @throws IOException If an I/O error occurs + */ + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoResponse.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoResponse.java new file mode 100644 index 0000000000000..805aa188ce37a --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoResponse.java @@ -0,0 +1,111 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.action.FailedNodeException; +import org.opensearch.action.support.nodes.BaseNodesResponse; +import org.opensearch.cluster.ClusterName; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; + +/** + * Represents the response for nodes flight information. + */ +public class NodesFlightInfoResponse extends BaseNodesResponse implements ToXContentObject { + /** + * Constructs a new NodesFlightInfoResponse instance. + * + * @param in The stream input to read from. + * @throws IOException If an I/O error occurs. + */ + public NodesFlightInfoResponse(StreamInput in) throws IOException { + super(in); + } + + /** + * Constructs a new NodesFlightInfoResponse instance. + * + * @param clusterName The cluster name. + * @param nodes The list of node flight information. + * @param failures The list of failed node exceptions. + */ + public NodesFlightInfoResponse(ClusterName clusterName, List nodes, List failures) { + super(clusterName, nodes, failures); + } + + /** + * Reads the nodes from the given stream input. + * + * @param in The stream input to read from. + * @return The list of node flight information. + * @throws IOException If an I/O error occurs. + */ + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readList(NodeFlightInfo::new); + } + + /** + * Writes the nodes to the given stream output. + * + * @param out The stream output to write to. + * @param nodes The list of node flight information. + * @throws IOException If an I/O error occurs. + */ + @Override + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeList(nodes); + } + + /** + * Converts the nodes flight information response to XContent. + * @param builder The XContent builder. + * @param params The parameters for the XContent conversion. + * @return The XContent builder. + * @throws IOException If an I/O error occurs. + */ + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + + builder.startObject(); + builder.startObject("_nodes"); + builder.field("total", getNodes().size()); + builder.field("successful", getNodes().size()); + builder.field("failed", failures().size()); + builder.endObject(); + + builder.field("cluster_name", getClusterName().value()); + + builder.startObject("nodes"); + for (NodeFlightInfo nodeInfo : getNodes()) { + builder.field(nodeInfo.getNode().getId()); + nodeInfo.toXContent(builder, params); + } + builder.endObject(); + + if (!failures().isEmpty()) { + builder.startArray("failures"); + for (FailedNodeException failure : failures()) { + builder.startObject(); + builder.field("node_id", failure.nodeId()); + builder.field("reason", failure.getMessage()); + builder.endObject(); + } + builder.endArray(); + } + + builder.endObject(); + return builder; + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/TransportNodesFlightInfoAction.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/TransportNodesFlightInfoAction.java new file mode 100644 index 0000000000000..51f4cc05b8001 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/TransportNodesFlightInfoAction.java @@ -0,0 +1,113 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.action.FailedNodeException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.nodes.TransportNodesAction; +import org.opensearch.arrow.flight.bootstrap.FlightService; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.List; + +/** + * Transport action for getting flight information from nodes + */ +public class TransportNodesFlightInfoAction extends TransportNodesAction< + NodesFlightInfoRequest, + NodesFlightInfoResponse, + NodesFlightInfoRequest.NodeFlightInfoRequest, + NodeFlightInfo> { + + private final FlightService flightService; + + /** + * Constructor for TransportNodesFlightInfoAction + * @param settings The settings for the action + * @param threadPool The thread pool for the action + * @param clusterService The cluster service for the action + * @param transportService The transport service for the action + * @param actionFilters The action filters for the action + * @param flightService The flight service for the action + */ + @Inject + public TransportNodesFlightInfoAction( + Settings settings, + ThreadPool threadPool, + ClusterService clusterService, + TransportService transportService, + ActionFilters actionFilters, + FlightService flightService + ) { + super( + NodesFlightInfoAction.NAME, + threadPool, + clusterService, + transportService, + actionFilters, + NodesFlightInfoRequest::new, + NodesFlightInfoRequest.NodeFlightInfoRequest::new, + ThreadPool.Names.MANAGEMENT, + NodeFlightInfo.class + ); + this.flightService = flightService; + } + + /** + * Creates a new response object for the action. + * @param request The associated request. + * @param nodeFlightInfos All successful node-level responses. + * @param failures All node-level failures. + * @return The response object. + */ + @Override + protected NodesFlightInfoResponse newResponse( + NodesFlightInfoRequest request, + List nodeFlightInfos, + List failures + ) { + return new NodesFlightInfoResponse(clusterService.getClusterName(), nodeFlightInfos, failures); + } + + /** + * Creates a new request object for a node. + * @param request The associated request. + * @return The request object. + */ + @Override + protected NodesFlightInfoRequest.NodeFlightInfoRequest newNodeRequest(NodesFlightInfoRequest request) { + return new NodesFlightInfoRequest.NodeFlightInfoRequest(request); + } + + /** + * Creates a new response object for a node. + * @param in The stream input to read from. + * @return The response object. + */ + @Override + protected NodeFlightInfo newNodeResponse(StreamInput in) throws IOException { + return new NodeFlightInfo(in); + } + + /** + * Creates a new response object for a node. + * @param request The associated request. + * @return The response object. + */ + @Override + protected NodeFlightInfo nodeOperation(NodesFlightInfoRequest.NodeFlightInfoRequest request) { + return new NodeFlightInfo(clusterService.localNode(), flightService.getBoundAddress()); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/package-info.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/package-info.java new file mode 100644 index 0000000000000..19dde32f32e8f --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Action to retrieve flight info from nodes + */ +package org.opensearch.arrow.flight.api.flightinfo; diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightClientManager.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightClientManager.java new file mode 100644 index 0000000000000..7fae432879512 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightClientManager.java @@ -0,0 +1,399 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.arrow.flight.bootstrap; + +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.OSFlightClient; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.VisibleForTesting; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.Version; +import org.opensearch.arrow.flight.api.flightinfo.NodeFlightInfo; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoAction; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoRequest; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoResponse; +import org.opensearch.arrow.flight.bootstrap.tls.SslContextProvider; +import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterStateListener; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.threadpool.ThreadPool; + +import java.util.Collection; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +import io.grpc.netty.shaded.io.netty.channel.EventLoopGroup; + +import static org.opensearch.common.util.FeatureFlags.ARROW_STREAMS_SETTING; + +/** + * Manages Flight client connections to OpenSearch nodes in a cluster. + * This class maintains a pool of Flight clients for internode communication, + * handles client lifecycle, and responds to cluster state changes. + * + *

The manager implements ClusterStateListener to automatically update + * client connections when nodes join or leave the cluster.

+ */ +public class FlightClientManager implements ClusterStateListener, AutoCloseable { + private static final Version MIN_SUPPORTED_VERSION = Version.fromString("2.19.0"); + private static final Logger logger = LogManager.getLogger(FlightClientManager.class); + static final int LOCATION_TIMEOUT_MS = 1000; + private final ExecutorService grpcExecutor; + private final ClientConfiguration clientConfig; + private final Map flightClients = new ConcurrentHashMap<>(); + private final Client client; + private static final long CLIENT_BUILD_TIMEOUT_MS = TimeUnit.MINUTES.toMillis(1); + + /** + * Creates a new FlightClientManager instance. + * + * @param allocator Supplier for buffer allocation + * @param clusterService Service for cluster state management + * @param sslContextProvider Provider for SSL/TLS context configuration + * @param elg Event loop group for network operations + * @param threadPool Thread pool for executing tasks asynchronously + * @param client OpenSearch client + */ + public FlightClientManager( + BufferAllocator allocator, + ClusterService clusterService, + SslContextProvider sslContextProvider, + EventLoopGroup elg, + ThreadPool threadPool, + Client client + ) { + grpcExecutor = threadPool.executor(ServerConfig.FLIGHT_CLIENT_THREAD_POOL_NAME); + this.clientConfig = new ClientConfiguration( + Objects.requireNonNull(allocator, "BufferAllocator cannot be null"), + Objects.requireNonNull(clusterService, "ClusterService cannot be null"), + Objects.requireNonNull(sslContextProvider, "SslContextProvider cannot be null"), + Objects.requireNonNull(elg, "EventLoopGroup cannot be null"), + Objects.requireNonNull(grpcExecutor, "ExecutorService cannot be null") + ); + this.client = Objects.requireNonNull(client, "Client cannot be null"); + clusterService.addListener(this); + } + + /** + * Returns the location of a Flight client for a given node ID. + * + * @param nodeId The ID of the node for which to retrieve the location + * @return The Location of the Flight client for the specified node + */ + public Location getFlightClientLocation(String nodeId) { + ClientHolder clientHolder = flightClients.get(nodeId); + if (clientHolder != null && clientHolder.location != null) { + return clientHolder.location; + } + buildClientAsync(nodeId); + return null; + } + + /** + * Returns a Flight client for a given node ID. + * + * @param nodeId The ID of the node for which to retrieve the Flight client + * @return An OpenSearchFlightClient instance for the specified node + */ + public Optional getFlightClient(String nodeId) { + if (nodeId == null || nodeId.isEmpty()) { + throw new IllegalArgumentException("Node ID cannot be null or empty"); + } + + ClientHolder holder = flightClients.get(nodeId); + + if (holder == null) { + buildClientAsync(nodeId); + return Optional.empty(); + } + + if (holder.state == BuildState.COMPLETE) { + return Optional.ofNullable(holder.flightClient); + } + + if (holder.isStale()) { + logger.warn("Detected stale building state for node [{}], triggering rebuild", nodeId); + if (flightClients.remove(nodeId, holder)) { + try { + holder.close(); + } catch (Exception e) { + logger.warn("Error closing stale client holder for node [{}]. {}", nodeId, e.getMessage()); + } + buildClientAsync(nodeId); + } + } + + return Optional.empty(); + } + + /** + * Represents the state and metadata of a Flight client + */ + private record ClientHolder(OSFlightClient flightClient, Location location, long buildStartTime, BuildState state) + implements + AutoCloseable { + + private static ClientHolder building() { + return new ClientHolder(null, null, System.currentTimeMillis(), BuildState.BUILDING); + } + + private static ClientHolder complete(OSFlightClient client, Location location) { + return new ClientHolder(client, location, System.currentTimeMillis(), BuildState.COMPLETE); + } + + boolean isStale() { + return state == BuildState.BUILDING && (System.currentTimeMillis() - buildStartTime) > CLIENT_BUILD_TIMEOUT_MS; + } + + /** + * Closes the client holder and logs the operation + * @param nodeId The ID of the node this holder belongs to + * @param reason The reason for closing + */ + public void close(String nodeId, String reason) { + try { + if (flightClient != null) { + flightClient.close(); + } + if (state == BuildState.BUILDING) { + logger.info("Cleaned up building state for node [{}]: {}", nodeId, reason); + } else { + logger.info("Closed client for node [{}]: {}", nodeId, reason); + } + } catch (Exception e) { + logger.error("Failed to close client for node [{}] ({}): {}", nodeId, reason, e.getMessage()); + } + } + + @Override + public void close() throws Exception { + if (flightClient != null) { + flightClient.close(); + } + } + } + + private enum BuildState { + BUILDING, + COMPLETE + } + + /** + * Initiates async build of a flight client for the given node + */ + void buildClientAsync(String nodeId) { + // Try to put a building placeholder + ClientHolder placeholder = ClientHolder.building(); + if (flightClients.putIfAbsent(nodeId, placeholder) != null) { + return; // Another thread is already handling this node + } + + CompletableFuture locationFuture = new CompletableFuture<>(); + locationFuture.thenAccept(location -> { + try { + DiscoveryNode node = getNodeFromClusterState(nodeId); + if (!isValidNode(node)) { + logger.warn("Node [{}] is not valid for client creation", nodeId); + flightClients.remove(nodeId, placeholder); + return; + } + + OSFlightClient flightClient = buildClient(location); + ClientHolder newHolder = ClientHolder.complete(flightClient, location); + + if (!flightClients.replace(nodeId, placeholder, newHolder)) { + // Something changed while we were building + logger.warn("Failed to store new client for node [{}], state changed during build", nodeId); + flightClient.close(); + } + } catch (Exception e) { + logger.error("Failed to build Flight client for node [{}]. {}", nodeId, e); + flightClients.remove(nodeId, placeholder); + throw new RuntimeException(e); + } + }).exceptionally(throwable -> { + flightClients.remove(nodeId, placeholder); + logger.error("Failed to get Flight server location for node [{}] {}", nodeId, throwable); + throw new CompletionException(throwable); + }); + + requestNodeLocation(nodeId, locationFuture); + } + + Collection getClients() { + return flightClients.values(); + } + + private void requestNodeLocation(String nodeId, CompletableFuture future) { + NodesFlightInfoRequest request = new NodesFlightInfoRequest(nodeId); + client.execute(NodesFlightInfoAction.INSTANCE, request, new ActionListener<>() { + @Override + public void onResponse(NodesFlightInfoResponse response) { + NodeFlightInfo nodeInfo = response.getNodesMap().get(nodeId); + if (nodeInfo != null) { + TransportAddress publishAddress = nodeInfo.getBoundAddress().publishAddress(); + String address = publishAddress.getAddress(); + int flightPort = publishAddress.address().getPort(); + Location location = clientConfig.sslContextProvider.isSslEnabled() + ? Location.forGrpcTls(address, flightPort) + : Location.forGrpcInsecure(address, flightPort); + + future.complete(location); + } else { + future.completeExceptionally(new IllegalStateException("No Flight info received for node: [" + nodeId + "]")); + } + } + + @Override + public void onFailure(Exception e) { + future.completeExceptionally(e); + logger.error("Failed to get Flight server info for node: [{}] {}", nodeId, e); + } + }); + } + + private OSFlightClient buildClient(Location location) { + return OSFlightClient.builder( + clientConfig.allocator, + location, + ServerConfig.clientChannelType(), + clientConfig.grpcExecutor, + clientConfig.workerELG, + clientConfig.sslContextProvider.getClientSslContext() + ).build(); + } + + private DiscoveryNode getNodeFromClusterState(String nodeId) { + return Objects.requireNonNull(clientConfig.clusterService).state().nodes().get(nodeId); + } + + /** + * Closes the FlightClientManager and all associated Flight clients. + */ + @Override + public void close() throws Exception { + for (ClientHolder clientHolder : flightClients.values()) { + clientHolder.close(); + } + flightClients.clear(); + grpcExecutor.shutdown(); + } + + /** + * Returns the ID of the local node in the cluster. + * + * @return String representing the local node ID + */ + public String getLocalNodeId() { + return Objects.requireNonNull(clientConfig.clusterService).state().nodes().getLocalNodeId(); + } + + /** + * Handles cluster state changes by updating node locations and managing client connections. + * + * @param event The ClusterChangedEvent containing information about the cluster state change + */ + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (!event.nodesChanged()) { + return; + } + + final DiscoveryNodes nodes = event.state().nodes(); + + cleanupStaleBuilding(); + removeStaleClients(nodes); + updateExistingClients(nodes); + } + + private void removeStaleClients(DiscoveryNodes nodes) { + flightClients.entrySet().removeIf(entry -> { + String nodeId = entry.getKey(); + ClientHolder holder = entry.getValue(); + + if (!nodes.nodeExists(nodeId)) { + holder.close(nodeId, "node no longer exists"); + return true; + } + + if (holder.state == BuildState.BUILDING && holder.isStale()) { + holder.close(nodeId, "client build state is stale"); + return true; + } + + return false; + }); + } + + /** + * Updates clients for existing nodes based on their validity + */ + private void updateExistingClients(DiscoveryNodes nodes) { + for (DiscoveryNode node : nodes) { + String nodeId = node.getId(); + + if (isValidNode(node)) { + ClientHolder existingHolder = flightClients.get(nodeId); + + if (existingHolder == null) { + buildClientAsync(nodeId); + } else if (existingHolder.state == BuildState.BUILDING && existingHolder.isStale()) { + if (flightClients.remove(nodeId, existingHolder)) { + existingHolder.close(nodeId, "rebuilding stale client"); + buildClientAsync(nodeId); + } + } + } else { + ClientHolder holder = flightClients.remove(nodeId); + if (holder != null) { + holder.close(nodeId, "node is no longer valid"); + } + } + } + } + + /** + * Cleans up any clients that are in a stale BUILDING state + */ + private void cleanupStaleBuilding() { + flightClients.entrySet().removeIf(entry -> { + ClientHolder holder = entry.getValue(); + if (holder.state == BuildState.BUILDING && holder.isStale()) { + holder.close(entry.getKey(), "cleaning up stale building state"); + return true; + } + return false; + }); + } + + private static boolean isValidNode(DiscoveryNode node) { + return node != null && !node.getVersion().before(MIN_SUPPORTED_VERSION) && FeatureFlags.isEnabled(ARROW_STREAMS_SETTING); + } + + @VisibleForTesting + Map getFlightClients() { + return flightClients; + } + + private record ClientConfiguration(BufferAllocator allocator, ClusterService clusterService, SslContextProvider sslContextProvider, + EventLoopGroup workerELG, ExecutorService grpcExecutor) { + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightService.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightService.java new file mode 100644 index 0000000000000..4fe944f8d5eb3 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightService.java @@ -0,0 +1,176 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.bootstrap; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.util.AutoCloseables; +import org.apache.arrow.util.VisibleForTesting; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.arrow.flight.bootstrap.tls.DefaultSslContextProvider; +import org.opensearch.arrow.flight.bootstrap.tls.DisabledSslContextProvider; +import org.opensearch.arrow.flight.bootstrap.tls.SslContextProvider; +import org.opensearch.arrow.flight.impl.BaseFlightProducer; +import org.opensearch.arrow.flight.impl.FlightStreamManager; +import org.opensearch.arrow.spi.StreamManager; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.plugins.NetworkPlugin; +import org.opensearch.plugins.SecureTransportSettingsProvider; +import org.opensearch.threadpool.ThreadPool; + +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Objects; + +/** + * FlightService manages the Arrow Flight server and client for OpenSearch. + * It handles the initialization, startup, and shutdown of the Flight server and client, + * as well as managing the stream operations through a FlightStreamManager. + */ +public class FlightService extends NetworkPlugin.AuxTransport { + private static final Logger logger = LogManager.getLogger(FlightService.class); + private final ServerComponents serverComponents; + private StreamManager streamManager; + private Client client; + private FlightClientManager clientManager; + private SecureTransportSettingsProvider secureTransportSettingsProvider; + private BufferAllocator allocator; + private ThreadPool threadPool; + + /** + * Constructor for FlightService. + * @param settings The settings for the FlightService. + */ + public FlightService(Settings settings) { + Objects.requireNonNull(settings, "Settings cannot be null"); + try { + ServerConfig.init(settings); + } catch (Exception e) { + throw new RuntimeException("Failed to initialize Arrow Flight server", e); + } + this.serverComponents = new ServerComponents(settings); + } + + void setClusterService(ClusterService clusterService) { + serverComponents.setClusterService(Objects.requireNonNull(clusterService, "ClusterService cannot be null")); + } + + void setNetworkService(NetworkService networkService) { + serverComponents.setNetworkService(Objects.requireNonNull(networkService, "NetworkService cannot be null")); + } + + void setThreadPool(ThreadPool threadPool) { + this.threadPool = Objects.requireNonNull(threadPool, "ThreadPool cannot be null"); + serverComponents.setThreadPool(threadPool); + } + + void setClient(Client client) { + this.client = client; + } + + void setSecureTransportSettingsProvider(SecureTransportSettingsProvider secureTransportSettingsProvider) { + this.secureTransportSettingsProvider = secureTransportSettingsProvider; + } + + /** + * Starts the FlightService by initializing the stream manager. + */ + @SuppressWarnings("removal") + @Override + protected void doStart() { + try { + allocator = AccessController.doPrivileged((PrivilegedAction) () -> new RootAllocator(Integer.MAX_VALUE)); + serverComponents.setAllocator(allocator); + SslContextProvider sslContextProvider = ServerConfig.isSslEnabled() + ? new DefaultSslContextProvider(secureTransportSettingsProvider) + : new DisabledSslContextProvider(); + serverComponents.setSslContextProvider(sslContextProvider); + serverComponents.initComponents(); + clientManager = new FlightClientManager( + allocator, // sharing the same allocator between server and client + serverComponents.clusterService, + sslContextProvider, + serverComponents.workerEventLoopGroup, // sharing the same worker ELG between server and client + threadPool, + client + ); + initializeStreamManager(clientManager); + serverComponents.setFlightProducer(new BaseFlightProducer(clientManager, (FlightStreamManager) streamManager, allocator)); + serverComponents.start(); + + } catch (Exception e) { + logger.error("Failed to start Flight server", e); + doClose(); + throw new RuntimeException("Failed to start Flight server", e); + } + } + + /** + * Retrieves the FlightClientManager used by the FlightService. + * @return The FlightClientManager instance. + */ + public FlightClientManager getFlightClientManager() { + return clientManager; + } + + /** + * Retrieves the StreamManager used by the FlightService. + * @return The StreamManager instance. + */ + public StreamManager getStreamManager() { + return streamManager; + } + + /** + * Retrieves the bound address of the FlightService. + * @return The BoundTransportAddress instance. + */ + public BoundTransportAddress getBoundAddress() { + return serverComponents.getBoundAddress(); + } + + @VisibleForTesting + SslContextProvider getSslContextProvider() { + return serverComponents.getSslContextProvider(); + } + + /** + * Stops the FlightService by closing the server components and network resources. + */ + @Override + protected void doStop() { + try { + AutoCloseables.close(serverComponents); + AutoCloseables.close(streamManager); + AutoCloseables.close(clientManager); + AutoCloseables.close(allocator); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * doStop() ensures all resources are cleaned up and resources are recreated on + * doStart() + */ + @Override + protected void doClose() { + doStop(); + } + + private void initializeStreamManager(FlightClientManager clientManager) { + streamManager = new FlightStreamManager(() -> allocator); + ((FlightStreamManager) streamManager).setClientManager(clientManager); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightStreamPluginImpl.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightStreamPluginImpl.java new file mode 100644 index 0000000000000..7f286b785daa0 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightStreamPluginImpl.java @@ -0,0 +1,230 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.bootstrap; + +import org.opensearch.arrow.flight.BaseFlightStreamPlugin; +import org.opensearch.arrow.flight.api.flightinfo.FlightServerInfoAction; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoAction; +import org.opensearch.arrow.flight.api.flightinfo.TransportNodesFlightInfoAction; +import org.opensearch.arrow.spi.StreamManager; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.IndexScopedSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsFilter; +import org.opensearch.common.util.PageCacheRecycler; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.indices.breaker.CircuitBreakerService; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; +import org.opensearch.plugins.SecureTransportSettingsProvider; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.rest.RestController; +import org.opensearch.rest.RestHandler; +import org.opensearch.script.ScriptService; +import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.threadpool.ExecutorBuilder; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.Transport; +import org.opensearch.watcher.ResourceWatcherService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +/** + * FlightStreamPlugin class extends BaseFlightStreamPlugin and provides implementation for FlightStream plugin. + */ +public class FlightStreamPluginImpl extends BaseFlightStreamPlugin { + + private final FlightService flightService; + + /** + * Constructor for FlightStreamPluginImpl. + * @param settings The settings for the FlightStreamPlugin. + */ + public FlightStreamPluginImpl(Settings settings) { + this.flightService = new FlightService(settings); + } + + /** + * Creates components for the FlightStream plugin. + * @param client The client instance. + * @param clusterService The cluster service instance. + * @param threadPool The thread pool instance. + * @param resourceWatcherService The resource watcher service instance. + * @param scriptService The script service instance. + * @param xContentRegistry The named XContent registry. + * @param environment The environment instance. + * @param nodeEnvironment The node environment instance. + * @param namedWriteableRegistry The named writeable registry. + * @param indexNameExpressionResolver The index name expression resolver instance. + * @param repositoriesServiceSupplier The supplier for the repositories service. + * @return FlightService + */ + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + flightService.setClusterService(clusterService); + flightService.setThreadPool(threadPool); + flightService.setClient(client); + return List.of(flightService); + } + + /** + * Gets the secure transports for the FlightStream plugin. + * @param settings The settings for the plugin. + * @param threadPool The thread pool instance. + * @param pageCacheRecycler The page cache recycler instance. + * @param circuitBreakerService The circuit breaker service instance. + * @param namedWriteableRegistry The named writeable registry. + * @param networkService The network service instance. + * @param secureTransportSettingsProvider The secure transport settings provider. + * @param tracer The tracer instance. + * @return A map of secure transports. + */ + @Override + public Map> getSecureTransports( + Settings settings, + ThreadPool threadPool, + PageCacheRecycler pageCacheRecycler, + CircuitBreakerService circuitBreakerService, + NamedWriteableRegistry namedWriteableRegistry, + NetworkService networkService, + SecureTransportSettingsProvider secureTransportSettingsProvider, + Tracer tracer + ) { + flightService.setSecureTransportSettingsProvider(secureTransportSettingsProvider); + return Collections.emptyMap(); + } + + /** + * Gets the auxiliary transports for the FlightStream plugin. + * @param settings The settings for the plugin. + * @param threadPool The thread pool instance. + * @param circuitBreakerService The circuit breaker service instance. + * @param networkService The network service instance. + * @param clusterSettings The cluster settings instance. + * @param tracer The tracer instance. + * @return A map of auxiliary transports. + */ + @Override + public Map> getAuxTransports( + Settings settings, + ThreadPool threadPool, + CircuitBreakerService circuitBreakerService, + NetworkService networkService, + ClusterSettings clusterSettings, + Tracer tracer + ) { + flightService.setNetworkService(networkService); + return Collections.singletonMap(FlightService.AUX_TRANSPORT_TYPES_KEY, () -> flightService); + } + + /** + * Gets the REST handlers for the FlightStream plugin. + * @param settings The settings for the plugin. + * @param restController The REST controller instance. + * @param clusterSettings The cluster settings instance. + * @param indexScopedSettings The index scoped settings instance. + * @param settingsFilter The settings filter instance. + * @param indexNameExpressionResolver The index name expression resolver instance. + * @param nodesInCluster The supplier for the discovery nodes. + * @return A list of REST handlers. + */ + @Override + public List getRestHandlers( + Settings settings, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster + ) { + return List.of(new FlightServerInfoAction()); + } + + /** + * Gets the list of action handlers for the FlightStream plugin. + * @return A list of action handlers. + */ + @Override + public List> getActions() { + return List.of(new ActionHandler<>(NodesFlightInfoAction.INSTANCE, TransportNodesFlightInfoAction.class)); + } + + /** + * Called when node is started. DiscoveryNode argument is passed to allow referring localNode value inside plugin + * + * @param localNode local Node info + */ + @Override + public void onNodeStarted(DiscoveryNode localNode) { + flightService.getFlightClientManager().buildClientAsync(localNode.getId()); + } + + /** + * Gets the StreamManager instance for managing flight streams. + */ + @Override + public Supplier getStreamManager() { + return flightService::getStreamManager; + } + + /** + * Gets the list of ExecutorBuilder instances for building thread pools used for FlightServer. + * @param settings The settings for the plugin + */ + @Override + public List> getExecutorBuilders(Settings settings) { + return List.of(ServerConfig.getServerExecutorBuilder(), ServerConfig.getClientExecutorBuilder()); + } + + /** + * Gets the list of settings for the Flight plugin. + */ + @Override + public List> getSettings() { + return new ArrayList<>( + Arrays.asList( + ServerComponents.SETTING_FLIGHT_PORTS, + ServerComponents.SETTING_FLIGHT_HOST, + ServerComponents.SETTING_FLIGHT_BIND_HOST, + ServerComponents.SETTING_FLIGHT_PUBLISH_HOST + ) + ) { + { + addAll(ServerConfig.getSettings()); + } + }; + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/ServerComponents.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/ServerComponents.java new file mode 100644 index 0000000000000..1b7f149541f3d --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/ServerComponents.java @@ -0,0 +1,284 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.bootstrap; + +import org.apache.arrow.flight.FlightProducer; +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.OSFlightServer; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.AutoCloseables; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.arrow.flight.bootstrap.tls.SslContextProvider; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.transport.PortsRange; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.BindTransportException; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; + +import io.grpc.netty.shaded.io.netty.channel.EventLoopGroup; +import io.grpc.netty.shaded.io.netty.util.NettyRuntime; +import io.grpc.netty.shaded.io.netty.util.concurrent.Future; + +import static java.util.Collections.emptyList; +import static org.opensearch.common.settings.Setting.intSetting; +import static org.opensearch.common.settings.Setting.listSetting; +import static org.opensearch.plugins.NetworkPlugin.AuxTransport.AUX_TRANSPORT_PORT; +import static org.opensearch.transport.Transport.resolveTransportPublishPort; + +@SuppressWarnings("removal") +final class ServerComponents implements AutoCloseable { + + public static final Setting> SETTING_FLIGHT_HOST = listSetting( + "flight.host", + emptyList(), + Function.identity(), + Setting.Property.NodeScope + ); + + public static final Setting> SETTING_FLIGHT_BIND_HOST = listSetting( + "flight.bind_host", + SETTING_FLIGHT_HOST, + Function.identity(), + Setting.Property.NodeScope + ); + + public static final Setting> SETTING_FLIGHT_PUBLISH_HOST = listSetting( + "flight.publish_host", + SETTING_FLIGHT_HOST, + Function.identity(), + Setting.Property.NodeScope + ); + + public static final Setting SETTING_FLIGHT_PUBLISH_PORT = intSetting( + "flight.publish_port", + -1, + -1, + Setting.Property.NodeScope + ); + + private static final Logger logger = LogManager.getLogger(ServerComponents.class); + + private static final String GRPC_WORKER_ELG = "os-grpc-worker-ELG"; + private static final String GRPC_BOSS_ELG = "os-grpc-boss-ELG"; + private static final int SHUTDOWN_TIMEOUT_SECONDS = 5; + + public static final String FLIGHT_TRANSPORT_SETTING_KEY = "transport-flight"; + public static final Setting SETTING_FLIGHT_PORTS = AUX_TRANSPORT_PORT.getConcreteSettingForNamespace( + FLIGHT_TRANSPORT_SETTING_KEY + ); + + private final Settings settings; + private final PortsRange port; + private final String[] bindHosts; + private final String[] publishHosts; + private volatile BoundTransportAddress boundAddress; + + private OSFlightServer server; + private BufferAllocator allocator; + ClusterService clusterService; + private NetworkService networkService; + private ThreadPool threadPool; + private SslContextProvider sslContextProvider; + private FlightProducer flightProducer; + + private EventLoopGroup bossEventLoopGroup; + EventLoopGroup workerEventLoopGroup; + private ExecutorService serverExecutor; + + ServerComponents(Settings settings) { + this.settings = settings; + this.port = SETTING_FLIGHT_PORTS.get(settings); + + List bindHosts = SETTING_FLIGHT_BIND_HOST.get(settings); + this.bindHosts = bindHosts.toArray(new String[0]); + + List publishHosts = SETTING_FLIGHT_PUBLISH_HOST.get(settings); + this.publishHosts = publishHosts.toArray(new String[0]); + } + + void setAllocator(BufferAllocator allocator) { + this.allocator = allocator; + } + + void setClusterService(ClusterService clusterService) { + this.clusterService = Objects.requireNonNull(clusterService); + } + + void setNetworkService(NetworkService networkService) { + this.networkService = Objects.requireNonNull(networkService); + } + + void setThreadPool(ThreadPool threadPool) { + this.threadPool = Objects.requireNonNull(threadPool); + } + + void setSslContextProvider(SslContextProvider sslContextProvider) { + this.sslContextProvider = Objects.requireNonNull(sslContextProvider); + } + + void setFlightProducer(FlightProducer flightProducer) { + this.flightProducer = Objects.requireNonNull(flightProducer); + } + + private OSFlightServer buildAndStartServer(Location location, FlightProducer producer) throws IOException { + OSFlightServer server = OSFlightServer.builder( + allocator, + location, + producer, + sslContextProvider.getServerSslContext(), + ServerConfig.serverChannelType(), + bossEventLoopGroup, + workerEventLoopGroup, + serverExecutor + ).build(); + AccessController.doPrivileged((PrivilegedAction) () -> { + try { + server.start(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return null; + }); + return server; + } + + SslContextProvider getSslContextProvider() { + return sslContextProvider; + } + + BoundTransportAddress getBoundAddress() { + return boundAddress; + } + + void start() { + InetAddress[] hostAddresses; + try { + hostAddresses = networkService.resolveBindHostAddresses(bindHosts); + } catch (IOException e) { + throw new BindTransportException("Failed to resolve host [" + Arrays.toString(bindHosts) + "]", e); + } + + List boundAddresses = new ArrayList<>(hostAddresses.length); + for (InetAddress address : hostAddresses) { + AccessController.doPrivileged((PrivilegedAction) () -> { + boundAddresses.add(bindAddress(address, port)); + return null; + }); + } + + final InetAddress publishInetAddress; + try { + publishInetAddress = networkService.resolvePublishHostAddresses(publishHosts); + } catch (Exception e) { + throw new BindTransportException("Failed to resolve publish address", e); + } + + final int publishPort = resolveTransportPublishPort(SETTING_FLIGHT_PUBLISH_PORT.get(settings), boundAddresses, publishInetAddress); + + if (publishPort < 0) { + throw new BindTransportException( + "Failed to auto-resolve flight publish port, multiple bound addresses " + + boundAddresses + + " with distinct ports and none of them matched the publish address (" + + publishInetAddress + + "). Please specify a unique port by setting " + + SETTING_FLIGHT_PUBLISH_PORT.getKey() + ); + } + + TransportAddress publishAddress = new TransportAddress(new InetSocketAddress(publishInetAddress, publishPort)); + this.boundAddress = new BoundTransportAddress(boundAddresses.toArray(new TransportAddress[0]), publishAddress); + } + + void initComponents() throws Exception { + bossEventLoopGroup = ServerConfig.createELG(GRPC_BOSS_ELG, 1); + workerEventLoopGroup = ServerConfig.createELG(GRPC_WORKER_ELG, NettyRuntime.availableProcessors() * 2); + serverExecutor = threadPool.executor(ServerConfig.FLIGHT_SERVER_THREAD_POOL_NAME); + } + + @Override + public void close() { + try { + AutoCloseables.close(server); + gracefullyShutdownELG(bossEventLoopGroup, GRPC_BOSS_ELG); + gracefullyShutdownELG(workerEventLoopGroup, GRPC_WORKER_ELG); + if (serverExecutor != null) { + serverExecutor.shutdown(); + } + } catch (Exception e) { + logger.error("Error while closing server components", e); + } + } + + private TransportAddress bindAddress(final InetAddress hostAddress, final PortsRange portsRange) { + final AtomicReference lastException = new AtomicReference<>(); + final AtomicReference boundSocket = new AtomicReference<>(); + final TransportAddress[] address = new TransportAddress[1]; + boolean success = portsRange.iterate(portNumber -> { + boundSocket.set(new InetSocketAddress(hostAddress, portNumber)); + address[0] = new TransportAddress(boundSocket.get()); + try { + return startFlightServer(address[0]); + } catch (Exception e) { + lastException.set(e); + return false; + } + }); + + if (!success) { + throw new BindTransportException("Failed to bind to [" + hostAddress + "]", lastException.get()); + } + return address[0]; + } + + private boolean startFlightServer(TransportAddress transportAddress) { + InetSocketAddress address = transportAddress.address(); + Location serverLocation = sslContextProvider.isSslEnabled() + ? Location.forGrpcTls(address.getHostString(), address.getPort()) + : Location.forGrpcInsecure(address.getHostString(), address.getPort()); + try { + this.server = buildAndStartServer(serverLocation, flightProducer); + logger.info("Arrow Flight server started. Listening at {}", serverLocation); + return true; + } catch (Exception e) { + String errorMsg = "Failed to start Arrow Flight server at " + serverLocation; + logger.error(errorMsg, e); + return false; + } + } + + private void gracefullyShutdownELG(EventLoopGroup group, String groupName) { + if (group != null) { + Future shutdownFuture = group.shutdownGracefully(0, SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS); + shutdownFuture.awaitUninterruptibly(); + if (!shutdownFuture.isSuccess()) { + logger.warn("Error closing {} netty event loop group {}", groupName, shutdownFuture.cause()); + } + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/ServerConfig.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/ServerConfig.java new file mode 100644 index 0000000000000..bca087d8ed592 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/ServerConfig.java @@ -0,0 +1,227 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.bootstrap; + +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.threadpool.ScalingExecutorBuilder; + +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import io.grpc.netty.shaded.io.netty.channel.Channel; +import io.grpc.netty.shaded.io.netty.channel.EventLoopGroup; +import io.grpc.netty.shaded.io.netty.channel.epoll.Epoll; +import io.grpc.netty.shaded.io.netty.channel.epoll.EpollEventLoopGroup; +import io.grpc.netty.shaded.io.netty.channel.epoll.EpollServerSocketChannel; +import io.grpc.netty.shaded.io.netty.channel.epoll.EpollSocketChannel; +import io.grpc.netty.shaded.io.netty.channel.nio.NioEventLoopGroup; +import io.grpc.netty.shaded.io.netty.channel.socket.nio.NioServerSocketChannel; +import io.grpc.netty.shaded.io.netty.channel.socket.nio.NioSocketChannel; +import io.grpc.netty.shaded.io.netty.util.concurrent.DefaultThreadFactory; + +/** + * Configuration class for OpenSearch Flight server settings. + * This class manages server-side configurations including port settings, Arrow memory settings, + * thread pool configurations, and SSL/TLS settings. + */ +public class ServerConfig { + /** + * Creates a new instance of the server configuration with default settings. + */ + public ServerConfig() {} + + static final Setting ARROW_ALLOCATION_MANAGER_TYPE = Setting.simpleString( + "arrow.allocation.manager.type", + "Netty", + Setting.Property.NodeScope + ); + + static final Setting ARROW_ENABLE_NULL_CHECK_FOR_GET = Setting.boolSetting( + "arrow.enable_null_check_for_get", + false, + Setting.Property.NodeScope + ); + + static final Setting ARROW_ENABLE_DEBUG_ALLOCATOR = Setting.boolSetting( + "arrow.memory.debug.allocator", + false, + Setting.Property.NodeScope + ); + + static final Setting ARROW_ENABLE_UNSAFE_MEMORY_ACCESS = Setting.boolSetting( + "arrow.enable_unsafe_memory_access", + true, + Setting.Property.NodeScope + ); + + static final Setting FLIGHT_THREAD_POOL_MIN_SIZE = Setting.intSetting( + "thread_pool.flight-server.min", + 0, + 0, + Setting.Property.NodeScope + ); + + static final Setting FLIGHT_THREAD_POOL_MAX_SIZE = Setting.intSetting( + "thread_pool.flight-server.max", + 100000, // TODO depends on max concurrent streams per node, decide after benchmark. To be controlled by admission control layer. + 1, + Setting.Property.NodeScope + ); + + static final Setting FLIGHT_THREAD_POOL_KEEP_ALIVE = Setting.timeSetting( + "thread_pool.flight-server.keep_alive", + TimeValue.timeValueSeconds(30), + Setting.Property.NodeScope + ); + + static final Setting ARROW_SSL_ENABLE = Setting.boolSetting( + "arrow.ssl.enable", + false, // TODO: get default from security enabled + Setting.Property.NodeScope + ); + + /** + * The thread pool name for the Flight server. + */ + public static final String FLIGHT_SERVER_THREAD_POOL_NAME = "flight-server"; + + /** + * The thread pool name for the Flight client. + */ + public static final String FLIGHT_CLIENT_THREAD_POOL_NAME = "flight-client"; + + private static final String host = "localhost"; + private static boolean enableSsl; + private static int threadPoolMin; + private static int threadPoolMax; + private static TimeValue keepAlive; + + /** + * Initializes the server configuration with the provided settings. + * Sets system properties for Arrow memory management and configures thread pool settings. + * + * @param settings The OpenSearch settings to initialize the server with + */ + @SuppressForbidden(reason = "required for arrow allocator") + @SuppressWarnings("removal") + public static void init(Settings settings) { + AccessController.doPrivileged((PrivilegedAction) () -> { + System.setProperty("arrow.allocation.manager.type", ARROW_ALLOCATION_MANAGER_TYPE.get(settings)); + System.setProperty("arrow.enable_null_check_for_get", Boolean.toString(ARROW_ENABLE_NULL_CHECK_FOR_GET.get(settings))); + System.setProperty("arrow.enable_unsafe_memory_access", Boolean.toString(ARROW_ENABLE_UNSAFE_MEMORY_ACCESS.get(settings))); + System.setProperty("arrow.memory.debug.allocator", Boolean.toString(ARROW_ENABLE_DEBUG_ALLOCATOR.get(settings))); + Netty4Configs.init(settings); + return null; + }); + enableSsl = ARROW_SSL_ENABLE.get(settings); + threadPoolMin = FLIGHT_THREAD_POOL_MIN_SIZE.get(settings); + threadPoolMax = FLIGHT_THREAD_POOL_MAX_SIZE.get(settings); + keepAlive = FLIGHT_THREAD_POOL_KEEP_ALIVE.get(settings); + } + + /** + * Checks if SSL/TLS is enabled for the Flight server. + * + * @return true if SSL is enabled, false otherwise + */ + public static boolean isSslEnabled() { + return enableSsl; + } + + /** + * Gets the thread pool executor builder configured for the Flight server. + * + * @return The configured ScalingExecutorBuilder instance + */ + public static ScalingExecutorBuilder getServerExecutorBuilder() { + return new ScalingExecutorBuilder(FLIGHT_SERVER_THREAD_POOL_NAME, threadPoolMin, threadPoolMax, keepAlive); + } + + /** + * Gets the thread pool executor builder configured for the Flight server. + * + * @return The configured ScalingExecutorBuilder instance + */ + public static ScalingExecutorBuilder getClientExecutorBuilder() { + return new ScalingExecutorBuilder(FLIGHT_CLIENT_THREAD_POOL_NAME, threadPoolMin, threadPoolMax, keepAlive); + } + + /** + * Returns a list of all settings managed by this configuration class. + * + * @return List of Setting instances + */ + public static List> getSettings() { + return new ArrayList<>( + Arrays.asList( + ARROW_ALLOCATION_MANAGER_TYPE, + ARROW_ENABLE_NULL_CHECK_FOR_GET, + ARROW_ENABLE_DEBUG_ALLOCATOR, + ARROW_ENABLE_UNSAFE_MEMORY_ACCESS, + ARROW_SSL_ENABLE + ) + ) { + { + addAll(Netty4Configs.getSettings()); + } + }; + } + + static EventLoopGroup createELG(String name, int eventLoopThreads) { + + return Epoll.isAvailable() + ? new EpollEventLoopGroup(eventLoopThreads, new DefaultThreadFactory(name, true)) + : new NioEventLoopGroup(eventLoopThreads, new DefaultThreadFactory(name, true)); + } + + static Class serverChannelType() { + return Epoll.isAvailable() ? EpollServerSocketChannel.class : NioServerSocketChannel.class; + } + + static Class clientChannelType() { + return Epoll.isAvailable() ? EpollSocketChannel.class : NioSocketChannel.class; + } + + private static class Netty4Configs { + public static final Setting NETTY_ALLOCATOR_NUM_DIRECT_ARENAS = Setting.intSetting( + "io.netty.allocator.numDirectArenas", + 1, // TODO - 2 * the number of available processors; to be confirmed and set after running benchmarks + 1, + Setting.Property.NodeScope + ); + + public static final Setting NETTY_TRY_REFLECTION_SET_ACCESSIBLE = Setting.boolSetting( + "io.netty.tryReflectionSetAccessible", + true, + Setting.Property.NodeScope + ); + + public static final Setting NETTY_NO_UNSAFE = Setting.boolSetting("io.netty.noUnsafe", false, Setting.Property.NodeScope); + + public static final Setting NETTY_TRY_UNSAFE = Setting.boolSetting("io.netty.tryUnsafe", true, Setting.Property.NodeScope); + + @SuppressForbidden(reason = "required for netty allocator configuration") + public static void init(Settings settings) { + System.setProperty("io.netty.allocator.numDirectArenas", Integer.toString(NETTY_ALLOCATOR_NUM_DIRECT_ARENAS.get(settings))); + System.setProperty("io.netty.noUnsafe", Boolean.toString(NETTY_NO_UNSAFE.get(settings))); + System.setProperty("io.netty.tryUnsafe", Boolean.toString(NETTY_TRY_UNSAFE.get(settings))); + System.setProperty("io.netty.tryReflectionSetAccessible", Boolean.toString(NETTY_TRY_REFLECTION_SET_ACCESSIBLE.get(settings))); + } + + public static List> getSettings() { + return Arrays.asList(NETTY_TRY_REFLECTION_SET_ACCESSIBLE, NETTY_ALLOCATOR_NUM_DIRECT_ARENAS, NETTY_NO_UNSAFE, NETTY_TRY_UNSAFE); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/package-info.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/package-info.java new file mode 100644 index 0000000000000..3ee247809b0c0 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Bootstrap classes for initializing and configuring OpenSearch Flight service. + */ +package org.opensearch.arrow.flight.bootstrap; diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/DefaultSslContextProvider.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/DefaultSslContextProvider.java new file mode 100644 index 0000000000000..5163f88e51dd7 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/DefaultSslContextProvider.java @@ -0,0 +1,112 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.bootstrap.tls; + +import org.opensearch.plugins.SecureTransportSettingsProvider; + +import javax.net.ssl.SSLException; + +import java.util.Locale; + +import io.grpc.netty.shaded.io.netty.handler.ssl.ApplicationProtocolConfig; +import io.grpc.netty.shaded.io.netty.handler.ssl.ApplicationProtocolNames; +import io.grpc.netty.shaded.io.netty.handler.ssl.ClientAuth; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslProvider; +import io.grpc.netty.shaded.io.netty.handler.ssl.SupportedCipherSuiteFilter; + +/** + * DefaultSslContextProvider is an implementation of the SslContextProvider interface that provides SSL contexts based on the provided SecureTransportSettingsProvider. + */ +public class DefaultSslContextProvider implements SslContextProvider { + + private final SecureTransportSettingsProvider secureTransportSettingsProvider; + + /** + * Constructor for DefaultSslContextProvider. + * @param secureTransportSettingsProvider The SecureTransportSettingsProvider instance. + */ + public DefaultSslContextProvider(SecureTransportSettingsProvider secureTransportSettingsProvider) { + this.secureTransportSettingsProvider = secureTransportSettingsProvider; + } + + /** + * Returns true to indicate that SSL is enabled. + * @return true + */ + @Override + public boolean isSslEnabled() { + return true; + } + + // TODO - handle certificates reload + /** + * Creates and returns the server SSL context based on the provided SecureTransportSettingsProvider. + * @return The server SSL context. + */ + @Override + public SslContext getServerSslContext() { + try { + SecureTransportSettingsProvider.SecureTransportParameters parameters = secureTransportSettingsProvider.parameters(null).get(); + return io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder.forServer(parameters.keyManagerFactory().get()) + .sslProvider(SslProvider.valueOf(parameters.sslProvider().toUpperCase(Locale.ROOT))) + .clientAuth(ClientAuth.valueOf(parameters.clientAuth().toUpperCase(Locale.ROOT))) + .protocols(parameters.protocols()) + .ciphers(parameters.cipherSuites(), SupportedCipherSuiteFilter.INSTANCE) + .sessionCacheSize(0) + .sessionTimeout(0) + .applicationProtocolConfig( + new ApplicationProtocolConfig( + ApplicationProtocolConfig.Protocol.ALPN, + // NO_ADVERTISE is currently the only mode supported by both OpenSsl and JDK providers. + ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, + // ACCEPT is currently the only mode supported by both OpenSsl and JDK providers. + ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, + ApplicationProtocolNames.HTTP_2, + ApplicationProtocolNames.HTTP_1_1 + ) + ) + .trustManager(parameters.trustManagerFactory()) + .build(); + } catch (SSLException e) { + throw new RuntimeException(e); + } + } + + /** + * Returns the client SSL context based on the provided SecureTransportSettingsProvider. + * @return The client SSL context. + */ + @Override + public SslContext getClientSslContext() { + try { + SecureTransportSettingsProvider.SecureTransportParameters parameters = secureTransportSettingsProvider.parameters(null).get(); + return io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder.forClient() + .sslProvider(SslProvider.valueOf(parameters.sslProvider().toUpperCase(Locale.ROOT))) + .protocols(parameters.protocols()) + .ciphers(parameters.cipherSuites(), SupportedCipherSuiteFilter.INSTANCE) + .applicationProtocolConfig( + new ApplicationProtocolConfig( + ApplicationProtocolConfig.Protocol.ALPN, + ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, + ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, + ApplicationProtocolNames.HTTP_2, + ApplicationProtocolNames.HTTP_1_1 + ) + ) + .sessionCacheSize(0) + .sessionTimeout(0) + .keyManager(parameters.keyManagerFactory().get()) + .trustManager(parameters.trustManagerFactory()) + .build(); + } catch (SSLException e) { + throw new RuntimeException(e); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/DisabledSslContextProvider.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/DisabledSslContextProvider.java new file mode 100644 index 0000000000000..27affda028ecb --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/DisabledSslContextProvider.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.bootstrap.tls; + +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; + +/** + * DisabledSslContextProvider is an implementation of the SslContextProvider interface that provides disabled SSL contexts. + * It is used when SSL is not enabled in the application. + */ +public class DisabledSslContextProvider implements SslContextProvider { + + /** + * Constructor for DisabledSslContextProvider. + */ + public DisabledSslContextProvider() {} + + /** + * Returns false to indicate that SSL is not enabled. + * @return false + */ + @Override + public boolean isSslEnabled() { + return false; + } + + /** + * Returns null as there is no server SSL context when SSL is disabled. + * @return null + */ + @Override + public SslContext getServerSslContext() { + return null; + } + + /** + * Returns null as there is no client SSL context when SSL is disabled. + * @return null + */ + @Override + public SslContext getClientSslContext() { + return null; + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/SslContextProvider.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/SslContextProvider.java new file mode 100644 index 0000000000000..11c0beeb5a990 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/SslContextProvider.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.bootstrap.tls; + +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; + +/** + * Provider interface for SSL/TLS context configuration in OpenSearch Flight. + * This interface defines methods for managing SSL contexts for both server and client-side + * Flight communications. + */ +public interface SslContextProvider { + /** + * Checks if SSL/TLS is enabled for Flight communications. + * + * @return true if SSL/TLS is enabled, false otherwise + */ + boolean isSslEnabled(); + + /** + * Gets the SSL context configuration for the Flight server. + * This context is used to secure incoming connections to the Flight server. + * + * @return SslContext configured for server-side TLS + */ + SslContext getServerSslContext(); + + /** + * Gets the SSL context configuration for Flight clients. + * This context is used when making outbound connections to other Flight servers. + * + * @return SslContext configured for client-side TLS + */ + SslContext getClientSslContext(); +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/package-info.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/package-info.java new file mode 100644 index 0000000000000..2ad8ae734c2da --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * TLS/SSL configuration and security components for OpenSearch Flight service. + */ +package org.opensearch.arrow.flight.bootstrap.tls; diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseBackpressureStrategy.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseBackpressureStrategy.java new file mode 100644 index 0000000000000..d06d2eda5f23f --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseBackpressureStrategy.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.BackpressureStrategy; + +/** + * Base class for backpressure strategy. + */ +public class BaseBackpressureStrategy extends BackpressureStrategy.CallbackBackpressureStrategy { + private final Runnable readyCallback; + private final Runnable cancelCallback; + + /** + * Constructor for BaseBackpressureStrategy. + * + * @param readyCallback Callback to execute when the listener is ready. + * @param cancelCallback Callback to execute when the listener is cancelled. + */ + BaseBackpressureStrategy(Runnable readyCallback, Runnable cancelCallback) { + this.readyCallback = readyCallback; + this.cancelCallback = cancelCallback; + } + + /** Callback to execute when the listener is ready. */ + protected void readyCallback() { + if (readyCallback != null) { + readyCallback.run(); + } + } + + /** Callback to execute when the listener is cancelled. */ + protected void cancelCallback() { + if (cancelCallback != null) { + cancelCallback.run(); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseFlightProducer.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseFlightProducer.java new file mode 100644 index 0000000000000..fc994583319e7 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseFlightProducer.java @@ -0,0 +1,173 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.BackpressureStrategy; +import org.apache.arrow.flight.CallStatus; +import org.apache.arrow.flight.FlightDescriptor; +import org.apache.arrow.flight.FlightEndpoint; +import org.apache.arrow.flight.FlightInfo; +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.NoOpFlightProducer; +import org.apache.arrow.flight.OSFlightClient; +import org.apache.arrow.flight.Ticket; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.arrow.flight.bootstrap.FlightClientManager; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.arrow.spi.StreamTicket; + +import java.util.Collections; +import java.util.Optional; + +/** + * BaseFlightProducer extends NoOpFlightProducer to provide stream management functionality + * for Arrow Flight in OpenSearch. This class handles the retrieval and streaming of data + * based on provided tickets, managing backpressure, and coordinating between the stream + * provider and the server stream listener. + */ +public class BaseFlightProducer extends NoOpFlightProducer { + private final FlightClientManager flightClientManager; + private final FlightStreamManager streamManager; + private final BufferAllocator allocator; + private static final Logger logger = LogManager.getLogger(BaseFlightProducer.class); + + /** + * Constructs a new BaseFlightProducer. + * + * @param flightClientManager The FlightClientManager to handle client connections. + * @param streamManager The StreamManager to handle stream operations, including + * retrieving and removing streams based on tickets. + * @param allocator The BufferAllocator for memory management in Arrow operations. + */ + public BaseFlightProducer(FlightClientManager flightClientManager, FlightStreamManager streamManager, BufferAllocator allocator) { + this.flightClientManager = flightClientManager; + this.streamManager = streamManager; + this.allocator = allocator; + } + + /** + * Handles the retrieval and streaming of data based on the provided ticket. + * This method orchestrates the entire process of setting up the stream, + * managing backpressure, and handling data flow to the client. + * + * @param context The call context (unused in this implementation) + * @param ticket The ticket containing stream information + * @param listener The server stream listener to handle the data flow + */ + @Override + public void getStream(CallContext context, Ticket ticket, ServerStreamListener listener) { + StreamTicket streamTicket = streamManager.getStreamTicketFactory().fromBytes(ticket.getBytes()); + Optional streamProducerHolder = Optional.empty(); + try { + if (streamTicket.getNodeId().equals(flightClientManager.getLocalNodeId())) { + streamProducerHolder = streamManager.removeStreamProducer(streamTicket); + } else { + Optional remoteClient = flightClientManager.getFlightClient(streamTicket.getNodeId()); + if (remoteClient.isEmpty()) { + listener.error( + CallStatus.UNAVAILABLE.withDescription("Either server is not up yet or node does not support Streams.").cause() + ); + return; + } + StreamProducer proxyProvider = new ProxyStreamProducer(new FlightStreamReader(remoteClient.get().getStream(ticket))); + streamProducerHolder = Optional.of(FlightStreamManager.StreamProducerHolder.create(proxyProvider, allocator)); + } + if (streamProducerHolder.isEmpty()) { + listener.error(CallStatus.NOT_FOUND.withDescription("Stream not found").toRuntimeException()); + return; + } + try (StreamProducer producer = streamProducerHolder.get().producer()) { + StreamProducer.BatchedJob batchedJob = producer.createJob(allocator); + if (context.isCancelled()) { + batchedJob.onCancel(); + listener.error(CallStatus.CANCELLED.cause()); + return; + } + listener.setOnCancelHandler(batchedJob::onCancel); + BackpressureStrategy backpressureStrategy = new BaseBackpressureStrategy(null, batchedJob::onCancel); + backpressureStrategy.register(listener); + StreamProducer.FlushSignal flushSignal = (timeout) -> { + BackpressureStrategy.WaitResult result = backpressureStrategy.waitForListener(timeout.millis()); + if (result.equals(BackpressureStrategy.WaitResult.READY)) { + listener.putNext(); + } else if (result.equals(BackpressureStrategy.WaitResult.TIMEOUT)) { + listener.error(CallStatus.TIMED_OUT.cause()); + throw new RuntimeException("Stream deadline exceeded for consumption"); + } else if (result.equals(BackpressureStrategy.WaitResult.CANCELLED)) { + batchedJob.onCancel(); + listener.error(CallStatus.CANCELLED.cause()); + throw new RuntimeException("Stream cancelled by client"); + } else if (result.equals(BackpressureStrategy.WaitResult.OTHER)) { + batchedJob.onCancel(); + listener.error(CallStatus.INTERNAL.toRuntimeException()); + throw new RuntimeException("Error while waiting for client: " + result); + } else { + batchedJob.onCancel(); + listener.error(CallStatus.INTERNAL.toRuntimeException()); + throw new RuntimeException("Error while waiting for client: " + result); + } + }; + try (VectorSchemaRoot root = streamProducerHolder.get().getRoot()) { + listener.start(root); + batchedJob.run(root, flushSignal); + } + listener.completed(); + } + } catch (Exception e) { + listener.error(CallStatus.INTERNAL.withDescription(e.getMessage()).withCause(e).cause()); + logger.error(e); + throw new RuntimeException(e); + } + } + + /** + * Retrieves FlightInfo for the given FlightDescriptor, handling both local and remote cases. + * + * @param context The call context + * @param descriptor The FlightDescriptor containing stream information + * @return FlightInfo for the requested stream + */ + @Override + public FlightInfo getFlightInfo(CallContext context, FlightDescriptor descriptor) { + // TODO: this api should only be used internally + StreamTicket streamTicket = streamManager.getStreamTicketFactory().fromBytes(descriptor.getCommand()); + if (streamTicket.getNodeId().equals(flightClientManager.getLocalNodeId())) { + Optional streamProducerHolder = streamManager.getStreamProducer(streamTicket); + if (streamProducerHolder.isEmpty()) { + throw CallStatus.NOT_FOUND.withDescription("FlightInfo not found").toRuntimeException(); + } + Location location = flightClientManager.getFlightClientLocation(streamTicket.getNodeId()); + if (location == null) { + throw CallStatus.UNAVAILABLE.withDescription("Internal error while determining location information from ticket.") + .toRuntimeException(); + } + FlightEndpoint endpoint = new FlightEndpoint(new Ticket(descriptor.getCommand()), location); + FlightInfo.Builder infoBuilder; + try { + infoBuilder = FlightInfo.builder( + streamProducerHolder.get().getRoot().getSchema(), + descriptor, + Collections.singletonList(endpoint) + ).setRecords(streamProducerHolder.get().producer().estimatedRowCount()); + } catch (Exception e) { + throw CallStatus.INTERNAL.withDescription("Internal error while creating VectorSchemaRoot.").toRuntimeException(); + } + return infoBuilder.build(); + } else { + Optional remoteClient = flightClientManager.getFlightClient(streamTicket.getNodeId()); + if (remoteClient.isEmpty()) { + throw CallStatus.UNAVAILABLE.withDescription("Client doesn't support Stream").toRuntimeException(); + } + return remoteClient.get().getInfo(descriptor); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamManager.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamManager.java new file mode 100644 index 0000000000000..6755483175570 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamManager.java @@ -0,0 +1,210 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.flight.OSFlightClient; +import org.apache.arrow.flight.Ticket; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.arrow.flight.bootstrap.FlightClientManager; +import org.opensearch.arrow.spi.StreamManager; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.arrow.spi.StreamReader; +import org.opensearch.arrow.spi.StreamTicket; +import org.opensearch.arrow.spi.StreamTicketFactory; +import org.opensearch.common.SetOnce; +import org.opensearch.common.cache.Cache; +import org.opensearch.common.cache.CacheBuilder; +import org.opensearch.common.cache.RemovalListener; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.tasks.TaskId; + +import java.io.IOException; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Supplier; + +/** + * FlightStreamManager is a concrete implementation of StreamManager that provides + * an abstraction layer for managing Arrow Flight streams in OpenSearch. + * It encapsulates the details of Flight client operations, allowing consumers to + * work with streams without direct exposure to Flight internals. + */ +public class FlightStreamManager implements StreamManager { + private static final Logger logger = LogManager.getLogger(FlightStreamManager.class); + + private FlightStreamTicketFactory ticketFactory; + private FlightClientManager clientManager; + private final Supplier allocatorSupplier; + private final Cache streamProducers; + // TODO read from setting + private static final TimeValue DEFAULT_CACHE_EXPIRE = TimeValue.timeValueMinutes(10); + private static final int MAX_WEIGHT = 1000; + + /** + * Holds a StreamProducer along with its metadata and resources + */ + public record StreamProducerHolder(StreamProducer producer, BufferAllocator allocator, long creationTime, SetOnce< + VectorSchemaRoot> root) { + public StreamProducerHolder { + Objects.requireNonNull(producer, "StreamProducer cannot be null"); + Objects.requireNonNull(allocator, "BufferAllocator cannot be null"); + } + + static StreamProducerHolder create(StreamProducer producer, BufferAllocator allocator) { + return new StreamProducerHolder(producer, allocator, System.currentTimeMillis(), new SetOnce<>()); + } + + boolean isExpired() { + return System.currentTimeMillis() - creationTime > producer.getJobDeadline().getMillis(); + } + + /** + * Gets the VectorSchemaRoot associated with the StreamProducer. + * If the root is not set, it creates a new one using the provided BufferAllocator. + */ + public VectorSchemaRoot getRoot() throws Exception { + root.trySet(producer.createRoot(allocator)); + return root.get(); + } + } + + /** + * Constructs a new FlightStreamManager. + * @param allocatorSupplier The supplier for BufferAllocator instances used for memory management. + * This parameter is required to be non-null. + + */ + public FlightStreamManager(Supplier allocatorSupplier) { + this.allocatorSupplier = allocatorSupplier; + RemovalListener onProducerRemoval = (notification) -> { + String ticketId = notification.getKey(); + StreamProducerHolder holder = notification.getValue(); + if (holder != null) { + try { + holder.producer.close(); + logger.debug("Closed stream producer for ticket [{}], reason: {}", ticketId, notification.getRemovalReason()); + } catch (Exception e) { + logger.warn("Error closing stream producer for ticket [{}]. {}", ticketId, e.getMessage()); + } + } + }; + this.streamProducers = CacheBuilder.builder() + .setExpireAfterWrite(DEFAULT_CACHE_EXPIRE) + .setMaximumWeight(MAX_WEIGHT) + .removalListener(onProducerRemoval) + .build(); + } + + /** + * Sets the FlightClientManager for this FlightStreamManager. + * @param clientManager The FlightClientManager instance to use for Flight client operations. + * This parameter is required to be non-null. + */ + public void setClientManager(FlightClientManager clientManager) { + this.clientManager = clientManager; + this.ticketFactory = new FlightStreamTicketFactory(clientManager::getLocalNodeId); + } + + /** + * Registers a new stream producer with the StreamManager. + * @param provider The StreamProducer instance to register. + * @param parentTaskId The parent task ID associated with the stream. + * @return A StreamTicket representing the registered stream. + */ + @Override + public StreamTicket registerStream(StreamProducer provider, TaskId parentTaskId) { + Objects.requireNonNull(provider, "StreamProducer cannot be null"); + StreamTicket ticket = ticketFactory.newTicket(); + streamProducers.put(ticket.getTicketId(), StreamProducerHolder.create(provider, allocatorSupplier.get())); + return ticket; + } + + /** + * Retrieves a StreamReader for the given StreamTicket. + * @param ticket The StreamTicket representing the stream to retrieve. + * @return A StreamReader instance for the specified stream. + */ + @Override + public StreamReader getStreamReader(StreamTicket ticket) { + Optional flightClient = clientManager.getFlightClient(ticket.getNodeId()); + if (flightClient.isEmpty()) { + throw new RuntimeException("Flight client not found for node [" + ticket.getNodeId() + "]."); + } + FlightStream stream = flightClient.get().getStream(new Ticket(ticket.toBytes())); + return new FlightStreamReader(stream); + } + + /** + * Retrieves the StreamTicketFactory used by this StreamManager. + * @return The StreamTicketFactory instance associated with this StreamManager. + */ + @Override + public StreamTicketFactory getStreamTicketFactory() { + return ticketFactory; + } + + /** + * Gets the StreamProducer associated with a ticket if it hasn't expired based on its deadline. + * + * @param ticket The StreamTicket identifying the stream + * @return Optional of StreamProducerHolder containing the producer if found and not expired + */ + public Optional getStreamProducer(StreamTicket ticket) { + Objects.requireNonNull(ticket, "StreamTicket cannot be null"); + StreamProducerHolder holder = streamProducers.get(ticket.getTicketId()); + if (holder != null) { + if (holder.isExpired()) { + removeStreamProducer(ticket); + return Optional.empty(); + } + return Optional.of(holder); + } + return Optional.empty(); + } + + /** + * Gets and removes the StreamProducer associated with a ticket. + * Ensure that close is called on the StreamProducer after use. + * @param ticket The StreamTicket identifying the stream + * @return Optional of StreamProducerHolder containing the producer if found + */ + public Optional removeStreamProducer(StreamTicket ticket) { + Objects.requireNonNull(ticket, "StreamTicket cannot be null"); + + String ticketId = ticket.getTicketId(); + StreamProducerHolder holder = streamProducers.get(ticketId); + + if (holder != null) { + streamProducers.invalidate(ticketId); + return Optional.of(holder); + } + return Optional.empty(); + } + + /** + * Closes the StreamManager and cancels all associated streams. + * This method should be called when the StreamManager is no longer needed to clean up resources. + * It is recommended to implement this method to cancel all threads and clear the streamManager queue. + */ + @Override + public void close() throws Exception { + streamProducers.values().forEach(holder -> { + try { + holder.producer().close(); + } catch (IOException e) { + logger.error("Error closing stream producer, this may cause memory leaks.", e); + } + }); + streamProducers.invalidateAll(); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamReader.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamReader.java new file mode 100644 index 0000000000000..6f9f7ce408802 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamReader.java @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.FlightRuntimeException; +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.opensearch.ExceptionsHelper; +import org.opensearch.arrow.spi.StreamReader; + +/** + * FlightStreamReader is a wrapper class that adapts the FlightStream interface + * to the StreamReader interface. + */ +public class FlightStreamReader implements StreamReader { + + private final FlightStream flightStream; + + /** + * Constructs a FlightStreamReader with the given FlightStream. + * + * @param flightStream The FlightStream to be adapted. + */ + public FlightStreamReader(FlightStream flightStream) { + this.flightStream = flightStream; + } + + /** + * Moves the flightStream to the next batch of data. + * @return true if there is a next batch of data, false otherwise. + * @throws FlightRuntimeException if an error occurs while advancing to the next batch like early termination of stream + */ + @Override + public boolean next() throws FlightRuntimeException { + return flightStream.next(); + } + + /** + * Returns the VectorSchemaRoot containing the current batch of data. + * @return The VectorSchemaRoot containing the current batch of data. + * @throws FlightRuntimeException if an error occurs while retrieving the root like early termination of stream + */ + @Override + public VectorSchemaRoot getRoot() throws FlightRuntimeException { + return flightStream.getRoot(); + } + + /** + * Closes the flightStream. + */ + @Override + public void close() { + ExceptionsHelper.catchAsRuntimeException(flightStream::close); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamTicket.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamTicket.java new file mode 100644 index 0000000000000..baa9e79fec6a1 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamTicket.java @@ -0,0 +1,111 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.opensearch.arrow.spi.StreamTicket; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Objects; + +class FlightStreamTicket implements StreamTicket { + private static final int MAX_TOTAL_SIZE = 4096; + private static final int MAX_ID_LENGTH = 256; + + private final String ticketID; + private final String nodeID; + + public FlightStreamTicket(String ticketID, String nodeID) { + this.ticketID = ticketID; + this.nodeID = nodeID; + } + + @Override + public String getTicketId() { + return ticketID; + } + + @Override + public String getNodeId() { + return nodeID; + } + + @Override + public byte[] toBytes() { + byte[] ticketIDBytes = ticketID.getBytes(StandardCharsets.UTF_8); + byte[] nodeIDBytes = nodeID.getBytes(StandardCharsets.UTF_8); + + if (ticketIDBytes.length > Short.MAX_VALUE || nodeIDBytes.length > Short.MAX_VALUE) { + throw new IllegalArgumentException("Field lengths exceed the maximum allowed size."); + } + ByteBuffer buffer = ByteBuffer.allocate(2 + ticketIDBytes.length + 2 + nodeIDBytes.length); + buffer.putShort((short) ticketIDBytes.length); + buffer.putShort((short) nodeIDBytes.length); + buffer.put(ticketIDBytes); + buffer.put(nodeIDBytes); + return Base64.getEncoder().encode(buffer.array()); + } + + static StreamTicket fromBytes(byte[] bytes) { + if (bytes == null || bytes.length < 4) { + throw new IllegalArgumentException("Invalid byte array input."); + } + + if (bytes.length > MAX_TOTAL_SIZE) { + throw new IllegalArgumentException("Input exceeds maximum allowed size"); + } + + ByteBuffer buffer = ByteBuffer.wrap(Base64.getDecoder().decode(bytes)); + + short ticketIDLength = buffer.getShort(); + if (ticketIDLength < 0 || ticketIDLength > MAX_ID_LENGTH) { + throw new IllegalArgumentException("Invalid ticketID length: " + ticketIDLength); + } + + short nodeIDLength = buffer.getShort(); + if (nodeIDLength < 0 || nodeIDLength > MAX_ID_LENGTH) { + throw new IllegalArgumentException("Invalid nodeID length: " + nodeIDLength); + } + + byte[] ticketIDBytes = new byte[ticketIDLength]; + if (buffer.remaining() < ticketIDLength) { + throw new IllegalArgumentException("Malformed byte array. Not enough data for TicketId."); + } + buffer.get(ticketIDBytes); + + byte[] nodeIDBytes = new byte[nodeIDLength]; + if (buffer.remaining() < nodeIDLength) { + throw new IllegalArgumentException("Malformed byte array. Not enough data for NodeId."); + } + buffer.get(nodeIDBytes); + + String ticketID = new String(ticketIDBytes, StandardCharsets.UTF_8); + String nodeID = new String(nodeIDBytes, StandardCharsets.UTF_8); + return new FlightStreamTicket(ticketID, nodeID); + } + + @Override + public int hashCode() { + return Objects.hash(ticketID, nodeID); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + FlightStreamTicket that = (FlightStreamTicket) obj; + return Objects.equals(ticketID, that.ticketID) && Objects.equals(nodeID, that.nodeID); + } + + @Override + public String toString() { + return "FlightStreamTicket{ticketID='" + ticketID + "', nodeID='" + nodeID + "'}"; + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamTicketFactory.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamTicketFactory.java new file mode 100644 index 0000000000000..473eb92cf2db3 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamTicketFactory.java @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.opensearch.arrow.spi.StreamTicket; +import org.opensearch.arrow.spi.StreamTicketFactory; +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.UUID; +import java.util.function.Supplier; + +/** + * Default implementation of StreamTicketFactory + */ +@ExperimentalApi +public class FlightStreamTicketFactory implements StreamTicketFactory { + + private final Supplier nodeId; + + /** + * Constructs a new DefaultStreamTicketFactory instance. + * + * @param nodeId A Supplier that provides the node ID for the StreamTicket + */ + public FlightStreamTicketFactory(Supplier nodeId) { + this.nodeId = nodeId; + } + + /** + * Creates a new StreamTicket with a unique ticket ID. + * + * @return A new StreamTicket instance + */ + @Override + public StreamTicket newTicket() { + return new FlightStreamTicket(generateUniqueTicket(), nodeId.get()); + } + + /** + * Deserializes a StreamTicket from its byte representation. + * + * @param bytes The byte array containing the serialized ticket data + * @return A StreamTicket instance reconstructed from the byte array + * @throws IllegalArgumentException if bytes is null or invalid + */ + @Override + public StreamTicket fromBytes(byte[] bytes) { + return FlightStreamTicket.fromBytes(bytes); + } + + private String generateUniqueTicket() { + return UUID.randomUUID().toString(); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/ProxyStreamProducer.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/ProxyStreamProducer.java new file mode 100644 index 0000000000000..65e6c793cb3ea --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/ProxyStreamProducer.java @@ -0,0 +1,123 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.opensearch.ExceptionsHelper; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.arrow.spi.StreamReader; +import org.opensearch.arrow.spi.StreamTicket; +import org.opensearch.common.unit.TimeValue; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * ProxyStreamProvider acts as forward proxy for FlightStream. + * It creates a BatchedJob to handle the streaming of data from the remote FlightStream. + * This is useful when stream is not present locally and needs to be fetched from a node + * retrieved using {@link StreamTicket#getNodeId()} where it is present. + */ +public class ProxyStreamProducer implements StreamProducer { + + private final StreamReader remoteStream; + + /** + * Constructs a new ProxyStreamProducer instance. + * + * @param remoteStream The remote FlightStream to be proxied. + */ + public ProxyStreamProducer(StreamReader remoteStream) { + this.remoteStream = remoteStream; + } + + /** + * Creates a VectorSchemaRoot for the remote FlightStream. + * @param allocator The allocator to use for creating vectors + * @return A VectorSchemaRoot representing the schema of the remote FlightStream + */ + @Override + public VectorSchemaRoot createRoot(BufferAllocator allocator) throws Exception { + return remoteStream.getRoot(); + } + + /** + * Creates a BatchedJob + * @param allocator The allocator to use for any additional memory allocations + */ + @Override + public BatchedJob createJob(BufferAllocator allocator) { + return new ProxyBatchedJob(remoteStream); + } + + /** + * Returns the deadline for the remote FlightStream. + * Since the stream is not present locally, the deadline is set to -1. It piggybacks on remote stream expiration + * @return The deadline for the remote FlightStream + */ + @Override + public TimeValue getJobDeadline() { + return TimeValue.MINUS_ONE; + } + + /** + * Provides an estimate of the total number of rows that will be produced. + */ + @Override + public int estimatedRowCount() { + // TODO get it from remote flight stream + return -1; + } + + /** + * Task action name + */ + @Override + public String getAction() { + // TODO get it from remote flight stream + return ""; + } + + /** + * Closes the remote FlightStream. + */ + @Override + public void close() { + ExceptionsHelper.catchAsRuntimeException(remoteStream::close); + } + + static class ProxyBatchedJob implements BatchedJob { + + private final StreamReader remoteStream; + private final AtomicBoolean isCancelled = new AtomicBoolean(false); + + ProxyBatchedJob(StreamReader remoteStream) { + this.remoteStream = remoteStream; + } + + @Override + public void run(VectorSchemaRoot root, FlushSignal flushSignal) throws Exception { + while (!isCancelled.get() && remoteStream.next()) { + flushSignal.awaitConsumption(TimeValue.timeValueMillis(1000)); + } + } + + @Override + public void onCancel() { + isCancelled.set(true); + } + + @Override + public boolean isCancelled() { + // Proxy stream don't have any business logic to set this flag, + // they piggyback on remote stream getting cancelled. + return isCancelled.get(); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/package-info.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/package-info.java new file mode 100644 index 0000000000000..90ca54b44a55d --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Core components and implementations for OpenSearch Flight service, including base producers and consumers. + */ +package org.opensearch.arrow.flight.impl; diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/package-info.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/package-info.java new file mode 100644 index 0000000000000..2341a24d0be85 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Root package for OpenSearch Flight functionality, providing core flight service integration with OpenSearch. + */ +package org.opensearch.arrow.flight; diff --git a/plugins/arrow-flight-rpc/src/main/plugin-metadata/plugin-security.policy b/plugins/arrow-flight-rpc/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000000000..984c7a6de89b9 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +grant codeBase "${codebase.grpc-netty-shaded}" { + permission java.net.SocketPermission "*", "accept,connect,listen,resolve"; + permission java.lang.RuntimePermission "*", "setContextClassLoader"; +}; + +grant codeBase "${codebase.grpc-core}" { + permission java.net.SocketPermission "*", "accept,connect,listen,resolve"; + permission java.lang.RuntimePermission "*", "setContextClassLoader"; +}; + +grant codeBase "${codebase.arrow-flight-rpc}" { + // arrow flight service permissions + permission java.util.PropertyPermission "arrow.allocation.manager.type", "write"; + permission java.util.PropertyPermission "arrow.enable_null_check_for_get", "write"; + permission java.util.PropertyPermission "arrow.enable_unsafe_memory_access", "write"; + permission java.util.PropertyPermission "arrow.memory.debug.allocator", "write"; + + permission java.util.PropertyPermission "io.netty.tryReflectionSetAccessible", "write"; + permission java.util.PropertyPermission "io.netty.allocator.numDirectArenas", "write"; + permission java.util.PropertyPermission "io.netty.noUnsafe", "write"; + permission java.util.PropertyPermission "io.netty.tryUnsafe", "write"; + + // Needed for netty based arrow flight server for netty configs related to buffer allocator + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + permission java.util.PropertyPermission "arrow.allocation.manager.type", "write"; + + permission java.lang.RuntimePermission "modifyThreadGroup"; + permission java.lang.RuntimePermission "modifyThread"; + permission java.net.SocketPermission "*", "accept,connect,listen,resolve"; + + // Reflection access needed by Arrow + permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + + // Memory access + permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; +}; diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/FlightStreamPluginTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/FlightStreamPluginTests.java new file mode 100644 index 0000000000000..80a6a3a12fc20 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/FlightStreamPluginTests.java @@ -0,0 +1,134 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight; + +import org.opensearch.arrow.flight.api.flightinfo.FlightServerInfoAction; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoAction; +import org.opensearch.arrow.flight.bootstrap.FlightService; +import org.opensearch.arrow.spi.StreamManager; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.plugins.SecureTransportSettingsProvider; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ExecutorBuilder; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.function.Supplier; + +import static org.opensearch.common.util.FeatureFlags.ARROW_STREAMS_SETTING; +import static org.opensearch.plugins.NetworkPlugin.AuxTransport.AUX_TRANSPORT_TYPES_KEY; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class FlightStreamPluginTests extends OpenSearchTestCase { + private Settings settings; + private ClusterService clusterService; + + @Override + public void setUp() throws Exception { + super.setUp(); + settings = Settings.builder().put(ARROW_STREAMS_SETTING.getKey(), true).build(); + clusterService = mock(ClusterService.class); + ClusterState clusterState = mock(ClusterState.class); + DiscoveryNodes nodes = mock(DiscoveryNodes.class); + when(clusterService.state()).thenReturn(clusterState); + when(clusterState.nodes()).thenReturn(nodes); + when(nodes.getLocalNodeId()).thenReturn("test-node"); + } + + public void testPluginEnableAndDisable() throws IOException { + + Settings disabledSettings = Settings.builder().put(ARROW_STREAMS_SETTING.getKey(), false).build(); + FeatureFlags.initializeFeatureFlags(disabledSettings); + FlightStreamPlugin disabledPlugin = new FlightStreamPlugin(disabledSettings); + + Collection disabledPluginComponents = disabledPlugin.createComponents( + null, + clusterService, + mock(ThreadPool.class), + null, + null, + null, + null, + null, + null, + null, + null + ); + + assertTrue(disabledPluginComponents.isEmpty()); + assertNull(disabledPlugin.getStreamManager().get()); + assertTrue(disabledPlugin.getExecutorBuilders(disabledSettings).isEmpty()); + assertNotNull(disabledPlugin.getSettings()); + assertTrue(disabledPlugin.getSettings().isEmpty()); + assertNotNull(disabledPlugin.getSecureTransports(null, null, null, null, null, null, null, null)); + assertTrue(disabledPlugin.getAuxTransports(null, null, null, null, null, null).isEmpty()); + assertEquals(0, disabledPlugin.getRestHandlers(null, null, null, null, null, null, null).size()); + assertEquals(0, disabledPlugin.getActions().size()); + + disabledPlugin.close(); + + FeatureFlags.initializeFeatureFlags(settings); + FeatureFlagSetter.set(ARROW_STREAMS_SETTING.getKey()); + FlightStreamPlugin plugin = new FlightStreamPlugin(settings); + Collection components = plugin.createComponents( + null, + clusterService, + mock(ThreadPool.class), + null, + null, + null, + null, + null, + null, + null, + null + ); + + assertNotNull(components); + assertFalse(components.isEmpty()); + assertEquals(1, components.size()); + assertTrue(components.iterator().next() instanceof FlightService); + + List> executorBuilders = plugin.getExecutorBuilders(settings); + assertNotNull(executorBuilders); + assertFalse(executorBuilders.isEmpty()); + assertEquals(2, executorBuilders.size()); + + Supplier streamManager = plugin.getStreamManager(); + assertNotNull(streamManager); + + List> settings = plugin.getSettings(); + assertNotNull(settings); + assertFalse(settings.isEmpty()); + + assertNotNull(plugin.getSecureTransports(null, null, null, null, null, null, mock(SecureTransportSettingsProvider.class), null)); + + assertTrue( + plugin.getAuxTransports(null, null, null, new NetworkService(List.of()), null, null) + .get(AUX_TRANSPORT_TYPES_KEY) + .get() instanceof FlightService + ); + assertEquals(1, plugin.getRestHandlers(null, null, null, null, null, null, null).size()); + assertTrue(plugin.getRestHandlers(null, null, null, null, null, null, null).get(0) instanceof FlightServerInfoAction); + assertEquals(1, plugin.getActions().size()); + assertEquals(NodesFlightInfoAction.INSTANCE.name(), plugin.getActions().get(0).getAction().name()); + + plugin.close(); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/FlightServerInfoActionTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/FlightServerInfoActionTests.java new file mode 100644 index 0000000000000..d3115fc745475 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/FlightServerInfoActionTests.java @@ -0,0 +1,101 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.SetOnce; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.rest.FakeRestRequest; +import org.opensearch.test.rest.RestActionTestCase; +import org.junit.Before; + +import java.util.Collections; + +import static org.mockito.Mockito.mock; + +public class FlightServerInfoActionTests extends RestActionTestCase { + private FlightServerInfoAction handler; + + @Before + public void setUpAction() { + handler = new FlightServerInfoAction(); + controller().registerHandler(handler); + } + + public void testGetName() { + assertEquals("flight_server_info_action", handler.getName()); + } + + public void testRoutes() { + var routes = handler.routes(); + assertEquals(2, routes.size()); + assertTrue( + routes.stream().anyMatch(route -> route.getPath().equals("/_flight/info") && route.getMethod() == RestRequest.Method.GET) + ); + assertTrue( + routes.stream() + .anyMatch(route -> route.getPath().equals("/_flight/info/{nodeId}") && route.getMethod() == RestRequest.Method.GET) + ); + } + + public void testFlightInfoRequest() { + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.GET) + .withPath("/_flight/info") + .build(); + SetOnce executeCalled = new SetOnce<>(); + verifyingClient.setExecuteVerifier((action, actionRequest) -> { + assertEquals(NodesFlightInfoAction.INSTANCE.name(), action.name()); + assertNotNull(actionRequest); + executeCalled.set(true); + return new NodesFlightInfoResponse( + new ClusterName("test-cluster"), + Collections.singletonList(new NodeFlightInfo(mock(DiscoveryNode.class), mock(BoundTransportAddress.class))), + Collections.emptyList() + ); + }); + dispatchRequest(request); + assertEquals(Boolean.TRUE, executeCalled.get()); + } + + public void testFlightInfoRequestWithNodeId() throws Exception { + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.GET) + .withPath("/_flight/info/local_node") + .build(); + SetOnce executeCalled = new SetOnce<>(); + verifyingClient.setExecuteVerifier((action, actionRequest) -> { + assertEquals(NodesFlightInfoAction.INSTANCE.name(), action.name()); + assertNotNull(actionRequest); + executeCalled.set(true); + return null; + }); + dispatchRequest(request); + assertEquals(Boolean.TRUE, executeCalled.get()); + } + + public void testFlightInfoRequestWithInvalidPath() throws Exception { + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.GET) + .withPath("/_flight/invalid_path") + .build(); + SetOnce executeCalled = new SetOnce<>(); + verifyingClient.setExecuteVerifier((action, actionRequest) -> { + assertEquals(NodesFlightInfoAction.INSTANCE.name(), action.name()); + assertNotNull(actionRequest); + executeCalled.set(true); + return new NodesFlightInfoResponse( + new ClusterName("test-cluster"), + Collections.singletonList(new NodeFlightInfo(mock(DiscoveryNode.class), mock(BoundTransportAddress.class))), + Collections.emptyList() + ); + }); + dispatchRequest(request); + assertNull(executeCalled.get()); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodeFlightInfoTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodeFlightInfoTests.java new file mode 100644 index 0000000000000..59e695313c16e --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodeFlightInfoTests.java @@ -0,0 +1,160 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.Version; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +import java.net.InetAddress; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +@SuppressWarnings("unchecked") +public class NodeFlightInfoTests extends OpenSearchTestCase { + + public void testNodeFlightInfoSerialization() throws Exception { + DiscoveryNode node = new DiscoveryNode( + "test_node", + "test_node", + "hostname", + "localhost", + "127.0.0.1", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + new HashMap<>(), + new HashSet<>(), + Version.CURRENT + ); + + TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 47470); + BoundTransportAddress boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + + NodeFlightInfo originalInfo = new NodeFlightInfo(node, boundAddress); + + BytesStreamOutput output = new BytesStreamOutput(); + originalInfo.writeTo(output); + + StreamInput input = output.bytes().streamInput(); + NodeFlightInfo deserializedInfo = new NodeFlightInfo(input); + + assertEquals(originalInfo.getNode(), deserializedInfo.getNode()); + assertEquals(originalInfo.getBoundAddress().boundAddresses().length, deserializedInfo.getBoundAddress().boundAddresses().length); + assertEquals(originalInfo.getBoundAddress().boundAddresses()[0], deserializedInfo.getBoundAddress().boundAddresses()[0]); + assertEquals(originalInfo.getBoundAddress().publishAddress(), deserializedInfo.getBoundAddress().publishAddress()); + } + + public void testNodeFlightInfoEquality() throws Exception { + DiscoveryNode node = new DiscoveryNode( + "test_node", + "test_node", + "hostname", + "localhost", + "127.0.0.1", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + new HashMap<>(), + new HashSet<>(), + Version.CURRENT + ); + + TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 47470); + BoundTransportAddress boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + + NodeFlightInfo info1 = new NodeFlightInfo(node, boundAddress); + NodeFlightInfo info2 = new NodeFlightInfo(node, boundAddress); + + assertEquals(info1.getBoundAddress(), info2.getBoundAddress()); + } + + public void testGetters() throws Exception { + DiscoveryNode node = new DiscoveryNode( + "test_node", + "test_node", + "hostname", + "localhost", + "127.0.0.1", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + new HashMap<>(), + new HashSet<>(), + Version.CURRENT + ); + + TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 47470); + BoundTransportAddress boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + + NodeFlightInfo info = new NodeFlightInfo(node, boundAddress); + + assertEquals(node, info.getNode()); + assertEquals(boundAddress, info.getBoundAddress()); + } + + public void testToXContent() throws Exception { + TransportAddress boundAddress1 = new TransportAddress(InetAddress.getLoopbackAddress(), 47470); + TransportAddress boundAddress2 = new TransportAddress(InetAddress.getLoopbackAddress(), 47471); + TransportAddress publishAddress = new TransportAddress(InetAddress.getLoopbackAddress(), 47472); + + BoundTransportAddress boundAddress = new BoundTransportAddress( + new TransportAddress[] { boundAddress1, boundAddress2 }, + publishAddress + ); + + NodeFlightInfo info = new NodeFlightInfo( + new DiscoveryNode( + "test_node", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + Collections.emptyMap(), + Collections.emptySet(), + Version.CURRENT + ), + boundAddress + ); + + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + builder.field("node_info"); + info.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map responseMap = parser.map(); + + Map nodeInfo = (Map) responseMap.get("node_info"); + assertNotNull("node_info object should exist", nodeInfo); + + Map flightServer = (Map) nodeInfo.get("flight_server"); + assertNotNull("flight_server object should exist", flightServer); + + List> boundAddresses = (List>) flightServer.get("bound_addresses"); + assertNotNull("bound_addresses array should exist", boundAddresses); + assertEquals("Should have 2 bound addresses", 2, boundAddresses.size()); + + assertEquals("localhost", boundAddresses.get(0).get("host")); + assertEquals(47470, boundAddresses.get(0).get("port")); + + assertEquals("localhost", boundAddresses.get(1).get("host")); + assertEquals(47471, boundAddresses.get(1).get("port")); + + Map publishAddressMap = (Map) flightServer.get("publish_address"); + assertNotNull("publish_address object should exist", publishAddressMap); + assertEquals("localhost", publishAddressMap.get("host")); + assertEquals(47472, publishAddressMap.get("port")); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoRequestTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoRequestTests.java new file mode 100644 index 0000000000000..ef8f88b78c3ee --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoRequestTests.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; + +public class NodesFlightInfoRequestTests extends OpenSearchTestCase { + + public void testNodesFlightInfoRequestSerialization() throws Exception { + NodesFlightInfoRequest originalRequest = new NodesFlightInfoRequest("node1", "node2"); + + BytesStreamOutput output = new BytesStreamOutput(); + originalRequest.writeTo(output); + + StreamInput input = output.bytes().streamInput(); + NodesFlightInfoRequest deserializedRequest = new NodesFlightInfoRequest(input); + + assertArrayEquals(originalRequest.nodesIds(), deserializedRequest.nodesIds()); + } + + public void testNodesFlightInfoRequestConcreteNodes() { + String[] nodeIds = new String[] { "node1", "node2" }; + NodesFlightInfoRequest request = new NodesFlightInfoRequest(nodeIds); + assertArrayEquals(nodeIds, request.nodesIds()); + } + + public void testNodesFlightInfoRequestAllNodes() { + NodesFlightInfoRequest request = new NodesFlightInfoRequest(); + assertEquals(0, request.nodesIds().length); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoResponseTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoResponseTests.java new file mode 100644 index 0000000000000..707a222fe381f --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoResponseTests.java @@ -0,0 +1,241 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.Version; +import org.opensearch.action.FailedNodeException; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +import java.net.ConnectException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +@SuppressWarnings("unchecked") +public class NodesFlightInfoResponseTests extends OpenSearchTestCase { + + public void testNodesFlightInfoResponseSerialization() throws Exception { + ClusterName clusterName = new ClusterName("test-cluster"); + List nodes = new ArrayList<>(); + + DiscoveryNode node1 = createTestNode("node1"); + DiscoveryNode node2 = createTestNode("node2"); + + nodes.add(createNodeFlightInfo(node1, 47470)); + nodes.add(createNodeFlightInfo(node2, 47471)); + + NodesFlightInfoResponse originalResponse = new NodesFlightInfoResponse(clusterName, nodes, List.of()); + + BytesStreamOutput output = new BytesStreamOutput(); + originalResponse.writeTo(output); + + StreamInput input = output.bytes().streamInput(); + NodesFlightInfoResponse deserializedResponse = new NodesFlightInfoResponse(input); + assertEquals(originalResponse.getNodes().size(), deserializedResponse.getNodes().size()); + + for (int i = 0; i < originalResponse.getNodes().size(); i++) { + NodeFlightInfo originalNode = originalResponse.getNodes().get(i); + NodeFlightInfo deserializedNode = deserializedResponse.getNodes().get(i); + + assertEquals(originalNode.getNode().getId(), deserializedNode.getNode().getId()); + assertEquals(originalNode.getNode().getName(), deserializedNode.getNode().getName()); + assertEquals(originalNode.getBoundAddress().publishAddress(), deserializedNode.getBoundAddress().publishAddress()); + } + assertEquals(originalResponse.getClusterName(), deserializedResponse.getClusterName()); + } + + public void testNodesFlightInfoResponseEmpty() { + ClusterName clusterName = new ClusterName("test-cluster"); + List nodes = new ArrayList<>(); + + NodesFlightInfoResponse response = new NodesFlightInfoResponse(clusterName, nodes, List.of()); + + assertTrue(response.getNodes().isEmpty()); + assertEquals(clusterName, response.getClusterName()); + } + + public void testToXContentWithFailures() throws Exception { + NodesFlightInfoResponse response = getNodesFlightInfoResponse(); + + XContentBuilder builder = JsonXContent.contentBuilder(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map responseMap = parser.map(); + + Map nodesStats = (Map) responseMap.get("_nodes"); + assertNotNull("_nodes object should exist", nodesStats); + assertEquals(2, nodesStats.get("total")); + assertEquals(2, nodesStats.get("successful")); + assertEquals(2, nodesStats.get("failed")); + + assertEquals("test-cluster", responseMap.get("cluster_name")); + + Map nodes = (Map) responseMap.get("nodes"); + assertNotNull("nodes object should exist", nodes); + assertEquals(2, nodes.size()); + + Map firstNode = (Map) nodes.get("successful_node_1"); + assertNotNull(firstNode); + Map firstNodeFlightServer = (Map) firstNode.get("flight_server"); + assertNotNull(firstNodeFlightServer); + Map firstNodePublishAddress = (Map) firstNodeFlightServer.get("publish_address"); + assertEquals("localhost", firstNodePublishAddress.get("host")); + assertEquals(47470, firstNodePublishAddress.get("port")); + + Map secondNode = (Map) nodes.get("successful_node_2"); + assertNotNull(secondNode); + Map secondNodeFlightServer = (Map) secondNode.get("flight_server"); + assertNotNull(secondNodeFlightServer); + Map secondNodePublishAddress = (Map) secondNodeFlightServer.get("publish_address"); + assertEquals("localhost", secondNodePublishAddress.get("host")); + assertEquals(47471, secondNodePublishAddress.get("port")); + + List> failuresList = (List>) responseMap.get("failures"); + assertNotNull("failures array should exist", failuresList); + assertEquals(2, failuresList.size()); + + Map firstFailure = failuresList.get(0); + assertEquals("failed_node_1", firstFailure.get("node_id")); + assertEquals("Connection refused", firstFailure.get("reason")); + + Map secondFailure = failuresList.get(1); + assertEquals("failed_node_2", secondFailure.get("node_id")); + assertEquals("Node not found", secondFailure.get("reason")); + } + } + + private static NodesFlightInfoResponse getNodesFlightInfoResponse() { + DiscoveryNode node1 = new DiscoveryNode( + "successful_node_1", + "successful_node_1", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + Collections.emptyMap(), + Collections.emptySet(), + Version.CURRENT + ); + + List successfulNodes = getNodeFlightInfos(node1); + + return getNodesFlightInfoResponse(successfulNodes); + } + + private static NodesFlightInfoResponse getNodesFlightInfoResponse(List successfulNodes) { + List failures = Arrays.asList( + new FailedNodeException("failed_node_1", "Connection refused", new ConnectException("Connection refused")), + new FailedNodeException("failed_node_2", "Node not found", new Exception("Node not found")) + ); + + return new NodesFlightInfoResponse(new ClusterName("test-cluster"), successfulNodes, failures); + } + + private static List getNodeFlightInfos(DiscoveryNode node1) { + DiscoveryNode node2 = new DiscoveryNode( + "successful_node_2", + "successful_node_2", + new TransportAddress(InetAddress.getLoopbackAddress(), 9301), + Collections.emptyMap(), + Collections.emptySet(), + Version.CURRENT + ); + + TransportAddress address1 = new TransportAddress(InetAddress.getLoopbackAddress(), 47470); + return getNodeFlightInfos(node1, address1, node2); + } + + private static List getNodeFlightInfos(DiscoveryNode node1, TransportAddress address1, DiscoveryNode node2) { + BoundTransportAddress boundAddress1 = new BoundTransportAddress(new TransportAddress[] { address1 }, address1); + + TransportAddress address2 = new TransportAddress(InetAddress.getLoopbackAddress(), 47471); + BoundTransportAddress boundAddress2 = new BoundTransportAddress(new TransportAddress[] { address2 }, address2); + + return Arrays.asList(new NodeFlightInfo(node1, boundAddress1), new NodeFlightInfo(node2, boundAddress2)); + } + + public void testToXContentWithNoFailures() throws Exception { + NodesFlightInfoResponse response = getFlightInfoResponse(); + + XContentBuilder builder = JsonXContent.contentBuilder(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map responseMap = parser.map(); + + Map nodesStats = (Map) responseMap.get("_nodes"); + assertNotNull(nodesStats); + assertEquals(1, nodesStats.get("total")); + assertEquals(1, nodesStats.get("successful")); + assertEquals(0, nodesStats.get("failed")); + + assertEquals("test-cluster", responseMap.get("cluster_name")); + + Map nodes = (Map) responseMap.get("nodes"); + assertNotNull(nodes); + assertEquals(1, nodes.size()); + + assertNull("failures array should not exist", responseMap.get("failures")); + } + } + + private static NodesFlightInfoResponse getFlightInfoResponse() { + DiscoveryNode node = new DiscoveryNode( + "successful_node", + "successful_node", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + Collections.emptyMap(), + Collections.emptySet(), + Version.CURRENT + ); + + TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 47470); + BoundTransportAddress boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + + return new NodesFlightInfoResponse( + new ClusterName("test-cluster"), + Collections.singletonList(new NodeFlightInfo(node, boundAddress)), + Collections.emptyList() + ); + } + + private DiscoveryNode createTestNode(String nodeId) { + return new DiscoveryNode( + nodeId, + nodeId, + "host" + nodeId, + "localhost", + "127.0.0.1", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + new HashMap<>(), + new HashSet<>(), + Version.CURRENT + ); + } + + private NodeFlightInfo createNodeFlightInfo(DiscoveryNode node, int port) { + TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), port); + BoundTransportAddress boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + return new NodeFlightInfo(node, boundAddress); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/TransportNodesFlightInfoActionTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/TransportNodesFlightInfoActionTests.java new file mode 100644 index 0000000000000..6bd70eec4ad3a --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/TransportNodesFlightInfoActionTests.java @@ -0,0 +1,176 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.Version; +import org.opensearch.action.FailedNodeException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.arrow.flight.bootstrap.FlightService; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; +import org.junit.Before; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TransportNodesFlightInfoActionTests extends OpenSearchTestCase { + + private DiscoveryNode localNode; + private TransportNodesFlightInfoAction action; + private BoundTransportAddress boundAddress; + + @Before + public void setUp() throws Exception { + super.setUp(); + + localNode = new DiscoveryNode( + "local_node", + "local_node", + "host", + "localhost", + "127.0.0.1", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + new HashMap<>(), + new HashSet<>(), + Version.CURRENT + ); + + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterName()).thenReturn(new ClusterName("test-cluster")); + when(clusterService.localNode()).thenReturn(localNode); + + TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 47470); + boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + + FlightService flightService = mock(FlightService.class); + when(flightService.getBoundAddress()).thenReturn(boundAddress); + + action = new TransportNodesFlightInfoAction( + Settings.EMPTY, + mock(ThreadPool.class), + clusterService, + mock(TransportService.class), + new ActionFilters(Collections.emptySet()), + flightService + ); + } + + public void testNewResponse() { + NodesFlightInfoRequest request = new NodesFlightInfoRequest(); + List nodeFlightInfos = Collections.singletonList(new NodeFlightInfo(localNode, boundAddress)); + List failures = Collections.emptyList(); + + NodesFlightInfoResponse response = action.newResponse(request, nodeFlightInfos, failures); + + assertNotNull(response); + assertEquals("test-cluster", response.getClusterName().value()); + assertEquals(1, response.getNodes().size()); + assertEquals(0, response.failures().size()); + + NodeFlightInfo nodeInfo = response.getNodes().get(0); + assertEquals(localNode, nodeInfo.getNode()); + assertEquals(boundAddress, nodeInfo.getBoundAddress()); + } + + public void testNewResponseWithFailures() { + NodesFlightInfoRequest request = new NodesFlightInfoRequest(); + List nodeFlightInfos = Collections.emptyList(); + List failures = Collections.singletonList(new FailedNodeException("failed_node", "test failure", null)); + + NodesFlightInfoResponse response = action.newResponse(request, nodeFlightInfos, failures); + + assertNotNull(response); + assertEquals("test-cluster", response.getClusterName().value()); + assertEquals(0, response.getNodes().size()); + assertEquals(1, response.failures().size()); + assertEquals("failed_node", response.failures().get(0).nodeId()); + assertEquals("test failure", response.failures().get(0).getMessage()); + } + + public void testNewNodeRequest() { + NodesFlightInfoRequest request = new NodesFlightInfoRequest("node1", "node2"); + NodesFlightInfoRequest.NodeFlightInfoRequest nodeRequest = action.newNodeRequest(request); + + assertNotNull(nodeRequest); + assertArrayEquals(new String[] { "node1", "node2" }, nodeRequest.request.nodesIds()); + } + + public void testNewNodeResponse() throws IOException { + NodeFlightInfo nodeInfo = new NodeFlightInfo(localNode, boundAddress); + BytesStreamOutput out = new BytesStreamOutput(); + nodeInfo.writeTo(out); + StreamInput in = out.bytes().streamInput(); + + NodeFlightInfo deserializedInfo = action.newNodeResponse(in); + + assertNotNull(deserializedInfo); + assertEquals(nodeInfo.getNode(), deserializedInfo.getNode()); + assertEquals(nodeInfo.getBoundAddress().publishAddress(), deserializedInfo.getBoundAddress().publishAddress()); + } + + public void testNodeOperation() { + NodesFlightInfoRequest.NodeFlightInfoRequest nodeRequest = new NodesFlightInfoRequest.NodeFlightInfoRequest( + new NodesFlightInfoRequest() + ); + + NodeFlightInfo response = action.nodeOperation(nodeRequest); + + assertNotNull(response); + assertEquals(localNode, response.getNode()); + assertEquals(boundAddress.publishAddress(), response.getBoundAddress().publishAddress()); + } + + public void testNodeOperationWithSpecificNodes() throws IOException { + NodesFlightInfoRequest request = new NodesFlightInfoRequest("local_node"); + NodesFlightInfoRequest.NodeFlightInfoRequest nodeRequest = new NodesFlightInfoRequest.NodeFlightInfoRequest(request); + + NodeFlightInfo response = action.nodeOperation(nodeRequest); + + assertNotNull(response); + assertEquals(localNode, response.getNode()); + assertEquals(boundAddress, response.getBoundAddress()); + } + + public void testNodeOperationWithInvalidNode() throws IOException { + NodesFlightInfoRequest request = new NodesFlightInfoRequest("invalid_node"); + NodesFlightInfoRequest.NodeFlightInfoRequest nodeRequest = new NodesFlightInfoRequest.NodeFlightInfoRequest(request); + + NodeFlightInfo response = action.nodeOperation(nodeRequest); + + assertNotNull(response); + assertEquals(localNode, response.getNode()); + assertEquals(boundAddress, response.getBoundAddress()); + } + + public void testSerialization() throws IOException { + NodesFlightInfoRequest request = new NodesFlightInfoRequest("node1", "node2"); + BytesStreamOutput out = new BytesStreamOutput(); + request.writeTo(out); + StreamInput in = out.bytes().streamInput(); + NodesFlightInfoRequest deserializedRequest = new NodesFlightInfoRequest(in); + + assertArrayEquals(request.nodesIds(), deserializedRequest.nodesIds()); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/FlightClientManagerTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/FlightClientManagerTests.java new file mode 100644 index 0000000000000..3e41e07c21164 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/FlightClientManagerTests.java @@ -0,0 +1,402 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.arrow.flight.bootstrap; + +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.OSFlightClient; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.opensearch.Version; +import org.opensearch.arrow.flight.api.flightinfo.NodeFlightInfo; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoAction; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoRequest; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoResponse; +import org.opensearch.arrow.flight.bootstrap.tls.SslContextProvider; +import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import io.grpc.netty.shaded.io.netty.channel.EventLoopGroup; +import io.grpc.netty.shaded.io.netty.util.NettyRuntime; + +import static org.opensearch.arrow.flight.bootstrap.FlightClientManager.LOCATION_TIMEOUT_MS; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.when; + +@SuppressWarnings("unchecked") +public class FlightClientManagerTests extends OpenSearchTestCase { + + private static BufferAllocator allocator; + private static EventLoopGroup elg; + private static ExecutorService executorService; + private static final AtomicInteger port = new AtomicInteger(0); + + private ClusterService clusterService; + private Client client; + private ClusterState state; + private FlightClientManager clientManager; + private ScheduledExecutorService locationUpdaterExecutor; + + @BeforeClass + public static void setupClass() throws Exception { + ServerConfig.init(Settings.EMPTY); + allocator = new RootAllocator(); + elg = ServerConfig.createELG("test-grpc-worker-elg", NettyRuntime.availableProcessors() * 2); + executorService = ServerConfig.createELG("test-grpc-worker", NettyRuntime.availableProcessors() * 2); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + locationUpdaterExecutor = Executors.newScheduledThreadPool(1); + + FeatureFlagSetter.set(FeatureFlags.ARROW_STREAMS_SETTING.getKey()); + clusterService = mock(ClusterService.class); + client = mock(Client.class); + state = getDefaultState(); + when(clusterService.state()).thenReturn(state); + + mockFlightInfoResponse(state.nodes(), 0); + + SslContextProvider sslContextProvider = mock(SslContextProvider.class); + when(sslContextProvider.isSslEnabled()).thenReturn(false); + + ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.executor(ServerConfig.FLIGHT_CLIENT_THREAD_POOL_NAME)).thenReturn(executorService); + clientManager = new FlightClientManager(allocator, clusterService, sslContextProvider, elg, threadPool, client); + ClusterChangedEvent event = new ClusterChangedEvent("test", state, ClusterState.EMPTY_STATE); + clientManager.clusterChanged(event); + assertBusy(() -> { + assertEquals("Flight client isn't built in time limit", 2, clientManager.getClients().size()); + assertTrue("local_node should exist", clientManager.getFlightClient("local_node").isPresent()); + assertNotNull("local_node should exist", clientManager.getFlightClient("local_node").get()); + assertTrue("remote_node should exist", clientManager.getFlightClient("remote_node").isPresent()); + assertNotNull("remote_node should exist", clientManager.getFlightClient("remote_node").get()); + }, 2, TimeUnit.SECONDS); + } + + private void mockFlightInfoResponse(DiscoveryNodes nodes, int sleepDuration) { + doAnswer(invocation -> { + locationUpdaterExecutor.schedule(() -> { + try { + NodesFlightInfoRequest request = invocation.getArgument(1); + ActionListener listener = invocation.getArgument(2); + + List nodeInfos = new ArrayList<>(); + for (DiscoveryNode node : nodes) { + if (request.nodesIds().length == 0 || Arrays.asList(request.nodesIds()).contains(node.getId())) { + int flightPort = getBaseStreamPort() + port.addAndGet(2); + TransportAddress address = new TransportAddress( + InetAddress.getByName(node.getAddress().getAddress()), + flightPort + ); + BoundTransportAddress boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + NodeFlightInfo nodeInfo = new NodeFlightInfo(node, boundAddress); + nodeInfos.add(nodeInfo); + } + } + NodesFlightInfoResponse response = new NodesFlightInfoResponse(ClusterName.DEFAULT, nodeInfos, Collections.emptyList()); + listener.onResponse(response); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + }, sleepDuration, TimeUnit.MILLISECONDS); + return null; + }).when(client).execute(eq(NodesFlightInfoAction.INSTANCE), any(NodesFlightInfoRequest.class), any(ActionListener.class)); + + } + + @Override + public void tearDown() throws Exception { + locationUpdaterExecutor.shutdown(); + super.tearDown(); + clientManager.close(); + } + + private ClusterState getDefaultState() throws Exception { + int testPort = getBasePort() + port.addAndGet(2); + + DiscoveryNode localNode = createNode("local_node", "127.0.0.1", testPort); + DiscoveryNode remoteNode = createNode("remote_node", "127.0.0.2", testPort + 1); + + // Setup initial cluster state + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); + nodesBuilder.add(remoteNode); + nodesBuilder.add(localNode); + nodesBuilder.localNodeId(localNode.getId()); + DiscoveryNodes nodes = nodesBuilder.build(); + + return ClusterState.builder(new ClusterName("test")).nodes(nodes).build(); + } + + private DiscoveryNode createNode(String nodeId, String host, int port) throws Exception { + TransportAddress address = new TransportAddress(InetAddress.getByName(host), port); + Map attributes = new HashMap<>(); + attributes.put("arrow.streams.enabled", "true"); + Set roles = Collections.singleton(DiscoveryNodeRole.DATA_ROLE); + return new DiscoveryNode(nodeId, address, attributes, roles, Version.CURRENT); + } + + @AfterClass + public static void tearClass() { + allocator.close(); + } + + public void testGetFlightClientForExistingNode() { + validateNodes(); + } + + public void testGetFlightClientLocation() { + for (DiscoveryNode node : state.nodes()) { + Location location = clientManager.getFlightClientLocation(node.getId()); + assertNotNull("Flight client location should be returned", location); + assertEquals("Location host should match", node.getHostAddress(), location.getUri().getHost()); + } + } + + public void testGetFlightClientForNonExistentNode() throws Exception { + assertTrue(clientManager.getFlightClient("non_existent_node").isEmpty()); + } + + public void testClusterChangedWithNodesChanged() throws Exception { + DiscoveryNode newNode = createNode("new_node", "127.0.0.3", getBasePort() + port.addAndGet(1)); + DiscoveryNodes.Builder newNodesBuilder = DiscoveryNodes.builder(); + + for (DiscoveryNode node : state.nodes()) { + newNodesBuilder.add(node); + } + newNodesBuilder.localNodeId("local_node"); + // Update cluster state with new node + newNodesBuilder.add(newNode); + DiscoveryNodes newNodes = newNodesBuilder.build(); + + ClusterState newState = ClusterState.builder(new ClusterName("test")).nodes(newNodes).build(); + mockFlightInfoResponse(newNodes, 0); + when(clusterService.state()).thenReturn(newState); + clientManager.clusterChanged(new ClusterChangedEvent("test", newState, state)); + + for (DiscoveryNode node : newState.nodes()) { + assertBusy( + () -> { assertNotNull("Flight client isn't built in time limit", clientManager.getFlightClient(node.getId())); }, + 2, + TimeUnit.SECONDS + ); + } + } + + public void testClusterChangedWithNoNodesChanged() throws Exception { + ClusterChangedEvent event = new ClusterChangedEvent("test", state, state); + clientManager.clusterChanged(event); + + // Verify original client still exists + for (DiscoveryNode node : state.nodes()) { + assertNotNull(clientManager.getFlightClient(node.getId())); + } + } + + public void testGetLocalNodeId() throws Exception { + assertEquals("Local node ID should match", "local_node", clientManager.getLocalNodeId()); + } + + public void testCloseWithActiveClients() throws Exception { + for (DiscoveryNode node : state.nodes()) { + OSFlightClient client = clientManager.getFlightClient(node.getId()).get(); + assertNotNull(client); + } + + clientManager.close(); + assertEquals(0, clientManager.getFlightClients().size()); + } + + public void testIncompatibleNodeVersion() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("arrow.streams.enabled", "true"); + DiscoveryNode oldVersionNode = new DiscoveryNode( + "old_version_node", + new TransportAddress(InetAddress.getByName("127.0.0.3"), getBasePort() + port.addAndGet(1)), + attributes, + Collections.singleton(DiscoveryNodeRole.DATA_ROLE), + Version.fromString("2.18.0") // Version before Arrow Flight introduction + ); + + // Update cluster state with old version node + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); + nodesBuilder.add(oldVersionNode); + nodesBuilder.localNodeId("local_node"); + DiscoveryNodes nodes = nodesBuilder.build(); + ClusterState oldVersionState = ClusterState.builder(new ClusterName("test")).nodes(nodes).build(); + + when(clusterService.state()).thenReturn(oldVersionState); + mockFlightInfoResponse(nodes, 0); + + assertFalse(clientManager.getFlightClient(oldVersionNode.getId()).isPresent()); + } + + public void testGetFlightClientLocationTimeout() throws Exception { + reset(client); + + String nodeId = "test_node"; + DiscoveryNode testNode = createNode(nodeId, "127.0.0.1", getBasePort() + port.addAndGet(2)); + + // Update cluster state with the test node + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); + nodesBuilder.add(testNode); + nodesBuilder.localNodeId(nodeId); + ClusterState newState = ClusterState.builder(new ClusterName("test")).nodes(nodesBuilder.build()).build(); + when(clusterService.state()).thenReturn(newState); + // Mock a delayed response that will cause timeout + mockFlightInfoResponse(newState.nodes(), LOCATION_TIMEOUT_MS + 100); + + ClusterChangedEvent event = new ClusterChangedEvent("test", newState, ClusterState.EMPTY_STATE); + clientManager.clusterChanged(event); + assertFalse(clientManager.getFlightClient(nodeId).isPresent()); + } + + public void testGetFlightClientLocationExecutionError() throws Exception { + reset(client); + + String nodeId = "test_node"; + DiscoveryNode testNode = createNode(nodeId, "127.0.0.1", getBasePort() + port.addAndGet(2)); + + // Update cluster state with the test node + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); + nodesBuilder.add(testNode); + nodesBuilder.localNodeId(nodeId); + ClusterState newState = ClusterState.builder(new ClusterName("test")).nodes(nodesBuilder.build()).build(); + + when(clusterService.state()).thenReturn(newState); + + // Mock failure + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(2); + listener.onFailure(new RuntimeException("Test execution error")); + return null; + }).when(client).execute(eq(NodesFlightInfoAction.INSTANCE), any(NodesFlightInfoRequest.class), any(ActionListener.class)); + + ClusterChangedEvent event = new ClusterChangedEvent("test", newState, ClusterState.EMPTY_STATE); + clientManager.clusterChanged(event); + + assertFalse(clientManager.getFlightClient(nodeId).isPresent()); + } + + public void testFailedClusterUpdateButSuccessfulDirectRequest() throws Exception { + reset(client); + + String nodeId = "test_node"; + DiscoveryNode testNode = createNode(nodeId, "127.0.0.1", getBasePort() + port.addAndGet(2)); + + // Update cluster state with the test node + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); + nodesBuilder.add(testNode); + nodesBuilder.localNodeId(nodeId); + ClusterState newState = ClusterState.builder(new ClusterName("test")).nodes(nodesBuilder.build()).build(); + + when(clusterService.state()).thenReturn(newState); + + // First mock call fails during cluster update + AtomicBoolean firstCall = new AtomicBoolean(true); + doAnswer(invocation -> { + locationUpdaterExecutor.schedule(() -> { + ActionListener listener = invocation.getArgument(2); + if (firstCall.getAndSet(false)) { + // Fail on first call (during cluster update) + listener.onFailure(new RuntimeException("Failed during cluster update")); + } else { + // Succeed on second call (direct request) + try { + NodesFlightInfoRequest request = invocation.getArgument(1); + List nodeInfos = new ArrayList<>(); + for (DiscoveryNode node : newState.nodes()) { + if (request.nodesIds().length == 0 || Arrays.asList(request.nodesIds()).contains(node.getId())) { + int flightPort = getBaseStreamPort() + port.addAndGet(2); + TransportAddress address = new TransportAddress( + InetAddress.getByName(node.getAddress().getAddress()), + flightPort + ); + BoundTransportAddress boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + NodeFlightInfo nodeInfo = new NodeFlightInfo(node, boundAddress); + nodeInfos.add(nodeInfo); + } + } + NodesFlightInfoResponse response = new NodesFlightInfoResponse( + ClusterName.DEFAULT, + nodeInfos, + Collections.emptyList() + ); + listener.onResponse(response); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + } + }, 0, TimeUnit.MICROSECONDS); + return null; + }).when(client).execute(eq(NodesFlightInfoAction.INSTANCE), any(NodesFlightInfoRequest.class), any(ActionListener.class)); + + ClusterChangedEvent event = new ClusterChangedEvent("test", newState, ClusterState.EMPTY_STATE); + clientManager.clusterChanged(event); + + // Verify that the client can still be created successfully on direct request + clientManager.buildClientAsync(nodeId); + assertBusy( + () -> { + assertNotNull("Flight client should be created successfully on direct request", clientManager.getFlightClient(nodeId)); + }, + 2, + TimeUnit.SECONDS + ); + assertFalse("first call should be invoked", firstCall.get()); + } + + private void validateNodes() { + for (DiscoveryNode node : state.nodes()) { + Optional client = clientManager.getFlightClient(node.getId()); + assertTrue("Flight client should be created for node [" + node.getId() + "].", client.isPresent()); + assertNotNull("Flight client should be created for node [" + node.getId() + "].", client.get()); + } + } + + protected static int getBaseStreamPort() { + return getBasePort(9401); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/FlightServiceTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/FlightServiceTests.java new file mode 100644 index 0000000000000..886a626c3a8a1 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/FlightServiceTests.java @@ -0,0 +1,170 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.arrow.flight.bootstrap; + +import org.opensearch.Version; +import org.opensearch.arrow.flight.bootstrap.tls.DisabledSslContextProvider; +import org.opensearch.arrow.flight.bootstrap.tls.SslContextProvider; +import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.plugins.SecureTransportSettingsProvider; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; + +import java.net.InetAddress; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class FlightServiceTests extends OpenSearchTestCase { + + private Settings settings; + private ClusterService clusterService; + private NetworkService networkService; + private ThreadPool threadPool; + private SecureTransportSettingsProvider secureTransportSettingsProvider; + private final AtomicInteger port = new AtomicInteger(0); + private DiscoveryNode localNode; + + @Override + public void setUp() throws Exception { + super.setUp(); + FeatureFlagSetter.set(FeatureFlags.ARROW_STREAMS_SETTING.getKey()); + int availablePort = getBasePort(9500) + port.addAndGet(1); + settings = Settings.EMPTY; + localNode = createNode(availablePort); + + // Setup initial cluster state + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); + nodesBuilder.localNodeId(localNode.getId()); + nodesBuilder.add(localNode); + DiscoveryNodes nodes = nodesBuilder.build(); + ClusterState clusterState = ClusterState.builder(new ClusterName("test")).nodes(nodes).build(); + clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(clusterState); + + threadPool = mock(ThreadPool.class); + when(threadPool.executor(ServerConfig.FLIGHT_SERVER_THREAD_POOL_NAME)).thenReturn(mock(ExecutorService.class)); + when(threadPool.executor(ServerConfig.FLIGHT_CLIENT_THREAD_POOL_NAME)).thenReturn(mock(ExecutorService.class)); + secureTransportSettingsProvider = mock(SecureTransportSettingsProvider.class); + networkService = new NetworkService(Collections.emptyList()); + } + + public void testInitializeWithSslDisabled() throws Exception { + + Settings noSslSettings = Settings.builder().put("arrow.ssl.enable", false).build(); + + try (FlightService noSslService = new FlightService(noSslSettings)) { + noSslService.setClusterService(clusterService); + noSslService.setThreadPool(threadPool); + noSslService.setClient(mock(Client.class)); + noSslService.setNetworkService(networkService); + noSslService.start(); + // Verify SSL is properly disabled + SslContextProvider sslContextProvider = noSslService.getSslContextProvider(); + assertNotNull("SSL context provider should not be null", sslContextProvider); + assertTrue( + "SSL context provider should be DisabledSslContextProvider", + sslContextProvider instanceof DisabledSslContextProvider + ); + assertFalse("SSL should be disabled", sslContextProvider.isSslEnabled()); + assertNotNull(noSslService.getFlightClientManager()); + assertNotNull(noSslService.getBoundAddress()); + } + } + + public void testStartAndStop() throws Exception { + try (FlightService testService = new FlightService(Settings.EMPTY)) { + testService.setClusterService(clusterService); + testService.setThreadPool(threadPool); + testService.setClient(mock(Client.class)); + testService.setNetworkService(networkService); + testService.start(); + testService.stop(); + testService.start(); + assertNotNull(testService.getStreamManager()); + } + } + + public void testInitializeWithoutSecureTransportSettingsProvider() { + Settings sslSettings = Settings.builder().put(settings).put("arrow.ssl.enable", true).build(); + + try (FlightService sslService = new FlightService(sslSettings)) { + // Should throw exception when initializing without provider + expectThrows(RuntimeException.class, () -> { + sslService.setClusterService(clusterService); + sslService.setThreadPool(threadPool); + sslService.setClient(mock(Client.class)); + sslService.setNetworkService(networkService); + sslService.start(); + }); + } + } + + public void testServerStartupFailure() { + Settings invalidSettings = Settings.builder() + .put(ServerComponents.SETTING_FLIGHT_PUBLISH_PORT.getKey(), "-100") // Invalid port + .build(); + try (FlightService invalidService = new FlightService(invalidSettings)) { + invalidService.setClusterService(clusterService); + invalidService.setThreadPool(threadPool); + invalidService.setClient(mock(Client.class)); + invalidService.setNetworkService(networkService); + expectThrows(RuntimeException.class, () -> { invalidService.doStart(); }); + } + } + + public void testLifecycleStateTransitions() throws Exception { + // Find new port for this test + try (FlightService testService = new FlightService(Settings.EMPTY)) { + testService.setClusterService(clusterService); + testService.setThreadPool(threadPool); + testService.setClient(mock(Client.class)); + testService.setNetworkService(networkService); + // Test all state transitions + testService.start(); + assertEquals("STARTED", testService.lifecycleState().toString()); + + testService.stop(); + assertEquals("STOPPED", testService.lifecycleState().toString()); + + testService.close(); + assertEquals("CLOSED", testService.lifecycleState().toString()); + } + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + private DiscoveryNode createNode(int port) throws Exception { + TransportAddress address = new TransportAddress(InetAddress.getByName("127.0.0.1"), port); + Map attributes = new HashMap<>(); + attributes.put("arrow.streams.enabled", "true"); + + Set roles = Collections.singleton(DiscoveryNodeRole.DATA_ROLE); + return new DiscoveryNode("local_node", address, attributes, roles, Version.CURRENT); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/ServerConfigTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/ServerConfigTests.java new file mode 100644 index 0000000000000..9419e26318046 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/ServerConfigTests.java @@ -0,0 +1,80 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.arrow.flight.bootstrap; + +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ScalingExecutorBuilder; + +import static org.opensearch.arrow.flight.bootstrap.ServerComponents.SETTING_FLIGHT_PUBLISH_PORT; + +public class ServerConfigTests extends OpenSearchTestCase { + + private Settings settings; + + @Override + public void setUp() throws Exception { + super.setUp(); + settings = Settings.builder() + .put("arrow.allocation.manager.type", "Netty") + .put("arrow.enable_null_check_for_get", false) + .put("arrow.enable_unsafe_memory_access", true) + .put("arrow.memory.debug.allocator", false) + .put("arrow.ssl.enable", true) + .put("thread_pool.flight-server.min", 1) + .put("thread_pool.flight-server.max", 4) + .put("thread_pool.flight-server.keep_alive", TimeValue.timeValueMinutes(5)) + .build(); + } + + public void testInit() { + ServerConfig.init(settings); + + // Verify system properties are set correctly + assertEquals("Netty", System.getProperty("arrow.allocation.manager.type")); + assertEquals("false", System.getProperty("arrow.enable_null_check_for_get")); + assertEquals("true", System.getProperty("arrow.enable_unsafe_memory_access")); + assertEquals("false", System.getProperty("arrow.memory.debug.allocator")); + + // Verify SSL settings + assertTrue(ServerConfig.isSslEnabled()); + + ScalingExecutorBuilder executorBuilder = ServerConfig.getServerExecutorBuilder(); + assertNotNull(executorBuilder); + assertEquals(3, executorBuilder.getRegisteredSettings().size()); + assertEquals(1, executorBuilder.getRegisteredSettings().get(0).get(settings)); // min + assertEquals(4, executorBuilder.getRegisteredSettings().get(1).get(settings)); // max + assertEquals(TimeValue.timeValueMinutes(5), executorBuilder.getRegisteredSettings().get(2).get(settings)); // keep alive + } + + public void testGetSettings() { + var settings = ServerConfig.getSettings(); + assertNotNull(settings); + assertFalse(settings.isEmpty()); + + assertTrue(settings.contains(ServerConfig.ARROW_ALLOCATION_MANAGER_TYPE)); + assertTrue(settings.contains(ServerConfig.ARROW_ENABLE_NULL_CHECK_FOR_GET)); + assertTrue(settings.contains(ServerConfig.ARROW_ENABLE_UNSAFE_MEMORY_ACCESS)); + assertTrue(settings.contains(ServerConfig.ARROW_ENABLE_DEBUG_ALLOCATOR)); + assertTrue(settings.contains(ServerConfig.ARROW_SSL_ENABLE)); + } + + public void testDefaultSettings() { + Settings defaultSettings = Settings.EMPTY; + ServerConfig.init(defaultSettings); + + // Verify default values + assertEquals(-1, SETTING_FLIGHT_PUBLISH_PORT.get(defaultSettings).intValue()); + assertEquals("Netty", ServerConfig.ARROW_ALLOCATION_MANAGER_TYPE.get(defaultSettings)); + assertFalse(ServerConfig.ARROW_ENABLE_NULL_CHECK_FOR_GET.get(defaultSettings)); + assertTrue(ServerConfig.ARROW_ENABLE_UNSAFE_MEMORY_ACCESS.get(defaultSettings)); + assertFalse(ServerConfig.ARROW_ENABLE_DEBUG_ALLOCATOR.get(defaultSettings)); + assertFalse(ServerConfig.ARROW_SSL_ENABLE.get(defaultSettings)); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/tls/SslContextProviderTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/tls/SslContextProviderTests.java new file mode 100644 index 0000000000000..d4e88d7eee9a6 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/tls/SslContextProviderTests.java @@ -0,0 +1,96 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.arrow.flight.bootstrap.tls; + +import org.opensearch.plugins.SecureTransportSettingsProvider; +import org.opensearch.test.OpenSearchTestCase; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.TrustManagerFactory; + +import java.io.IOException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; +import java.util.Arrays; +import java.util.Optional; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class SslContextProviderTests extends OpenSearchTestCase { + + private SecureTransportSettingsProvider mockSecureTransportSettingsProvider; + private SecureTransportSettingsProvider.SecureTransportParameters mockParameters; + private KeyManagerFactory keyManagerFactory; + private TrustManagerFactory trustManagerFactory; + + @Override + public void setUp() throws Exception { + super.setUp(); + + mockSecureTransportSettingsProvider = mock(SecureTransportSettingsProvider.class); + mockParameters = mock(SecureTransportSettingsProvider.SecureTransportParameters.class); + + String[] protocols = { "TLSv1.2", "TLSv1.3" }; + Iterable cipherSuites = Arrays.asList("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"); + setupDummyFactories(); + when(mockParameters.sslProvider()).thenReturn("JDK"); + when(mockParameters.clientAuth()).thenReturn("REQUIRE"); + when(mockParameters.protocols()).thenReturn(protocols); + when(mockParameters.cipherSuites()).thenReturn(cipherSuites); + when(mockParameters.keyManagerFactory()).thenReturn(Optional.of(keyManagerFactory)); + when(mockParameters.trustManagerFactory()).thenReturn(trustManagerFactory); + + when(mockSecureTransportSettingsProvider.parameters(null)).thenReturn(java.util.Optional.of(mockParameters)); + } + + private void setupDummyFactories() throws KeyStoreException, CertificateException, IOException, NoSuchAlgorithmException, + UnrecoverableKeyException { + KeyStore keystore = KeyStore.getInstance(KeyStore.getDefaultType()); + keystore.load(null, new char[0]); + + keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(keystore, new char[0]); + + trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keystore); + } + + public void testEnabledSslContextProvider() { + SslContextProvider provider = new DefaultSslContextProvider(mockSecureTransportSettingsProvider); + + assertTrue("SSL should be enabled", provider.isSslEnabled()); + assertNotNull(provider.getServerSslContext()); + assertNotNull(provider.getClientSslContext()); + } + + public void testDisabledSslContextProvider() { + SslContextProvider provider = new DisabledSslContextProvider(); + + assertFalse("SSL should be disabled", provider.isSslEnabled()); + assertNull("Server SSL context should be null", provider.getServerSslContext()); + assertNull("Client SSL context should be null", provider.getClientSslContext()); + } + + public void testDefaultSslContextProviderWithNullSupplier() { + expectThrows(NullPointerException.class, () -> { + SslContextProvider sslContextProvider = new DefaultSslContextProvider(null); + sslContextProvider.getServerSslContext(); + }); + } + + public void testDefaultSslContextProviderWithInvalidSslProvider() { + when(mockParameters.sslProvider()).thenReturn("INVALID"); + DefaultSslContextProvider provider = new DefaultSslContextProvider(mockSecureTransportSettingsProvider); + expectThrows(IllegalArgumentException.class, provider::getServerSslContext); + expectThrows(IllegalArgumentException.class, provider::getClientSslContext); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/BaseFlightProducerTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/BaseFlightProducerTests.java new file mode 100644 index 0000000000000..aefb415626a36 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/BaseFlightProducerTests.java @@ -0,0 +1,462 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.FlightProducer; +import org.apache.arrow.flight.Ticket; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.ipc.message.IpcOption; +import org.opensearch.arrow.flight.bootstrap.FlightClientManager; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class BaseFlightProducerTests extends OpenSearchTestCase { + + private BaseFlightProducer baseFlightProducer; + private FlightStreamManager streamManager; + private StreamProducer streamProducer; + private StreamProducer.BatchedJob batchedJob; + private static final String LOCAL_NODE_ID = "localNodeId"; + private static final FlightClientManager flightClientManager = mock(FlightClientManager.class); + private final Ticket ticket = new Ticket((new FlightStreamTicket("test-ticket", LOCAL_NODE_ID)).toBytes()); + private BufferAllocator allocator; + + @Override + public void setUp() throws Exception { + super.setUp(); + FeatureFlagSetter.set(FeatureFlags.ARROW_STREAMS_SETTING.getKey()); + streamManager = mock(FlightStreamManager.class); + when(streamManager.getStreamTicketFactory()).thenReturn(new FlightStreamTicketFactory(() -> LOCAL_NODE_ID)); + when(flightClientManager.getLocalNodeId()).thenReturn(LOCAL_NODE_ID); + allocator = mock(BufferAllocator.class); + streamProducer = mock(StreamProducer.class); + batchedJob = mock(StreamProducer.BatchedJob.class); + baseFlightProducer = new BaseFlightProducer(flightClientManager, streamManager, allocator); + } + + private static class TestServerStreamListener implements FlightProducer.ServerStreamListener { + private final CountDownLatch completionLatch = new CountDownLatch(1); + private final AtomicInteger putNextCount = new AtomicInteger(0); + private final AtomicBoolean isCancelled = new AtomicBoolean(false); + private Throwable error; + private final AtomicBoolean dataConsumed = new AtomicBoolean(false); + private final AtomicBoolean ready = new AtomicBoolean(false); + private Runnable onReadyHandler; + private Runnable onCancelHandler; + + @Override + public void putNext() { + assertFalse(dataConsumed.get()); + putNextCount.incrementAndGet(); + dataConsumed.set(true); + } + + @Override + public boolean isReady() { + return ready.get(); + } + + public void setReady(boolean val) { + ready.set(val); + if (this.onReadyHandler != null) { + this.onReadyHandler.run(); + } + } + + @Override + public void start(VectorSchemaRoot root) { + // No-op for this test + } + + @Override + public void start(VectorSchemaRoot root, DictionaryProvider dictionaries, IpcOption option) {} + + @Override + public void putNext(ArrowBuf metadata) { + putNext(); + } + + @Override + public void putMetadata(ArrowBuf metadata) { + + } + + @Override + public void completed() { + completionLatch.countDown(); + } + + @Override + public void error(Throwable t) { + error = t; + completionLatch.countDown(); + } + + @Override + public boolean isCancelled() { + return isCancelled.get(); + } + + @Override + public void setOnReadyHandler(Runnable handler) { + this.onReadyHandler = handler; + } + + @Override + public void setOnCancelHandler(Runnable handler) { + this.onCancelHandler = handler; + } + + public void resetConsumptionLatch() { + dataConsumed.set(false); + } + + public boolean getDataConsumed() { + return dataConsumed.get(); + } + + public int getPutNextCount() { + return putNextCount.get(); + } + + public Throwable getError() { + return error; + } + + public void cancel() { + isCancelled.set(true); + if (this.onCancelHandler != null) { + this.onCancelHandler.run(); + } + } + } + + public void testGetStream_SuccessfulFlow() throws Exception { + final VectorSchemaRoot root = mock(VectorSchemaRoot.class); + when(streamManager.removeStreamProducer(any(FlightStreamTicket.class))).thenReturn( + Optional.of(FlightStreamManager.StreamProducerHolder.create(streamProducer, allocator)) + ); + when(streamProducer.createJob(any(BufferAllocator.class))).thenReturn(batchedJob); + when(streamProducer.createRoot(any(BufferAllocator.class))).thenReturn(root); + + AtomicInteger flushCount = new AtomicInteger(0); + TestServerStreamListener listener = new TestServerStreamListener(); + doAnswer(invocation -> { + StreamProducer.FlushSignal flushSignal = invocation.getArgument(1); + for (int i = 0; i < 3; i++) { + Thread clientThread = new Thread(() -> { + listener.setReady(false); + listener.setReady(true); + }); + listener.setReady(false); + clientThread.start(); + flushSignal.awaitConsumption(TimeValue.timeValueMillis(100)); + assertTrue(listener.getDataConsumed()); + flushCount.incrementAndGet(); + listener.resetConsumptionLatch(); + } + return null; + }).when(batchedJob).run(any(VectorSchemaRoot.class), any(StreamProducer.FlushSignal.class)); + baseFlightProducer.getStream(mock(FlightProducer.CallContext.class), ticket, listener); + + assertNull(listener.getError()); + assertEquals(3, listener.getPutNextCount()); + assertEquals(3, flushCount.get()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + verify(root).close(); + } + + public void testGetStream_WithSlowClient() throws Exception { + final VectorSchemaRoot root = mock(VectorSchemaRoot.class); + + when(streamManager.removeStreamProducer(any(FlightStreamTicket.class))).thenReturn( + Optional.of(FlightStreamManager.StreamProducerHolder.create(streamProducer, allocator)) + ); + when(streamProducer.createJob(any(BufferAllocator.class))).thenReturn(batchedJob); + when(streamProducer.createRoot(any(BufferAllocator.class))).thenReturn(root); + + AtomicInteger flushCount = new AtomicInteger(0); + TestServerStreamListener listener = new TestServerStreamListener(); + + doAnswer(invocation -> { + StreamProducer.FlushSignal flushSignal = invocation.getArgument(1); + for (int i = 0; i < 5; i++) { + Thread clientThread = new Thread(() -> { + try { + listener.setReady(false); + Thread.sleep(100); + listener.setReady(true); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + listener.setReady(false); + clientThread.start(); + flushSignal.awaitConsumption(TimeValue.timeValueMillis(300)); // waiting for consumption for more than client thread sleep + assertTrue(listener.getDataConsumed()); + flushCount.incrementAndGet(); + listener.resetConsumptionLatch(); + } + return null; + }).when(batchedJob).run(any(), any()); + + baseFlightProducer.getStream(mock(FlightProducer.CallContext.class), ticket, listener); + + assertNull(listener.getError()); + assertEquals(5, listener.getPutNextCount()); + assertEquals(5, flushCount.get()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + verify(root).close(); + } + + public void testGetStream_WithSlowClientTimeout() throws Exception { + final VectorSchemaRoot root = mock(VectorSchemaRoot.class); + + when(streamManager.removeStreamProducer(any(FlightStreamTicket.class))).thenReturn( + Optional.of(FlightStreamManager.StreamProducerHolder.create(streamProducer, allocator)) + ); + when(streamProducer.createJob(any(BufferAllocator.class))).thenReturn(batchedJob); + when(streamProducer.createRoot(any(BufferAllocator.class))).thenReturn(root); + + AtomicInteger flushCount = new AtomicInteger(0); + TestServerStreamListener listener = new TestServerStreamListener(); + doAnswer(invocation -> { + StreamProducer.FlushSignal flushSignal = invocation.getArgument(1); + for (int i = 0; i < 5; i++) { + Thread clientThread = new Thread(() -> { + try { + listener.setReady(false); + Thread.sleep(400); + listener.setReady(true); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + listener.setReady(false); + clientThread.start(); + flushSignal.awaitConsumption(TimeValue.timeValueMillis(100)); // waiting for consumption for less than client thread sleep + assertTrue(listener.getDataConsumed()); + flushCount.incrementAndGet(); + listener.resetConsumptionLatch(); + } + return null; + }).when(batchedJob).run(any(), any()); + + assertThrows(RuntimeException.class, () -> baseFlightProducer.getStream(mock(FlightProducer.CallContext.class), ticket, listener)); + + assertNotNull(listener.getError()); + assertEquals("Stream deadline exceeded for consumption", listener.getError().getMessage()); + assertEquals(0, listener.getPutNextCount()); + assertEquals(0, flushCount.get()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + verify(root).close(); + } + + public void testGetStream_WithClientCancel() throws Exception { + final VectorSchemaRoot root = mock(VectorSchemaRoot.class); + + when(streamManager.removeStreamProducer(any(FlightStreamTicket.class))).thenReturn( + Optional.of(FlightStreamManager.StreamProducerHolder.create(streamProducer, allocator)) + ); + when(streamProducer.createJob(any(BufferAllocator.class))).thenReturn(batchedJob); + when(streamProducer.createRoot(any(BufferAllocator.class))).thenReturn(root); + + AtomicInteger flushCount = new AtomicInteger(0); + TestServerStreamListener listener = new TestServerStreamListener(); + doAnswer(invocation -> { + StreamProducer.FlushSignal flushSignal = invocation.getArgument(1); + for (int i = 0; i < 5; i++) { + int finalI = i; + Thread clientThread = new Thread(() -> { + if (finalI == 4) { + listener.cancel(); + } else { + listener.setReady(false); + listener.setReady(true); + } + }); + listener.setReady(false); + clientThread.start(); + flushSignal.awaitConsumption(TimeValue.timeValueMillis(100)); // waiting for consumption for less than client thread sleep + assertTrue(listener.getDataConsumed()); + flushCount.incrementAndGet(); + listener.resetConsumptionLatch(); + } + return null; + }).when(batchedJob).run(any(), any()); + + assertThrows(RuntimeException.class, () -> baseFlightProducer.getStream(mock(FlightProducer.CallContext.class), ticket, listener)); + assertNotNull(listener.getError()); + assertEquals("Stream cancelled by client", listener.getError().getMessage()); + assertEquals(4, listener.getPutNextCount()); + assertEquals(4, flushCount.get()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + verify(root).close(); + } + + public void testGetStream_WithUnresponsiveClient() throws Exception { + final VectorSchemaRoot root = mock(VectorSchemaRoot.class); + + when(streamManager.removeStreamProducer(any(FlightStreamTicket.class))).thenReturn( + Optional.of(FlightStreamManager.StreamProducerHolder.create(streamProducer, allocator)) + ); + when(streamProducer.createJob(any(BufferAllocator.class))).thenReturn(batchedJob); + when(streamProducer.createRoot(any(BufferAllocator.class))).thenReturn(root); + + AtomicInteger flushCount = new AtomicInteger(0); + TestServerStreamListener listener = new TestServerStreamListener(); + doAnswer(invocation -> { + StreamProducer.FlushSignal flushSignal = invocation.getArgument(1); + for (int i = 0; i < 5; i++) { + Thread clientThread = new Thread(() -> { + listener.setReady(false); + // not setting ready to simulate unresponsive behaviour + }); + listener.setReady(false); + clientThread.start(); + flushSignal.awaitConsumption(TimeValue.timeValueMillis(100)); // waiting for consumption for less than client thread sleep + assertTrue(listener.getDataConsumed()); + flushCount.incrementAndGet(); + listener.resetConsumptionLatch(); + } + return null; + }).when(batchedJob).run(any(), any()); + + assertThrows(RuntimeException.class, () -> baseFlightProducer.getStream(mock(FlightProducer.CallContext.class), ticket, listener)); + + assertNotNull(listener.getError()); + assertEquals("Stream deadline exceeded for consumption", listener.getError().getMessage()); + assertEquals(0, listener.getPutNextCount()); + assertEquals(0, flushCount.get()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + verify(root).close(); + } + + public void testGetStream_WithServerBackpressure() throws Exception { + final VectorSchemaRoot root = mock(VectorSchemaRoot.class); + + when(streamManager.removeStreamProducer(any(FlightStreamTicket.class))).thenReturn( + Optional.of(FlightStreamManager.StreamProducerHolder.create(streamProducer, allocator)) + ); + when(streamProducer.createJob(any(BufferAllocator.class))).thenReturn(batchedJob); + when(streamProducer.createRoot(any(BufferAllocator.class))).thenReturn(root); + + TestServerStreamListener listener = new TestServerStreamListener(); + AtomicInteger flushCount = new AtomicInteger(0); + doAnswer(invocation -> { + StreamProducer.FlushSignal flushSignal = invocation.getArgument(1); + for (int i = 0; i < 5; i++) { + Thread clientThread = new Thread(() -> { + listener.setReady(false); + listener.setReady(true); + }); + listener.setReady(false); + clientThread.start(); + Thread.sleep(100); // simulating writer backpressure + flushSignal.awaitConsumption(TimeValue.timeValueMillis(100)); + assertTrue(listener.getDataConsumed()); + flushCount.incrementAndGet(); + listener.resetConsumptionLatch(); + } + return null; + }).when(batchedJob).run(any(VectorSchemaRoot.class), any(StreamProducer.FlushSignal.class)); + + baseFlightProducer.getStream(mock(FlightProducer.CallContext.class), ticket, listener); + + assertNull(listener.getError()); + assertEquals(5, listener.getPutNextCount()); + assertEquals(5, flushCount.get()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + verify(root).close(); + } + + public void testGetStream_WithServerError() throws Exception { + final VectorSchemaRoot root = mock(VectorSchemaRoot.class); + + when(streamManager.removeStreamProducer(any(FlightStreamTicket.class))).thenReturn( + Optional.of(FlightStreamManager.StreamProducerHolder.create(streamProducer, allocator)) + ); + when(streamProducer.createJob(any(BufferAllocator.class))).thenReturn(batchedJob); + when(streamProducer.createRoot(any(BufferAllocator.class))).thenReturn(root); + + TestServerStreamListener listener = new TestServerStreamListener(); + AtomicInteger flushCount = new AtomicInteger(0); + doAnswer(invocation -> { + StreamProducer.FlushSignal flushSignal = invocation.getArgument(1); + for (int i = 0; i < 5; i++) { + Thread clientThread = new Thread(() -> { + listener.setReady(false); + listener.setReady(true); + }); + listener.setReady(false); + clientThread.start(); + if (i == 4) { + throw new RuntimeException("Server error"); + } + flushSignal.awaitConsumption(TimeValue.timeValueMillis(100)); + assertTrue(listener.getDataConsumed()); + flushCount.incrementAndGet(); + listener.resetConsumptionLatch(); + } + return null; + }).when(batchedJob).run(any(VectorSchemaRoot.class), any(StreamProducer.FlushSignal.class)); + + assertThrows(RuntimeException.class, () -> baseFlightProducer.getStream(mock(FlightProducer.CallContext.class), ticket, listener)); + + assertNotNull(listener.getError()); + assertEquals("Server error", listener.getError().getMessage()); + assertEquals(4, listener.getPutNextCount()); + assertEquals(4, flushCount.get()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + verify(root).close(); + } + + public void testGetStream_StreamNotFound() throws Exception { + + when(streamManager.getStreamProducer(any(FlightStreamTicket.class))).thenReturn(null); + + TestServerStreamListener listener = new TestServerStreamListener(); + + baseFlightProducer.getStream(null, ticket, listener); + + assertNotNull(listener.getError()); + assertTrue(listener.getError().getMessage().contains("Stream not found")); + assertEquals(0, listener.getPutNextCount()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + } + + public void testProxyStreamProviderCreationWithDifferentNodeIDs() { + // TODO: proxy stream provider coverage + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamManagerTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamManagerTests.java new file mode 100644 index 0000000000000..aa519399cb64d --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamManagerTests.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.flight.OSFlightClient; +import org.apache.arrow.flight.Ticket; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.Schema; +import org.opensearch.arrow.flight.bootstrap.FlightClientManager; +import org.opensearch.arrow.spi.StreamReader; +import org.opensearch.arrow.spi.StreamTicket; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Collections; +import java.util.Optional; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class FlightStreamManagerTests extends OpenSearchTestCase { + + private OSFlightClient flightClient; + private FlightStreamManager flightStreamManager; + private static final String NODE_ID = "testNodeId"; + private static final String TICKET_ID = "testTicketId"; + + @Override + public void setUp() throws Exception { + super.setUp(); + flightClient = mock(OSFlightClient.class); + FlightClientManager clientManager = mock(FlightClientManager.class); + when(clientManager.getFlightClient(NODE_ID)).thenReturn(Optional.of(flightClient)); + BufferAllocator allocator = mock(BufferAllocator.class); + flightStreamManager = new FlightStreamManager(() -> allocator); + flightStreamManager.setClientManager(clientManager); + } + + public void testGetStreamReader() throws Exception { + StreamTicket ticket = new FlightStreamTicket(TICKET_ID, NODE_ID); + FlightStream mockFlightStream = mock(FlightStream.class); + VectorSchemaRoot mockRoot = mock(VectorSchemaRoot.class); + when(flightClient.getStream(new Ticket(ticket.toBytes()))).thenReturn(mockFlightStream); + when(mockFlightStream.getRoot()).thenReturn(mockRoot); + when(mockRoot.getSchema()).thenReturn(new Schema(Collections.emptyList())); + + StreamReader streamReader = flightStreamManager.getStreamReader(ticket); + + assertNotNull(streamReader); + assertNotNull(streamReader.getRoot()); + assertEquals(new Schema(Collections.emptyList()), streamReader.getRoot().getSchema()); + verify(flightClient).getStream(new Ticket(ticket.toBytes())); + } + + public void testGetVectorSchemaRootWithException() { + StreamTicket ticket = new FlightStreamTicket(TICKET_ID, NODE_ID); + when(flightClient.getStream(new Ticket(ticket.toBytes()))).thenThrow(new RuntimeException("Test exception")); + + expectThrows(RuntimeException.class, () -> flightStreamManager.getStreamReader(ticket)); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamReaderTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamReaderTests.java new file mode 100644 index 0000000000000..20e112dc730f6 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamReaderTests.java @@ -0,0 +1,89 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.opensearch.arrow.flight.bootstrap.ServerConfig; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class FlightStreamReaderTests extends OpenSearchTestCase { + + private FlightStream mockFlightStream; + + private FlightStreamReader iterator; + private VectorSchemaRoot root; + private BufferAllocator allocator; + + @Override + public void setUp() throws Exception { + super.setUp(); + FeatureFlagSetter.set(FeatureFlags.ARROW_STREAMS_SETTING.getKey()); + ServerConfig.init(Settings.EMPTY); + mockFlightStream = mock(FlightStream.class); + allocator = new RootAllocator(100000); + Field field = new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null); + Schema schema = new Schema(List.of(field)); + root = VectorSchemaRoot.create(schema, allocator); + when(mockFlightStream.getRoot()).thenReturn(root); + iterator = new FlightStreamReader(mockFlightStream); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + root.close(); + allocator.close(); + } + + public void testNext_ReturnsTrue_WhenFlightStreamHasNext() throws Exception { + when(mockFlightStream.next()).thenReturn(true); + assertTrue(iterator.next()); + assert(mockFlightStream).next(); + } + + public void testNext_ReturnsFalse_WhenFlightStreamHasNoNext() throws Exception { + when(mockFlightStream.next()).thenReturn(false); + assertFalse(iterator.next()); + verify(mockFlightStream).next(); + } + + public void testGetRoot_ReturnsRootFromFlightStream() throws Exception { + VectorSchemaRoot returnedRoot = iterator.getRoot(); + assertEquals(root, returnedRoot); + verify(mockFlightStream).getRoot(); + } + + public void testClose_CallsCloseOnFlightStream() throws Exception { + iterator.close(); + verify(mockFlightStream).close(); + } + + public void testClose_WrapsExceptionInRuntimeException() throws Exception { + doThrow(new Exception("Test exception")).when(mockFlightStream).close(); + assertThrows(RuntimeException.class, () -> iterator.close()); + verify(mockFlightStream).close(); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamTicketTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamTicketTests.java new file mode 100644 index 0000000000000..819da2826c173 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamTicketTests.java @@ -0,0 +1,111 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.opensearch.arrow.spi.StreamTicket; +import org.opensearch.test.OpenSearchTestCase; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class FlightStreamTicketTests extends OpenSearchTestCase { + + public void testConstructorAndGetters() { + String ticketID = "ticket123"; + String nodeID = "node456"; + StreamTicket ticket = new FlightStreamTicket(ticketID, nodeID); + + assertEquals(ticketID, ticket.getTicketId()); + assertEquals(nodeID, ticket.getNodeId()); + } + + public void testToBytes() { + StreamTicket ticket = new FlightStreamTicket("ticket123", "node456"); + byte[] bytes = ticket.toBytes(); + + assertNotNull(bytes); + assertTrue(bytes.length > 0); + + // Decode the Base64 and check the structure + byte[] decoded = Base64.getDecoder().decode(bytes); + assertEquals(2 + 9 + 2 + 7, decoded.length); // 2 shorts + "ticket123" + "node456" + } + + public void testFromBytes() { + StreamTicket original = new FlightStreamTicket("ticket123", "node456"); + byte[] bytes = original.toBytes(); + + StreamTicket reconstructed = FlightStreamTicket.fromBytes(bytes); + + assertEquals(original.getTicketId(), reconstructed.getTicketId()); + assertEquals(original.getNodeId(), reconstructed.getNodeId()); + } + + public void testToBytesWithLongStrings() { + String longString = randomAlphaOfLength(Short.MAX_VALUE + 1); + StreamTicket ticket = new FlightStreamTicket(longString, "node456"); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, ticket::toBytes); + assertEquals("Field lengths exceed the maximum allowed size.", exception.getMessage()); + } + + public void testNullInput() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> FlightStreamTicket.fromBytes(null)); + assertEquals("Invalid byte array input.", e.getMessage()); + } + + public void testEmptyInput() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> FlightStreamTicket.fromBytes(new byte[0])); + assertEquals("Invalid byte array input.", e.getMessage()); + } + + public void testMalformedBase64() { + byte[] invalidBase64 = "Invalid Base64!@#$".getBytes(StandardCharsets.UTF_8); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> FlightStreamTicket.fromBytes(invalidBase64)); + assertEquals("Illegal base64 character 20", e.getMessage()); + } + + public void testModifiedLengthFields() { + StreamTicket original = new FlightStreamTicket("ticket123", "node456"); + byte[] bytes = original.toBytes(); + byte[] decoded = Base64.getDecoder().decode(bytes); + + // Modify the length field to be larger than actual data + decoded[0] = (byte) 0xFF; + decoded[1] = (byte) 0xFF; + + byte[] modified = Base64.getEncoder().encode(decoded); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> FlightStreamTicket.fromBytes(modified)); + assertEquals("Invalid ticketID length: -1", e.getMessage()); + } + + public void testEquals() { + StreamTicket ticket1 = new FlightStreamTicket("ticket123", "node456"); + StreamTicket ticket2 = new FlightStreamTicket("ticket123", "node456"); + StreamTicket ticket3 = new FlightStreamTicket("ticket789", "node456"); + + assertEquals(ticket1, ticket2); + assertNotEquals(ticket1, ticket3); + assertNotEquals(null, ticket1); + assertNotEquals("Not a StreamTicket", ticket1); + } + + public void testHashCode() { + StreamTicket ticket1 = new FlightStreamTicket("ticket123", "node456"); + StreamTicket ticket2 = new FlightStreamTicket("ticket123", "node456"); + + assertEquals(ticket1.hashCode(), ticket2.hashCode()); + } + + public void testToString() { + StreamTicket ticket = new FlightStreamTicket("ticket123", "node456"); + String expected = "FlightStreamTicket{ticketID='ticket123', nodeID='node456'}"; + assertEquals(expected, ticket.toString()); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/ProxyStreamProducerTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/ProxyStreamProducerTests.java new file mode 100644 index 0000000000000..e99c5ee10f715 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/ProxyStreamProducerTests.java @@ -0,0 +1,112 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.After; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class ProxyStreamProducerTests extends OpenSearchTestCase { + + private FlightStream mockRemoteStream; + private BufferAllocator mockAllocator; + private ProxyStreamProducer proxyStreamProducer; + + @Override + public void setUp() throws Exception { + super.setUp(); + mockRemoteStream = mock(FlightStream.class); + mockAllocator = mock(BufferAllocator.class); + proxyStreamProducer = new ProxyStreamProducer(new FlightStreamReader(mockRemoteStream)); + } + + public void testCreateRoot() throws Exception { + VectorSchemaRoot mockRoot = mock(VectorSchemaRoot.class); + when(mockRemoteStream.getRoot()).thenReturn(mockRoot); + + VectorSchemaRoot result = proxyStreamProducer.createRoot(mockAllocator); + + assertEquals(mockRoot, result); + verify(mockRemoteStream).getRoot(); + } + + public void testDefaults() { + assertEquals("", proxyStreamProducer.getAction()); + assertEquals(-1, proxyStreamProducer.estimatedRowCount()); + } + + public void testCreateJob() { + StreamProducer.BatchedJob job = proxyStreamProducer.createJob(mockAllocator); + + assertNotNull(job); + assertTrue(job instanceof ProxyStreamProducer.ProxyBatchedJob); + } + + public void testProxyBatchedJob() throws Exception { + StreamProducer.BatchedJob job = proxyStreamProducer.createJob(mockAllocator); + VectorSchemaRoot mockRoot = mock(VectorSchemaRoot.class); + StreamProducer.FlushSignal mockFlushSignal = mock(StreamProducer.FlushSignal.class); + + when(mockRemoteStream.next()).thenReturn(true, true, false); + + job.run(mockRoot, mockFlushSignal); + + verify(mockRemoteStream, times(3)).next(); + verify(mockFlushSignal, times(2)).awaitConsumption(TimeValue.timeValueMillis(1000)); + } + + public void testProxyBatchedJobWithException() throws Exception { + StreamProducer.BatchedJob job = proxyStreamProducer.createJob(mockAllocator); + VectorSchemaRoot mockRoot = mock(VectorSchemaRoot.class); + StreamProducer.FlushSignal mockFlushSignal = mock(StreamProducer.FlushSignal.class); + + doThrow(new RuntimeException("Test exception")).when(mockRemoteStream).next(); + + try { + job.run(mockRoot, mockFlushSignal); + fail("Expected RuntimeException"); + } catch (RuntimeException e) { + assertEquals("Test exception", e.getMessage()); + } + + verify(mockRemoteStream, times(1)).next(); + } + + public void testProxyBatchedJobOnCancel() throws Exception { + StreamProducer.BatchedJob job = proxyStreamProducer.createJob(mockAllocator); + VectorSchemaRoot mockRoot = mock(VectorSchemaRoot.class); + StreamProducer.FlushSignal mockFlushSignal = mock(StreamProducer.FlushSignal.class); + when(mockRemoteStream.next()).thenReturn(true, true, false); + + // cancel the job + job.onCancel(); + job.run(mockRoot, mockFlushSignal); + verify(mockRemoteStream, times(0)).next(); + verify(mockFlushSignal, times(0)).awaitConsumption(TimeValue.timeValueMillis(1000)); + assertTrue(job.isCancelled()); + } + + @After + public void tearDown() throws Exception { + if (proxyStreamProducer != null) { + proxyStreamProducer.close(); + } + super.tearDown(); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/resources/org/opensearch/bootstrap/test.policy b/plugins/arrow-flight-rpc/src/test/resources/org/opensearch/bootstrap/test.policy new file mode 100644 index 0000000000000..5626d8cb250f6 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/resources/org/opensearch/bootstrap/test.policy @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +grant codeBase "${codebase.grpc-netty-shaded}" { + permission java.net.SocketPermission "*", "accept,connect,listen,resolve"; + permission java.lang.RuntimePermission "*", "setContextClassLoader"; +}; + +grant codeBase "${codebase.grpc-core}" { + permission java.net.SocketPermission "*", "accept,connect,listen,resolve"; + permission java.lang.RuntimePermission "*", "setContextClassLoader"; +}; + +grant { + // arrow flight service permissions + permission java.util.PropertyPermission "arrow.allocation.manager.type", "write"; + permission java.util.PropertyPermission "arrow.enable_null_check_for_get", "write"; + permission java.util.PropertyPermission "arrow.enable_unsafe_memory_access", "write"; + permission java.util.PropertyPermission "arrow.memory.debug.allocator", "write"; + + permission java.util.PropertyPermission "io.netty.tryReflectionSetAccessible", "write"; + permission java.util.PropertyPermission "io.netty.allocator.numDirectArenas", "write"; + permission java.util.PropertyPermission "io.netty.noUnsafe", "write"; + permission java.util.PropertyPermission "io.netty.tryUnsafe", "write"; + + // Needed for netty based arrow flight server for netty configs related to buffer allocator + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + permission java.util.PropertyPermission "arrow.allocation.manager.type", "write"; + + permission java.lang.RuntimePermission "modifyThreadGroup"; + permission java.lang.RuntimePermission "modifyThread"; + permission java.net.SocketPermission "*", "accept,connect,listen,resolve"; + + // Reflection access needed by Arrow + permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + + // Memory access + permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; +}; diff --git a/server/build.gradle b/server/build.gradle index 82eafb07a7ad3..99bef1e109f25 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -69,13 +69,13 @@ dependencies { api project(":libs:opensearch-geo") api project(":libs:opensearch-telemetry") api project(":libs:opensearch-task-commons") + implementation project(':libs:opensearch-arrow-spi') compileOnly project(':libs:opensearch-plugin-classloader') testRuntimeOnly project(':libs:opensearch-plugin-classloader') api libs.bundles.lucene - // utilities api project(":libs:opensearch-cli") // time handling, remove with java 8 time @@ -381,7 +381,6 @@ tasks.test { jvmArgs += ["--add-opens", "java.base/java.nio.file=ALL-UNNAMED"] } } - tasks.named("sourcesJar").configure { // Ignore duplicates for protobuf generated code (main and generatedSources). filesMatching("**/proto/*") { diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java index 43132b5cf58ab..edcf451848e88 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java @@ -67,6 +67,8 @@ import org.opensearch.search.profile.SearchProfileShardResults; import org.opensearch.search.query.QuerySearchResult; import org.opensearch.search.sort.SortedWiderNumericSortField; +import org.opensearch.search.stream.OSTicket; +import org.opensearch.search.stream.StreamSearchResult; import org.opensearch.search.suggest.Suggest; import org.opensearch.search.suggest.Suggest.Suggestion; import org.opensearch.search.suggest.completion.CompletionSuggestion; @@ -471,7 +473,8 @@ ReducedQueryPhase reducedQueryPhase( numReducePhases, 0, 0, - true + true, + null ); } int total = queryResults.size(); @@ -493,8 +496,12 @@ ReducedQueryPhase reducedQueryPhase( : Collections.emptyMap(); int from = 0; int size = 0; + List tickets = new ArrayList<>(); for (SearchPhaseResult entry : queryResults) { QuerySearchResult result = entry.queryResult(); + if (entry instanceof StreamSearchResult) { + tickets.addAll(((StreamSearchResult) entry).getFlightTickets()); + } from = result.from(); // sorted queries can set the size to 0 if they have enough competitive hits. size = Math.max(result.size(), size); @@ -544,7 +551,8 @@ ReducedQueryPhase reducedQueryPhase( numReducePhases, size, from, - false + false, + tickets ); } @@ -700,6 +708,8 @@ public static final class ReducedQueryPhase { // sort value formats used to sort / format the result final DocValueFormat[] sortValueFormats; + final List osTickets; + ReducedQueryPhase( TotalHits totalHits, long fetchHits, @@ -714,7 +724,8 @@ public static final class ReducedQueryPhase { int numReducePhases, int size, int from, - boolean isEmptyResult + boolean isEmptyResult, + List osTickets ) { if (numReducePhases <= 0) { throw new IllegalArgumentException("at least one reduce phase must have been applied but was: " + numReducePhases); @@ -733,6 +744,7 @@ public static final class ReducedQueryPhase { this.from = from; this.isEmptyResult = isEmptyResult; this.sortValueFormats = sortValueFormats; + this.osTickets = osTickets; } /** diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java index 8cf92934c8a52..814ae8f12cf44 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java @@ -22,7 +22,8 @@ public enum SearchPhaseName { FETCH("fetch"), DFS_QUERY("dfs_query"), EXPAND("expand"), - CAN_MATCH("can_match"); + CAN_MATCH("can_match"), + STREAM_REDUCE("stream_reduce"); private final String name; diff --git a/server/src/main/java/org/opensearch/action/search/SearchQueryThenFetchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/SearchQueryThenFetchAsyncAction.java index c8ab5fdaf61a1..c5ff4504941ab 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/SearchQueryThenFetchAsyncAction.java @@ -149,6 +149,7 @@ protected void onShardResult(SearchPhaseResult result, SearchShardIterator shard if (queryResult.isNull() == false // disable sort optims for scroll requests because they keep track of the last bottom doc locally (per shard) && getRequest().scroll() == null + && !queryResult.hasConsumedTopDocs() && queryResult.topDocs() != null && queryResult.topDocs().topDocs.getClass() == TopFieldDocs.class) { TopFieldDocs topDocs = (TopFieldDocs) queryResult.topDocs().topDocs; diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequest.java b/server/src/main/java/org/opensearch/action/search/SearchRequest.java index 4d3bb868b779a..2ab4172cab749 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequest.java @@ -680,6 +680,10 @@ public boolean isSuggestOnly() { return source != null && source.isSuggestOnly(); } + public boolean isStreamRequest() { + return searchType == SearchType.STREAM; + } + public int resolveTrackTotalHitsUpTo() { return resolveTrackTotalHitsUpTo(scroll, source); } diff --git a/server/src/main/java/org/opensearch/action/search/SearchResponse.java b/server/src/main/java/org/opensearch/action/search/SearchResponse.java index 0d55fbf2e7f88..27dc8070e6ec3 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/opensearch/action/search/SearchResponse.java @@ -62,6 +62,7 @@ import org.opensearch.search.pipeline.ProcessorExecutionDetail; import org.opensearch.search.profile.ProfileShardResult; import org.opensearch.search.profile.SearchProfileShardResults; +import org.opensearch.search.stream.OSTicket; import org.opensearch.search.suggest.Suggest; import java.io.IOException; @@ -75,6 +76,7 @@ import static org.opensearch.action.search.SearchResponseSections.EXT_FIELD; import static org.opensearch.action.search.SearchResponseSections.PROCESSOR_RESULT_FIELD; +import static org.opensearch.action.search.SearchResponseSections.TICKET_FIELD; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** @@ -320,6 +322,11 @@ public Map getProfileResults() { return internalResponse.profile(); } + @Nullable + public List getTickets() { + return internalResponse.tickets(); + } + /** * Returns info about what clusters the search was executed against. Available only in responses obtained * from a Cross Cluster Search request, otherwise null @@ -383,6 +390,7 @@ public static SearchResponse innerFromXContent(XContentParser parser) throws IOE Aggregations aggs = null; Suggest suggest = null; SearchProfileShardResults profile = null; + List tickets = null; boolean timedOut = false; Boolean terminatedEarly = null; int numReducePhases = 1; @@ -425,6 +433,8 @@ public static SearchResponse innerFromXContent(XContentParser parser) throws IOE suggest = Suggest.fromXContent(parser); } else if (SearchProfileShardResults.PROFILE_FIELD.equals(currentFieldName)) { profile = SearchProfileShardResults.fromXContent(parser); + } else if (TICKET_FIELD.equals(currentFieldName)) { + tickets = null; } else if (RestActions._SHARDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { while ((token = parser.nextToken()) != Token.END_OBJECT) { if (token == Token.FIELD_NAME) { @@ -539,7 +549,8 @@ public static SearchResponse innerFromXContent(XContentParser parser) throws IOE profile, numReducePhases, extBuilders, - processorResult + processorResult, + tickets ); return new SearchResponse( searchResponseSections, diff --git a/server/src/main/java/org/opensearch/action/search/SearchResponseMerger.java b/server/src/main/java/org/opensearch/action/search/SearchResponseMerger.java index 538e7fd54e2c3..5e689edabe51d 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchResponseMerger.java +++ b/server/src/main/java/org/opensearch/action/search/SearchResponseMerger.java @@ -50,6 +50,7 @@ import org.opensearch.search.internal.InternalSearchResponse; import org.opensearch.search.profile.ProfileShardResult; import org.opensearch.search.profile.SearchProfileShardResults; +import org.opensearch.search.stream.OSTicket; import org.opensearch.search.suggest.Suggest; import org.opensearch.search.suggest.completion.CompletionSuggestion; @@ -139,6 +140,7 @@ SearchResponse getMergedResponse(SearchResponse.Clusters clusters, SearchRequest int numReducePhases = 1; List failures = new ArrayList<>(); Map profileResults = new HashMap<>(); + List tickets = new ArrayList<>(); List aggs = new ArrayList<>(); Map shards = new TreeMap<>(); List topDocsList = new ArrayList<>(searchResponses.size()); @@ -156,7 +158,9 @@ SearchResponse getMergedResponse(SearchResponse.Clusters clusters, SearchRequest Collections.addAll(failures, searchResponse.getShardFailures()); profileResults.putAll(searchResponse.getProfileResults()); - + if (searchResponse.getTickets() != null && !searchResponse.getTickets().isEmpty()) { + tickets.addAll(searchResponse.getTickets()); + } if (searchResponse.getAggregations() != null) { InternalAggregations internalAggs = (InternalAggregations) searchResponse.getAggregations(); aggs.add(internalAggs); @@ -217,6 +221,7 @@ SearchResponse getMergedResponse(SearchResponse.Clusters clusters, SearchRequest InternalAggregations reducedAggs = InternalAggregations.topLevelReduce(aggs, aggReduceContextBuilder.forFinalReduction()); ShardSearchFailure[] shardFailures = failures.toArray(ShardSearchFailure.EMPTY_ARRAY); SearchProfileShardResults profileShardResults = profileResults.isEmpty() ? null : new SearchProfileShardResults(profileResults); + // make failures ordering consistent between ordinary search and CCS by looking at the shard they come from Arrays.sort(shardFailures, FAILURES_COMPARATOR); InternalSearchResponse response = new InternalSearchResponse( @@ -226,7 +231,10 @@ SearchResponse getMergedResponse(SearchResponse.Clusters clusters, SearchRequest profileShardResults, topDocsStats.timedOut, topDocsStats.terminatedEarly, - numReducePhases + numReducePhases, + Collections.emptyList(), + Collections.emptyList(), + (tickets.isEmpty() ? null : tickets) ); long tookInMillis = searchTimeProvider.buildTookInMillis(); return new SearchResponse( diff --git a/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java b/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java index f93508b0ba55b..31e3d7d79cd89 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java +++ b/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java @@ -43,6 +43,7 @@ import org.opensearch.search.pipeline.ProcessorExecutionDetail; import org.opensearch.search.profile.ProfileShardResult; import org.opensearch.search.profile.SearchProfileShardResults; +import org.opensearch.search.stream.OSTicket; import org.opensearch.search.suggest.Suggest; import java.io.IOException; @@ -67,10 +68,13 @@ public class SearchResponseSections implements ToXContentFragment { public static final ParseField EXT_FIELD = new ParseField("ext"); public static final ParseField PROCESSOR_RESULT_FIELD = new ParseField("processor_results"); + public static final String TICKET_FIELD = "ticket"; + protected final SearchHits hits; protected final Aggregations aggregations; protected final Suggest suggest; protected final SearchProfileShardResults profileResults; + protected final List tickets; protected final boolean timedOut; protected final Boolean terminatedEarly; protected final int numReducePhases; @@ -86,17 +90,7 @@ public SearchResponseSections( SearchProfileShardResults profileResults, int numReducePhases ) { - this( - hits, - aggregations, - suggest, - timedOut, - terminatedEarly, - profileResults, - numReducePhases, - Collections.emptyList(), - Collections.emptyList() - ); + this(hits, aggregations, suggest, timedOut, terminatedEarly, profileResults, numReducePhases, Collections.emptyList()); } public SearchResponseSections( @@ -108,7 +102,8 @@ public SearchResponseSections( SearchProfileShardResults profileResults, int numReducePhases, List searchExtBuilders, - List processorResult + List processorResult, + List tickets ) { this.hits = hits; this.aggregations = aggregations; @@ -119,6 +114,32 @@ public SearchResponseSections( this.numReducePhases = numReducePhases; this.processorResult.addAll(processorResult); this.searchExtBuilders.addAll(Objects.requireNonNull(searchExtBuilders, "searchExtBuilders must not be null")); + this.tickets = tickets; + } + + public SearchResponseSections( + SearchHits hits, + Aggregations aggregations, + Suggest suggest, + boolean timedOut, + Boolean terminatedEarly, + SearchProfileShardResults profileResults, + int numReducePhases, + List searchExtBuilders, + List processorResult + ) { + this( + hits, + aggregations, + suggest, + timedOut, + terminatedEarly, + profileResults, + numReducePhases, + searchExtBuilders, + Collections.emptyList(), + null + ); } public SearchResponseSections( @@ -140,7 +161,8 @@ public SearchResponseSections( profileResults, numReducePhases, searchExtBuilders, - Collections.emptyList() + Collections.emptyList(), + null ); } @@ -184,6 +206,19 @@ public final Map profile() { return profileResults.getShardResults(); } + /** + * Returns the profile results for this search response (including all shards). + * An empty map is returned if profiling was not enabled + * + * @return Profile results + */ + public final List tickets() { + if (tickets == null) { + return Collections.emptyList(); + } + return tickets; + } + @Override public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { hits.toXContent(builder, params); @@ -207,6 +242,14 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params) if (!processorResult.isEmpty()) { builder.field(PROCESSOR_RESULT_FIELD.getPreferredName(), processorResult); } + + if (tickets != null && !tickets.isEmpty()) { + builder.startArray(TICKET_FIELD); + for (OSTicket ticket : tickets) { + ticket.toXContent(builder, params); + } + builder.endArray(); + } return builder; } diff --git a/server/src/main/java/org/opensearch/action/search/SearchTransportService.java b/server/src/main/java/org/opensearch/action/search/SearchTransportService.java index 64c738f633f2e..8c30d37076cc9 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/opensearch/action/search/SearchTransportService.java @@ -60,6 +60,7 @@ import org.opensearch.search.query.QuerySearchRequest; import org.opensearch.search.query.QuerySearchResult; import org.opensearch.search.query.ScrollQuerySearchResult; +import org.opensearch.search.stream.StreamSearchResult; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.RemoteClusterService; import org.opensearch.transport.Transport; @@ -94,6 +95,7 @@ public class SearchTransportService { public static final String QUERY_ACTION_NAME = "indices:data/read/search[phase/query]"; public static final String QUERY_ID_ACTION_NAME = "indices:data/read/search[phase/query/id]"; public static final String QUERY_SCROLL_ACTION_NAME = "indices:data/read/search[phase/query/scroll]"; + public static final String QUERY_STREAM_ACTION_NAME = "indices:data/read/search[phase/query/stream]"; public static final String QUERY_FETCH_SCROLL_ACTION_NAME = "indices:data/read/search[phase/query+fetch/scroll]"; public static final String FETCH_ID_SCROLL_ACTION_NAME = "indices:data/read/search[phase/fetch/id/scroll]"; public static final String FETCH_ID_ACTION_NAME = "indices:data/read/search[phase/fetch/id]"; @@ -240,17 +242,29 @@ public void sendExecuteQuery( ) { // we optimize this and expect a QueryFetchSearchResult if we only have a single shard in the search request // this used to be the QUERY_AND_FETCH which doesn't exist anymore. - final boolean fetchDocuments = request.numberOfShards() == 1; - Writeable.Reader reader = fetchDocuments ? QueryFetchSearchResult::new : QuerySearchResult::new; - final ActionListener handler = responseWrapper.apply(connection, listener); - transportService.sendChildRequest( - connection, - QUERY_ACTION_NAME, - request, - task, - new ConnectionCountingHandler<>(handler, reader, clientConnections, connection.getNode().getId()) - ); + if (request.isStreamRequest()) { + Writeable.Reader reader = StreamSearchResult::new; + final ActionListener handler = responseWrapper.apply(connection, listener); + transportService.sendChildRequest( + connection, + QUERY_STREAM_ACTION_NAME, + request, + task, + new ConnectionCountingHandler<>(handler, reader, clientConnections, connection.getNode().getId()) + ); + } else { + final boolean fetchDocuments = request.numberOfShards() == 1; + Writeable.Reader reader = fetchDocuments ? QueryFetchSearchResult::new : QuerySearchResult::new; + final ActionListener handler = responseWrapper.apply(connection, listener); + transportService.sendChildRequest( + connection, + QUERY_ACTION_NAME, + request, + task, + new ConnectionCountingHandler<>(handler, reader, clientConnections, connection.getNode().getId()) + ); + } } public void sendExecuteQuery( @@ -610,6 +624,28 @@ public static void registerRequestHandler(TransportService transportService, Sea ); TransportActionProxy.registerProxyAction(transportService, QUERY_SCROLL_ACTION_NAME, ScrollQuerySearchResult::new); + transportService.registerRequestHandler( + QUERY_STREAM_ACTION_NAME, + ThreadPool.Names.SAME, + false, + true, + AdmissionControlActionType.SEARCH, + ShardSearchRequest::new, + (request, channel, task) -> { + searchService.executeStreamPhase( + request, + false, + (SearchShardTask) task, + new ChannelActionListener<>(channel, QUERY_STREAM_ACTION_NAME, request) + ); + } + ); + TransportActionProxy.registerProxyActionWithDynamicResponseType( + transportService, + QUERY_STREAM_ACTION_NAME, + (request) -> StreamSearchResult::new + ); + transportService.registerRequestHandler( QUERY_FETCH_SCROLL_ACTION_NAME, ThreadPool.Names.SAME, diff --git a/server/src/main/java/org/opensearch/action/search/SearchType.java b/server/src/main/java/org/opensearch/action/search/SearchType.java index e549ec598380a..a8e75c5f89113 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchType.java +++ b/server/src/main/java/org/opensearch/action/search/SearchType.java @@ -52,9 +52,10 @@ public enum SearchType { * document content. The return number of hits is exactly as specified in size, since they are the only ones that * are fetched. This is very handy when the index has a lot of shards (not replicas, shard id groups). */ - QUERY_THEN_FETCH((byte) 1); + QUERY_THEN_FETCH((byte) 1), // 2 used to be DFS_QUERY_AND_FETCH // 3 used to be QUERY_AND_FETCH + STREAM((byte) 5); /** * The default search type ({@link #QUERY_THEN_FETCH}. @@ -64,7 +65,7 @@ public enum SearchType { /** * Non-deprecated types */ - public static final SearchType[] CURRENTLY_SUPPORTED = { QUERY_THEN_FETCH, DFS_QUERY_THEN_FETCH }; + public static final SearchType[] CURRENTLY_SUPPORTED = { QUERY_THEN_FETCH, DFS_QUERY_THEN_FETCH, STREAM }; private byte id; @@ -88,6 +89,8 @@ public static SearchType fromId(byte id) { } else if (id == 1 || id == 3) { // TODO this bwc layer can be removed once this is back-ported to 5.3 QUERY_AND_FETCH is removed // now return QUERY_THEN_FETCH; + } else if (id == 5) { + return STREAM; } else { throw new IllegalArgumentException("No search type for [" + id + "]"); } @@ -106,6 +109,8 @@ public static SearchType fromString(String searchType) { return SearchType.DFS_QUERY_THEN_FETCH; } else if ("query_then_fetch".equals(searchType)) { return SearchType.QUERY_THEN_FETCH; + } else if ("stream".equals(searchType)) { + return SearchType.STREAM; } else { throw new IllegalArgumentException("No search type for [" + searchType + "]"); } diff --git a/server/src/main/java/org/opensearch/action/search/StreamAsyncAction.java b/server/src/main/java/org/opensearch/action/search/StreamAsyncAction.java new file mode 100644 index 0000000000000..b5cac957aa7bd --- /dev/null +++ b/server/src/main/java/org/opensearch/action/search/StreamAsyncAction.java @@ -0,0 +1,162 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.action.search; + +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.routing.GroupShardsIterator; +import org.opensearch.common.util.concurrent.AbstractRunnable; +import org.opensearch.core.action.ActionListener; +import org.opensearch.search.SearchHits; +import org.opensearch.search.SearchPhaseResult; +import org.opensearch.search.internal.AliasFilter; +import org.opensearch.search.internal.InternalSearchResponse; +import org.opensearch.search.stream.OSTicket; +import org.opensearch.search.stream.StreamSearchResult; +import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.transport.Transport; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executor; +import java.util.function.BiFunction; + +/** + * Async transport action for query then fetch + * + * @opensearch.internal + */ +class StreamAsyncAction extends SearchQueryThenFetchAsyncAction { + + public StreamAsyncAction( + Logger logger, + SearchTransportService searchTransportService, + BiFunction nodeIdToConnection, + Map aliasFilter, + Map concreteIndexBoosts, + Map> indexRoutings, + SearchPhaseController searchPhaseController, + Executor executor, + QueryPhaseResultConsumer resultConsumer, + SearchRequest request, + ActionListener listener, + GroupShardsIterator shardsIts, + TransportSearchAction.SearchTimeProvider timeProvider, + ClusterState clusterState, + SearchTask task, + SearchResponse.Clusters clusters, + SearchRequestContext searchRequestContext, + Tracer tracer + ) { + super( + logger, + searchTransportService, + nodeIdToConnection, + aliasFilter, + concreteIndexBoosts, + indexRoutings, + searchPhaseController, + executor, + resultConsumer, + request, + listener, + shardsIts, + timeProvider, + clusterState, + task, + clusters, + searchRequestContext, + tracer + ); + } + + @Override + protected SearchPhase getNextPhase(final SearchPhaseResults results, SearchPhaseContext context) { + return new StreamSearchReducePhase(SearchPhaseName.STREAM_REDUCE.getName(), context); + } + + class StreamSearchReducePhase extends SearchPhase { + private SearchPhaseContext context; + + protected StreamSearchReducePhase(String name, SearchPhaseContext context) { + super(name); + this.context = context; + } + + @Override + public void run() { + context.execute(new StreamReduceAction(context, this)); + } + }; + + class StreamReduceAction extends AbstractRunnable { + private SearchPhaseContext context; + private SearchPhase phase; + + StreamReduceAction(SearchPhaseContext context, SearchPhase phase) { + this.context = context; + + } + + @Override + protected void doRun() throws Exception { + List tickets = new ArrayList<>(); + for (SearchPhaseResult entry : results.getAtomicArray().asList()) { + if (entry instanceof StreamSearchResult) { + tickets.addAll(((StreamSearchResult) entry).getFlightTickets()); + } + } + InternalSearchResponse internalSearchResponse = new InternalSearchResponse( + SearchHits.empty(), + null, + null, + null, + false, + false, + 1, + Collections.emptyList(), + Collections.emptyList(), + tickets + ); + context.sendSearchResponse(internalSearchResponse, results.getAtomicArray()); + } + + @Override + public void onFailure(Exception e) { + context.onPhaseFailure(phase, "", e); + } + } +} diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java index 898174d60de76..5d7279a55073c 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java @@ -1068,7 +1068,7 @@ private void executeSearch( failIfOverShardCountLimit(clusterService, shardIterators.size()); Map concreteIndexBoosts = resolveIndexBoosts(searchRequest, clusterState); // optimize search type for cases where there is only one shard group to search on - if (shardIterators.size() == 1) { + if (!searchRequest.isStreamRequest() && shardIterators.size() == 1) { // if we only have one group, then we always want Q_T_F, no need for DFS, and no need to do THEN since we hit one shard searchRequest.searchType(QUERY_THEN_FETCH); } @@ -1313,6 +1313,28 @@ private AbstractSearchAsyncAction searchAsyncAction tracer ); break; + case STREAM: + searchAsyncAction = new StreamAsyncAction( + logger, + searchTransportService, + connectionLookup, + aliasFilter, + concreteIndexBoosts, + indexRoutings, + searchPhaseController, + executor, + queryResultConsumer, + searchRequest, + listener, + shardIterators, + timeProvider, + clusterState, + task, + clusters, + searchRequestContext, + tracer + ); + break; default: throw new IllegalStateException("Unknown search type: [" + searchRequest.searchType() + "]"); } diff --git a/server/src/main/java/org/opensearch/arrow/custom/ArrowDocIdCollector.java b/server/src/main/java/org/opensearch/arrow/custom/ArrowDocIdCollector.java new file mode 100644 index 0000000000000..fa689db4cb7e0 --- /dev/null +++ b/server/src/main/java/org/opensearch/arrow/custom/ArrowDocIdCollector.java @@ -0,0 +1,91 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.custom; + +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.FilterCollector; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Weight; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.common.unit.TimeValue; + +import java.io.IOException; + +/** + * Collects docIDs from a search and writes them to a VectorSchemaRoot. + */ +public class ArrowDocIdCollector extends FilterCollector { + private final VectorSchemaRoot root; + private final StreamProducer.FlushSignal flushSignal; + private final int batchSize; + private final IntVector docIDVector; + private int currentRow; + + public ArrowDocIdCollector(Collector in, VectorSchemaRoot root, StreamProducer.FlushSignal flushSignal, int batchSize) { + super(in); + this.root = root; + this.docIDVector = (IntVector) root.getVector("docID"); + this.flushSignal = flushSignal; + this.batchSize = batchSize; + this.currentRow = 0; + } + + @Override + public void setWeight(Weight weight) { + if (this.in != null) { + this.in.setWeight(weight); + } + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.TOP_DOCS; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + LeafCollector inner = (this.in == null ? null : super.getLeafCollector(context)); + return new LeafCollector() { + @Override + public void setScorer(Scorable scorer) throws IOException { + if (inner != null) { + inner.setScorer(scorer); + } + } + + @Override + public void collect(int doc) throws IOException { + if (inner != null) { + inner.collect(doc); + } + docIDVector.setSafe(currentRow, doc); + currentRow++; + if (currentRow >= batchSize) { + root.setRowCount(batchSize); + flushSignal.awaitConsumption(TimeValue.timeValueMillis(1000)); + currentRow = 0; + } + } + + @Override + public void finish() throws IOException { + if (currentRow > 0) { + root.setRowCount(currentRow); + flushSignal.awaitConsumption(TimeValue.timeValueMillis(1000)); + currentRow = 0; + } + } + }; + } +} diff --git a/server/src/main/java/org/opensearch/arrow/custom/StreamManagerWrapper.java b/server/src/main/java/org/opensearch/arrow/custom/StreamManagerWrapper.java new file mode 100644 index 0000000000000..313bb55bad19f --- /dev/null +++ b/server/src/main/java/org/opensearch/arrow/custom/StreamManagerWrapper.java @@ -0,0 +1,189 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.custom; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.opensearch.arrow.spi.StreamManager; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.arrow.spi.StreamReader; +import org.opensearch.arrow.spi.StreamTicket; +import org.opensearch.arrow.spi.StreamTicketFactory; +import org.opensearch.common.lease.Releasable; +import org.opensearch.common.lease.Releasables; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.core.tasks.TaskCancelledException; +import org.opensearch.core.tasks.TaskId; +import org.opensearch.tasks.Task; +import org.opensearch.tasks.TaskAwareRequest; +import org.opensearch.tasks.TaskManager; + +import java.io.IOException; +import java.util.function.Supplier; + +/** + * Wraps a StreamManager to make it work with the TaskManager. + */ +public class StreamManagerWrapper implements StreamManager { + + private final Supplier streamManager; + private final TaskManager taskManager; + + public StreamManagerWrapper(Supplier streamManager, TaskManager taskManager) { + super(); + this.streamManager = streamManager; + this.taskManager = taskManager; + } + + @Override + public StreamTicket registerStream(StreamProducer producer, TaskId parentTaskId) { + StreamProducerTaskWrapper wrappedProducer = new StreamProducerTaskWrapper(producer, taskManager, parentTaskId); + StreamTicket ticket = streamManager.get().registerStream(wrappedProducer, parentTaskId); + wrappedProducer.setDescription(ticket.toString()); + return ticket; + } + + @Override + public StreamReader getStreamReader(StreamTicket ticket) { + return streamManager.get().getStreamReader(ticket); + } + + @Override + public StreamTicketFactory getStreamTicketFactory() { + return streamManager.get().getStreamTicketFactory(); + } + + @Override + public void close() throws Exception { + streamManager.get().close(); + } + + static class StreamProducerTaskWrapper implements StreamProducer { + + private final StreamProducer streamProducer; + private final TaskManager taskManager; + private final TaskId parentTaskId; + private String description = ""; + + public StreamProducerTaskWrapper(StreamProducer streamProducer, TaskManager taskManager, TaskId parentTaskId) { + this.streamProducer = streamProducer; + this.taskManager = taskManager; + this.parentTaskId = parentTaskId; + } + + void setDescription(String description) { + this.description = description; + } + + @Override + public String getAction() { + return streamProducer.getAction(); + } + + @Override + public VectorSchemaRoot createRoot(BufferAllocator allocator) throws Exception { + return streamProducer.createRoot(allocator); + } + + @Override + public BatchedJob createJob(BufferAllocator allocator) { + BatchedJob job = streamProducer.createJob(allocator); + return new BatchedJobTaskWrapper(job, parentTaskId, taskManager, description, getAction()); + } + + @Override + public TimeValue getJobDeadline() { + return streamProducer.getJobDeadline(); + } + + @Override + public int estimatedRowCount() { + return streamProducer.estimatedRowCount(); + } + + @Override + public void close() throws IOException { + streamProducer.close(); + } + + static class BatchedJobTaskWrapper implements BatchedJob, TaskAwareRequest { + private final BatchedJob batchedJob; + private final TaskManager taskManager; + private TaskId parentTaskId; + private final String description; + private final String action; + + public BatchedJobTaskWrapper( + BatchedJob batchedJob, + TaskId parentTaskId, + TaskManager taskManager, + String description, + String action + ) { + this.batchedJob = batchedJob; + this.taskManager = taskManager; + this.parentTaskId = parentTaskId; + this.description = description; + this.action = action; + } + + @Override + public void run(VectorSchemaRoot root, FlushSignal flushSignal) throws Exception { + final Task task; + final Releasable unregisterChildNode = registerChildNode(); + try { + task = taskManager.register("stream_task", action, this); + } catch (TaskCancelledException e) { + unregisterChildNode.close(); + throw e; + } + + try (ThreadContext.StoredContext ignored = taskManager.taskExecutionStarted(task)) { + batchedJob.run(root, flushSignal); + } finally { + Releasables.close(unregisterChildNode, () -> taskManager.unregister(task)); + } + } + + @Override + public void onCancel() { + batchedJob.onCancel(); + } + + @Override + public boolean isCancelled() { + return batchedJob.isCancelled(); + } + + @Override + public void setParentTask(TaskId taskId) { + this.parentTaskId = taskId; + } + + @Override + public TaskId getParentTask() { + return parentTaskId; + } + + @Override + public String getDescription() { + return description; + } + + private Releasable registerChildNode() { + if (parentTaskId.isSet()) { + return taskManager.registerChildNode(parentTaskId.getId(), taskManager.localNode()); + } else { + return () -> {}; + } + } + } + } +} diff --git a/server/src/main/java/org/opensearch/arrow/custom/package-info.java b/server/src/main/java/org/opensearch/arrow/custom/package-info.java new file mode 100644 index 0000000000000..42debdeeafd97 --- /dev/null +++ b/server/src/main/java/org/opensearch/arrow/custom/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Contains all StreamProducer's and Arrow Flight Stream related customizations + */ +package org.opensearch.arrow.custom; diff --git a/server/src/main/java/org/opensearch/bootstrap/BootstrapChecks.java b/server/src/main/java/org/opensearch/bootstrap/BootstrapChecks.java index 0e0b4e9be261a..b7d3d94015bf1 100644 --- a/server/src/main/java/org/opensearch/bootstrap/BootstrapChecks.java +++ b/server/src/main/java/org/opensearch/bootstrap/BootstrapChecks.java @@ -712,9 +712,9 @@ static class AllPermissionCheck implements BootstrapCheck { @Override public final BootstrapCheckResult check(BootstrapContext context) { - if (isAllPermissionGranted()) { - return BootstrapCheck.BootstrapCheckResult.failure("granting the all permission effectively disables security"); - } + // if (isAllPermissionGranted()) { + // return BootstrapCheck.BootstrapCheckResult.failure("granting the all permission effectively disables security"); + // } return BootstrapCheckResult.success(); } diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 59d999798868e..6753bb8eac083 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -39,6 +39,7 @@ protected FeatureFlagSettings( FeatureFlags.STAR_TREE_INDEX_SETTING, FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING, FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL_SETTING, - FeatureFlags.TERM_VERSION_PRECOMMIT_ENABLE_SETTING + FeatureFlags.TERM_VERSION_PRECOMMIT_ENABLE_SETTING, + FeatureFlags.ARROW_STREAMS_SETTING ); } diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 6df68013a8119..4f0462f0b5cdd 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -128,6 +128,9 @@ public class FeatureFlags { Property.NodeScope ); + public static final String ARROW_STREAMS = "opensearch.experimental.feature.arrow.streams.enabled"; + public static final Setting ARROW_STREAMS_SETTING = Setting.boolSetting(ARROW_STREAMS, true, Property.NodeScope); + private static final List> ALL_FEATURE_FLAG_SETTINGS = List.of( REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, EXTENSIONS_SETTING, @@ -138,7 +141,8 @@ public class FeatureFlags { STAR_TREE_INDEX_SETTING, APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING, READER_WRITER_SPLIT_EXPERIMENTAL_SETTING, - TERM_VERSION_PRECOMMIT_ENABLE_SETTING + TERM_VERSION_PRECOMMIT_ENABLE_SETTING, + ARROW_STREAMS_SETTING ); /** diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index c2c4e68dae257..866d25b6e9d73 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -56,6 +56,8 @@ import org.opensearch.action.search.SearchTransportService; import org.opensearch.action.support.TransportAction; import org.opensearch.action.update.UpdateHelper; +import org.opensearch.arrow.custom.StreamManagerWrapper; +import org.opensearch.arrow.spi.StreamManager; import org.opensearch.bootstrap.BootstrapCheck; import org.opensearch.bootstrap.BootstrapContext; import org.opensearch.client.Client; @@ -218,6 +220,7 @@ import org.opensearch.plugins.SearchPipelinePlugin; import org.opensearch.plugins.SearchPlugin; import org.opensearch.plugins.SecureSettingsFactory; +import org.opensearch.plugins.StreamManagerPlugin; import org.opensearch.plugins.SystemIndexPlugin; import org.opensearch.plugins.TaskManagerClientPlugin; import org.opensearch.plugins.TelemetryAwarePlugin; @@ -240,6 +243,7 @@ import org.opensearch.search.fetch.FetchPhase; import org.opensearch.search.pipeline.SearchPipelineService; import org.opensearch.search.query.QueryPhase; +import org.opensearch.search.query.StreamSearchPhase; import org.opensearch.snapshots.InternalSnapshotsInfoService; import org.opensearch.snapshots.RestoreService; import org.opensearch.snapshots.SnapshotShardsService; @@ -307,11 +311,13 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.Collectors; import java.util.stream.Stream; import static java.util.stream.Collectors.toList; +import static org.opensearch.common.util.FeatureFlags.ARROW_STREAMS_SETTING; import static org.opensearch.common.util.FeatureFlags.BACKGROUND_TASK_EXECUTION_EXPERIMENTAL; import static org.opensearch.common.util.FeatureFlags.TELEMETRY; import static org.opensearch.env.NodeEnvironment.collectFileCacheDataPath; @@ -1377,7 +1383,22 @@ protected Node( admissionControlService, cacheService ); - + StreamManager streamManager = null; + if (FeatureFlags.isEnabled(ARROW_STREAMS_SETTING)) { + List streamManagerPlugins = pluginsService.filterPlugins(StreamManagerPlugin.class); + if (streamManagerPlugins.size() > 1) { + throw new IllegalStateException( + String.format(Locale.ROOT, "Only one StreamManagerPlugin can be installed. Found: %d", streamManagerPlugins.size()) + ); + } + if (!streamManagerPlugins.isEmpty()) { + Supplier baseStreamManager = streamManagerPlugins.get(0).getStreamManager(); + if (baseStreamManager != null) { + streamManager = new StreamManagerWrapper(baseStreamManager, transportService.getTaskManager()); + logger.info("StreamManager initialized"); + } + } + } final SearchService searchService = newSearchService( clusterService, indicesService, @@ -1386,11 +1407,13 @@ protected Node( bigArrays, searchModule.getQueryPhase(), searchModule.getFetchPhase(), + searchModule.getStreamPhase(), responseCollectorService, circuitBreakerService, searchModule.getIndexSearcherExecutor(threadPool), taskResourceTrackingService, - searchModule.getConcurrentSearchRequestDeciderFactories() + searchModule.getConcurrentSearchRequestDeciderFactories(), + streamManager ); final List> tasksExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class) @@ -2052,11 +2075,13 @@ protected SearchService newSearchService( BigArrays bigArrays, QueryPhase queryPhase, FetchPhase fetchPhase, + StreamSearchPhase streamSearchPhase, ResponseCollectorService responseCollectorService, CircuitBreakerService circuitBreakerService, Executor indexSearcherExecutor, TaskResourceTrackingService taskResourceTrackingService, - Collection concurrentSearchDeciderFactories + Collection concurrentSearchDeciderFactories, + StreamManager streamManager ) { return new SearchService( clusterService, @@ -2066,11 +2091,13 @@ protected SearchService newSearchService( bigArrays, queryPhase, fetchPhase, + streamSearchPhase, responseCollectorService, circuitBreakerService, indexSearcherExecutor, taskResourceTrackingService, - concurrentSearchDeciderFactories + concurrentSearchDeciderFactories, + streamManager ); } @@ -2143,7 +2170,6 @@ public DiscoveryNode apply(BoundTransportAddress boundTransportAddress) { if (isRemoteStoreAttributePresent(settings)) { remoteStoreNodeService.createAndVerifyRepositories(discoveryNode); } - localNode.set(discoveryNode); return localNode.get(); } diff --git a/server/src/main/java/org/opensearch/plugins/DefaultSecureTransportParameters.java b/server/src/main/java/org/opensearch/plugins/DefaultSecureTransportParameters.java index e3771f224a7db..78dbcc732c380 100644 --- a/server/src/main/java/org/opensearch/plugins/DefaultSecureTransportParameters.java +++ b/server/src/main/java/org/opensearch/plugins/DefaultSecureTransportParameters.java @@ -11,6 +11,11 @@ import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.TrustManagerFactory; + +import java.util.Optional; + /** * Default implementation of {@link SecureTransportSettingsProvider.SecureTransportParameters}. */ @@ -25,4 +30,34 @@ class DefaultSecureTransportParameters implements SecureTransportSettingsProvide public boolean dualModeEnabled() { return NetworkModule.TRANSPORT_SSL_DUAL_MODE_ENABLED.get(settings); } + + @Override + public Optional keyManagerFactory() { + return Optional.empty(); + } + + @Override + public String sslProvider() { + return ""; + } + + @Override + public String clientAuth() { + return ""; + } + + @Override + public String[] protocols() { + return null; + } + + @Override + public Iterable cipherSuites() { + return null; + } + + @Override + public TrustManagerFactory trustManagerFactory() { + return null; + } } diff --git a/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java b/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java index 5f9e1a952b6e8..05b26c0f4a073 100644 --- a/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java +++ b/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java @@ -13,8 +13,10 @@ import org.opensearch.transport.Transport; import org.opensearch.transport.TransportAdapterProvider; +import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLException; +import javax.net.ssl.TrustManagerFactory; import java.util.Collection; import java.util.Collections; @@ -52,6 +54,18 @@ default Optional parameters(Settings settings) { @ExperimentalApi interface SecureTransportParameters { boolean dualModeEnabled(); + + Optional keyManagerFactory(); + + String sslProvider(); + + String clientAuth(); + + String[] protocols(); + + Iterable cipherSuites(); + + TrustManagerFactory trustManagerFactory(); } /** diff --git a/server/src/main/java/org/opensearch/plugins/StreamManagerPlugin.java b/server/src/main/java/org/opensearch/plugins/StreamManagerPlugin.java new file mode 100644 index 0000000000000..aefde6b4842d0 --- /dev/null +++ b/server/src/main/java/org/opensearch/plugins/StreamManagerPlugin.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugins; + +import org.opensearch.arrow.spi.StreamManager; + +import java.util.function.Supplier; + +/** + * An interface for OpenSearch plugins to implement to provide a StreamManager. + * This interface is used by the Arrow Flight plugin to get the StreamManager instance. + * Other plugins can also implement this interface to provide their own StreamManager implementation. + * @see StreamManager + */ +public interface StreamManagerPlugin { + /** + * Returns the StreamManager instance for this plugin. + * + * @return The StreamManager instance + */ + Supplier getStreamManager(); +} diff --git a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java index 74a7482d975df..29504e5e0c5dd 100644 --- a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java @@ -45,6 +45,7 @@ import org.opensearch.Version; import org.opensearch.action.search.SearchShardTask; import org.opensearch.action.search.SearchType; +import org.opensearch.arrow.spi.StreamManager; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; @@ -98,6 +99,7 @@ import org.opensearch.search.rescore.RescoreContext; import org.opensearch.search.slice.SliceBuilder; import org.opensearch.search.sort.SortAndFormats; +import org.opensearch.search.stream.StreamSearchResult; import org.opensearch.search.suggest.SuggestionSearchContext; import java.io.IOException; @@ -144,9 +146,11 @@ final class DefaultSearchContext extends SearchContext { private final IndexShard indexShard; private final ClusterService clusterService; private final IndexService indexService; + private final StreamManager streamManager; private final ContextIndexSearcher searcher; private final DfsSearchResult dfsResult; private final QuerySearchResult queryResult; + private final StreamSearchResult streamSearchResult; private final FetchSearchResult fetchResult; private final float queryBoost; private final boolean lowLevelCancellation; @@ -224,7 +228,8 @@ final class DefaultSearchContext extends SearchContext { boolean validate, Executor executor, Function requestToAggReduceContextBuilder, - Collection concurrentSearchDeciderFactories + Collection concurrentSearchDeciderFactories, + StreamManager streamManager ) throws IOException { this.readerContext = readerContext; this.request = request; @@ -235,6 +240,7 @@ final class DefaultSearchContext extends SearchContext { this.bigArrays = bigArrays.withCircuitBreaking(); this.dfsResult = new DfsSearchResult(readerContext.id(), shardTarget, request); this.queryResult = new QuerySearchResult(readerContext.id(), shardTarget, request); + this.streamSearchResult = new StreamSearchResult(readerContext.id(), shardTarget, request); this.fetchResult = new FetchSearchResult(readerContext.id(), shardTarget); this.indexService = readerContext.indexService(); this.indexShard = readerContext.indexShard(); @@ -270,6 +276,7 @@ final class DefaultSearchContext extends SearchContext { this.cardinalityAggregationPruningThreshold = evaluateCardinalityAggregationPruningThreshold(); this.concurrentSearchDeciderFactories = concurrentSearchDeciderFactories; this.keywordIndexOrDocValuesEnabled = evaluateKeywordIndexOrDocValuesEnabled(); + this.streamManager = streamManager; } @Override @@ -604,6 +611,11 @@ public SimilarityService similarityService() { return indexService.similarityService(); } + @Override + public StreamManager streamManager() { + return streamManager; + } + @Override public BigArrays bigArrays() { return bigArrays; @@ -879,6 +891,11 @@ public QuerySearchResult queryResult() { return queryResult; } + @Override + public StreamSearchResult streamSearchResult() { + return streamSearchResult; + } + @Override public FetchPhase fetchPhase() { return fetchPhase; diff --git a/server/src/main/java/org/opensearch/search/SearchModule.java b/server/src/main/java/org/opensearch/search/SearchModule.java index 24bdb2d28cba2..d1371853bc06f 100644 --- a/server/src/main/java/org/opensearch/search/SearchModule.java +++ b/server/src/main/java/org/opensearch/search/SearchModule.java @@ -260,6 +260,7 @@ import org.opensearch.search.query.QueryPhase; import org.opensearch.search.query.QueryPhaseSearcher; import org.opensearch.search.query.QueryPhaseSearcherWrapper; +import org.opensearch.search.query.StreamSearchPhase; import org.opensearch.search.rescore.QueryRescorerBuilder; import org.opensearch.search.rescore.RescorerBuilder; import org.opensearch.search.sort.FieldSortBuilder; @@ -1308,4 +1309,8 @@ public QueryPhase getQueryPhase() { public @Nullable ExecutorService getIndexSearcherExecutor(ThreadPool pool) { return (indexSearcherExecutorProvider != null) ? indexSearcherExecutorProvider.getExecutor(pool) : null; } + + public StreamSearchPhase getStreamPhase() { + return new StreamSearchPhase(); + } } diff --git a/server/src/main/java/org/opensearch/search/SearchPhaseResult.java b/server/src/main/java/org/opensearch/search/SearchPhaseResult.java index a351b3bd2dda6..af7643455021c 100644 --- a/server/src/main/java/org/opensearch/search/SearchPhaseResult.java +++ b/server/src/main/java/org/opensearch/search/SearchPhaseResult.java @@ -41,6 +41,7 @@ import org.opensearch.search.internal.ShardSearchContextId; import org.opensearch.search.internal.ShardSearchRequest; import org.opensearch.search.query.QuerySearchResult; +import org.opensearch.search.stream.StreamSearchResult; import java.io.IOException; @@ -106,6 +107,10 @@ public void setShardIndex(int shardIndex) { * Returns the query result iff it's included in this response otherwise null */ public QuerySearchResult queryResult() { + return QuerySearchResult.nullInstance(); + } + + public StreamSearchResult streamSearchResult() { return null; } diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java index d4380eb09e360..b27e4dd9462eb 100644 --- a/server/src/main/java/org/opensearch/search/SearchService.java +++ b/server/src/main/java/org/opensearch/search/SearchService.java @@ -49,6 +49,7 @@ import org.opensearch.action.search.UpdatePitContextRequest; import org.opensearch.action.search.UpdatePitContextResponse; import org.opensearch.action.support.TransportActions; +import org.opensearch.arrow.spi.StreamManager; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.CheckedSupplier; @@ -133,6 +134,7 @@ import org.opensearch.search.query.QuerySearchRequest; import org.opensearch.search.query.QuerySearchResult; import org.opensearch.search.query.ScrollQuerySearchResult; +import org.opensearch.search.query.StreamSearchPhase; import org.opensearch.search.rescore.RescorerBuilder; import org.opensearch.search.searchafter.SearchAfterBuilder; import org.opensearch.search.sort.FieldSortBuilder; @@ -141,6 +143,7 @@ import org.opensearch.search.sort.SortBuilder; import org.opensearch.search.sort.SortOrder; import org.opensearch.search.startree.StarTreeQueryContext; +import org.opensearch.search.stream.StreamSearchResult; import org.opensearch.search.suggest.Suggest; import org.opensearch.search.suggest.completion.CompletionSuggestion; import org.opensearch.tasks.TaskResourceTrackingService; @@ -368,6 +371,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv private final QueryPhase queryPhase; + private final StreamSearchPhase streamSearchPhase; + private final FetchPhase fetchPhase; private final Collection concurrentSearchDeciderFactories; @@ -402,6 +407,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv private final String sessionId = UUIDs.randomBase64UUID(); private final Executor indexSearcherExecutor; private final TaskResourceTrackingService taskResourceTrackingService; + private final StreamManager streamManager; public SearchService( ClusterService clusterService, @@ -411,11 +417,13 @@ public SearchService( BigArrays bigArrays, QueryPhase queryPhase, FetchPhase fetchPhase, + StreamSearchPhase streamSearchPhase, ResponseCollectorService responseCollectorService, CircuitBreakerService circuitBreakerService, Executor indexSearcherExecutor, TaskResourceTrackingService taskResourceTrackingService, - Collection concurrentSearchDeciderFactories + Collection concurrentSearchDeciderFactories, + StreamManager streamManager ) { Settings settings = clusterService.getSettings(); this.threadPool = threadPool; @@ -426,6 +434,7 @@ public SearchService( this.bigArrays = bigArrays; this.queryPhase = queryPhase; this.fetchPhase = fetchPhase; + this.streamSearchPhase = streamSearchPhase; this.multiBucketConsumerService = new MultiBucketConsumerService( clusterService, settings, @@ -443,7 +452,7 @@ public SearchService( this::setPitKeepAlives, this::validatePitKeepAlives ); - + this.streamManager = streamManager; clusterService.getClusterSettings() .addSettingsUpdateConsumer(DEFAULT_KEEPALIVE_SETTING, MAX_KEEPALIVE_SETTING, this::setKeepAlives, this::validateKeepAlives); @@ -648,6 +657,16 @@ private void loadOrExecuteQueryPhase(final ShardSearchRequest request, final Sea } } + private void loadOrExecuteStreamPhase(final ShardSearchRequest request, final SearchContext context) throws Exception { + final boolean canCache = false;// indicesService.canCache(request, context); + context.getQueryShardContext().freezeContext(); + if (canCache) { + indicesService.loadIntoContext(request, context, queryPhase); + } else { + streamSearchPhase.execute(context); + } + } + public void executeQueryPhase( ShardSearchRequest request, boolean keepStatesInContext, @@ -687,6 +706,45 @@ public void onFailure(Exception exc) { }); } + public void executeStreamPhase( + ShardSearchRequest request, + boolean keepStatesInContext, + SearchShardTask task, + ActionListener listener + ) { + assert request.canReturnNullResponseIfMatchNoDocs() == false || request.numberOfShards() > 1 + : "empty responses require more than one shard"; + final IndexShard shard = getShard(request); + rewriteAndFetchShardRequest(shard, request, new ActionListener() { + @Override + public void onResponse(ShardSearchRequest orig) { + // check if we can shortcut the query phase entirely. + if (orig.canReturnNullResponseIfMatchNoDocs()) { + assert orig.scroll() == null; + final CanMatchResponse canMatchResp; + try { + ShardSearchRequest clone = new ShardSearchRequest(orig); + canMatchResp = canMatch(clone, false); + } catch (Exception exc) { + listener.onFailure(exc); + return; + } + if (canMatchResp.canMatch == false) { + listener.onResponse(new StreamSearchResult()); + return; + } + } + // fork the execution in the search thread pool + runAsync(getExecutor(shard), () -> executeStreamPhase(orig, task, keepStatesInContext), listener); + } + + @Override + public void onFailure(Exception exc) { + listener.onFailure(exc); + } + }); + } + private IndexShard getShard(ShardSearchRequest request) { if (request.readerId() != null) { return findReaderContext(request.readerId(), request).indexShard(); @@ -740,6 +798,35 @@ private SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchSh } } + private StreamSearchResult executeStreamPhase(ShardSearchRequest request, SearchShardTask task, boolean keepStatesInContext) + throws Exception { + final ReaderContext readerContext = createOrGetReaderContext(request, keepStatesInContext); + try ( + Releasable ignored = readerContext.markAsUsed(getKeepAlive(request)); + SearchContext context = createContext(readerContext, request, task, true) + ) { + try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) { + loadOrExecuteStreamPhase(request, context); + if (context.queryResult().hasSearchContext() == false && readerContext.singleSession()) { + freeReaderContext(readerContext.id()); + } + } + return context.streamSearchResult(); + } catch (Exception e) { + // execution exception can happen while loading the cache, strip it + if (e instanceof ExecutionException) { + e = (e.getCause() == null || e.getCause() instanceof Exception) + ? (Exception) e.getCause() + : new OpenSearchException(e.getCause()); + } + logger.trace("Query phase failed", e); + processFailure(readerContext, e); + throw e; + } finally { + taskResourceTrackingService.writeTaskResourceUsage(task, clusterService.localNode().getId()); + } + } + private QueryFetchSearchResult executeFetchPhase(ReaderContext reader, SearchContext context, long afterQueryTime) { try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context, true, afterQueryTime)) { shortcutDocIdsToLoad(context); @@ -788,6 +875,40 @@ public void executeQueryPhase( }, wrapFailureListener(listener, readerContext, markAsUsed)); } + public void executeStreamPhase(QuerySearchRequest request, SearchShardTask task, ActionListener listener) { + final ReaderContext readerContext = findReaderContext(request.contextId(), request.shardSearchRequest()); + final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.shardSearchRequest()); + final Releasable markAsUsed = readerContext.markAsUsed(getKeepAlive(shardSearchRequest)); + runAsync(getExecutor(readerContext.indexShard()), () -> { + readerContext.setAggregatedDfs(request.dfs()); + try ( + SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, true); + SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext) + ) { + searchContext.searcher().setAggregatedDfs(request.dfs()); + queryPhase.execute(searchContext); + if (searchContext.queryResult().hasSearchContext() == false && readerContext.singleSession()) { + // no hits, we can release the context since there will be no fetch phase + freeReaderContext(readerContext.id()); + } + executor.success(); + // Pass the rescoreDocIds to the queryResult to send them the coordinating node and receive them back in the fetch phase. + // We also pass the rescoreDocIds to the LegacyReaderContext in case the search state needs to stay in the data node. + final RescoreDocIds rescoreDocIds = searchContext.rescoreDocIds(); + searchContext.queryResult().setRescoreDocIds(rescoreDocIds); + readerContext.setRescoreDocIds(rescoreDocIds); + return null; + } catch (Exception e) { + assert TransportActions.isShardNotAvailableException(e) == false : new AssertionError(e); + logger.trace("Query phase failed", e); + // we handle the failure in the failure listener below + throw e; + } finally { + taskResourceTrackingService.writeTaskResourceUsage(task, clusterService.localNode().getId()); + } + }, wrapFailureListener(listener, readerContext, markAsUsed)); + } + public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task, ActionListener listener) { final ReaderContext readerContext = findReaderContext(request.contextId(), request.shardSearchRequest()); final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(request.shardSearchRequest()); @@ -1173,7 +1294,8 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear validate, indexSearcherExecutor, this::aggReduceContextBuilder, - concurrentSearchDeciderFactories + concurrentSearchDeciderFactories, + streamManager ); // we clone the query shard context here just for rewriting otherwise we // might end up with incorrect state since we are using now() or script services @@ -1657,7 +1779,8 @@ public CanMatchResponse canMatch(ShardSearchRequest request) throws IOException } private CanMatchResponse canMatch(ShardSearchRequest request, boolean checkRefreshPending) throws IOException { - assert request.searchType() == SearchType.QUERY_THEN_FETCH : "unexpected search type: " + request.searchType(); + assert (request.searchType() == SearchType.QUERY_THEN_FETCH || request.searchType() == SearchType.STREAM) + : "unexpected search type: " + request.searchType(); final ReaderContext readerContext = request.readerId() != null ? findReaderContext(request.readerId(), request) : null; final Releasable markAsUsed = readerContext != null ? readerContext.markAsUsed(getKeepAlive(request)) : () -> {}; try (Releasable ignored = markAsUsed) { diff --git a/server/src/main/java/org/opensearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/opensearch/search/internal/FilteredSearchContext.java index 3a3b46366a6d2..a323155a9c8a4 100644 --- a/server/src/main/java/org/opensearch/search/internal/FilteredSearchContext.java +++ b/server/src/main/java/org/opensearch/search/internal/FilteredSearchContext.java @@ -67,6 +67,7 @@ import org.opensearch.search.query.ReduceableSearchResult; import org.opensearch.search.rescore.RescoreContext; import org.opensearch.search.sort.SortAndFormats; +import org.opensearch.search.stream.StreamSearchResult; import org.opensearch.search.suggest.SuggestionSearchContext; import java.util.List; @@ -468,6 +469,11 @@ public QuerySearchResult queryResult() { return in.queryResult(); } + @Override + public StreamSearchResult streamSearchResult() { + return in.streamSearchResult(); + } + @Override public FetchSearchResult fetchResult() { return in.fetchResult(); diff --git a/server/src/main/java/org/opensearch/search/internal/InternalSearchResponse.java b/server/src/main/java/org/opensearch/search/internal/InternalSearchResponse.java index 29c8826f76957..859c0b32bc1dc 100644 --- a/server/src/main/java/org/opensearch/search/internal/InternalSearchResponse.java +++ b/server/src/main/java/org/opensearch/search/internal/InternalSearchResponse.java @@ -44,6 +44,7 @@ import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.pipeline.ProcessorExecutionDetail; import org.opensearch.search.profile.SearchProfileShardResults; +import org.opensearch.search.stream.OSTicket; import org.opensearch.search.suggest.Suggest; import java.io.IOException; @@ -119,8 +120,33 @@ public InternalSearchResponse( boolean timedOut, Boolean terminatedEarly, int numReducePhases, - List searchExtBuilderList + List searchExtBuilderList, + List processorResult, + List osTickets + ) { + super( + hits, + aggregations, + suggest, + timedOut, + terminatedEarly, + profileResults, + numReducePhases, + searchExtBuilderList, + processorResult, + osTickets + ); + } + public InternalSearchResponse( + SearchHits hits, + InternalAggregations aggregations, + Suggest suggest, + SearchProfileShardResults profileResults, + boolean timedOut, + Boolean terminatedEarly, + int numReducePhases, + List searchExtBuilderList ) { super( hits, @@ -131,7 +157,8 @@ public InternalSearchResponse( profileResults, numReducePhases, searchExtBuilderList, - Collections.emptyList() + Collections.emptyList(), + null ); } @@ -145,7 +172,8 @@ public InternalSearchResponse(StreamInput in) throws IOException { in.readOptionalWriteable(SearchProfileShardResults::new), in.readVInt(), readSearchExtBuildersOnOrAfter(in), - readProcessorResultOnOrAfter(in) + readProcessorResultOnOrAfter(in), + (in.readBoolean() ? in.readList(OSTicket::new) : null) ); } @@ -160,6 +188,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(numReducePhases); writeSearchExtBuildersOnOrAfter(out, searchExtBuilders); writeProcessorResultOnOrAfter(out, processorResult); + if (tickets != null && !tickets.isEmpty()) { + out.writeBoolean(true); + out.writeList(tickets); + } } private static List readSearchExtBuildersOnOrAfter(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/opensearch/search/internal/SearchContext.java b/server/src/main/java/org/opensearch/search/internal/SearchContext.java index b7ea06d2989e5..93b9ad931a612 100644 --- a/server/src/main/java/org/opensearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/opensearch/search/internal/SearchContext.java @@ -37,6 +37,7 @@ import org.apache.lucene.search.Query; import org.opensearch.action.search.SearchShardTask; import org.opensearch.action.search.SearchType; +import org.opensearch.arrow.spi.StreamManager; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.lease.Releasable; @@ -77,6 +78,7 @@ import org.opensearch.search.rescore.RescoreContext; import org.opensearch.search.sort.SortAndFormats; import org.opensearch.search.startree.StarTreeQueryContext; +import org.opensearch.search.stream.StreamSearchResult; import org.opensearch.search.suggest.SuggestionSearchContext; import java.util.Collection; @@ -280,6 +282,8 @@ public final void assignRescoreDocIds(RescoreDocIds rescoreDocIds) { public abstract SimilarityService similarityService(); + public abstract StreamManager streamManager(); + public abstract BigArrays bigArrays(); public abstract BitsetFilterCache bitsetFilterCache(); @@ -418,6 +422,8 @@ public boolean includeNamedQueriesScore() { public abstract QuerySearchResult queryResult(); + public abstract StreamSearchResult streamSearchResult(); + public abstract FetchPhase fetchPhase(); public abstract FetchSearchResult fetchResult(); diff --git a/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java index de1d5fb8b4098..213121c273d10 100644 --- a/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/opensearch/search/internal/ShardSearchRequest.java @@ -425,6 +425,10 @@ public String preference() { return preference; } + public boolean isStreamRequest() { + return searchType == SearchType.STREAM; + } + /** * Sets the bottom sort values that can be used by the searcher to filter documents * that are after it. This value is computed by coordinating nodes that throttles the diff --git a/server/src/main/java/org/opensearch/search/internal/SubSearchContext.java b/server/src/main/java/org/opensearch/search/internal/SubSearchContext.java index b2c97baf78d91..c6694ec3bab6f 100644 --- a/server/src/main/java/org/opensearch/search/internal/SubSearchContext.java +++ b/server/src/main/java/org/opensearch/search/internal/SubSearchContext.java @@ -32,6 +32,7 @@ package org.opensearch.search.internal; import org.apache.lucene.search.Query; +import org.opensearch.arrow.spi.StreamManager; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.query.ParsedQuery; import org.opensearch.search.aggregations.SearchContextAggregations; @@ -180,6 +181,11 @@ public SearchContext fetchFieldsContext(FetchFieldsContext fetchFieldsContext) { return this; } + @Override + public StreamManager streamManager() { + return null; + } + @Override public void timeout(TimeValue timeout) { throw new UnsupportedOperationException("Not supported"); diff --git a/server/src/main/java/org/opensearch/search/lookup/SearchLookup.java b/server/src/main/java/org/opensearch/search/lookup/SearchLookup.java index dff8fae1a9ad1..6d8e3330bc042 100644 --- a/server/src/main/java/org/opensearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/opensearch/search/lookup/SearchLookup.java @@ -51,7 +51,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class SearchLookup { +public class /**/ SearchLookup { /** * The maximum depth of field dependencies. * When a runtime field's doc values depends on another runtime field's doc values, diff --git a/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java b/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java index 08b048cf682bb..c48b8867b43f2 100644 --- a/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java +++ b/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java @@ -145,7 +145,7 @@ void postProcess(QuerySearchResult result) throws IOException {} * Creates the collector tree from the provided collectors * @param collectors Ordered list of collector context */ - static Collector createQueryCollector(List collectors) throws IOException { + public static Collector createQueryCollector(List collectors) throws IOException { Collector collector = null; for (QueryCollectorContext ctx : collectors) { collector = ctx.create(collector); diff --git a/server/src/main/java/org/opensearch/search/query/StreamSearchPhase.java b/server/src/main/java/org/opensearch/search/query/StreamSearchPhase.java new file mode 100644 index 0000000000000..2bdd94f2edfab --- /dev/null +++ b/server/src/main/java/org/opensearch/search/query/StreamSearchPhase.java @@ -0,0 +1,213 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.Query; +import org.opensearch.OpenSearchException; +import org.opensearch.arrow.custom.ArrowDocIdCollector; +import org.opensearch.arrow.spi.StreamManager; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.arrow.spi.StreamTicket; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.search.SearchContextSourcePrinter; +import org.opensearch.search.aggregations.AggregationProcessor; +import org.opensearch.search.internal.ContextIndexSearcher; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.profile.ProfileShardResult; +import org.opensearch.search.profile.SearchProfileShardResults; +import org.opensearch.search.stream.OSTicket; +import org.opensearch.search.stream.StreamSearchResult; + +import java.io.IOException; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +/** + * StreamSearchPhase is the search phase for streaming search. + */ +public class StreamSearchPhase extends QueryPhase { + + private static final Logger LOGGER = LogManager.getLogger(StreamSearchPhase.class); + public static final QueryPhaseSearcher DEFAULT_QUERY_PHASE_SEARCHER = new DefaultStreamSearchPhaseSearcher(); + + public StreamSearchPhase() { + super(DEFAULT_QUERY_PHASE_SEARCHER); + } + + @Override + public void execute(SearchContext searchContext) throws QueryPhaseExecutionException { + + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("{}", new SearchContextSourcePrinter(searchContext)); + } + executeInternal(searchContext, this.getQueryPhaseSearcher()); + if (searchContext.getProfilers() != null) { + ProfileShardResult shardResults = SearchProfileShardResults.buildShardResults( + searchContext.getProfilers(), + searchContext.request() + ); + searchContext.queryResult().profileResults(shardResults); + } + } + + /** + * DefaultStreamSearchPhaseSearcher + */ + public static class DefaultStreamSearchPhaseSearcher extends DefaultQueryPhaseSearcher { + + @Override + public boolean searchWith( + SearchContext searchContext, + ContextIndexSearcher searcher, + Query query, + LinkedList collectors, + boolean hasFilterCollector, + boolean hasTimeout + ) { + return searchWithCollector(searchContext, searcher, query, collectors, hasFilterCollector, hasTimeout); + } + + @Override + public AggregationProcessor aggregationProcessor(SearchContext searchContext) { + return new AggregationProcessor() { + @Override + public void preProcess(SearchContext context) { + + } + + @Override + public void postProcess(SearchContext context) { + + } + }; + } + + protected boolean searchWithCollector( + SearchContext searchContext, + ContextIndexSearcher searcher, + Query query, + LinkedList collectors, + boolean hasFilterCollector, + boolean hasTimeout + ) { + return searchWithCollector(searchContext, searcher, query, collectors, hasTimeout); + } + + private boolean searchWithCollector( + SearchContext searchContext, + ContextIndexSearcher searcher, + Query query, + LinkedList collectors, + boolean timeoutSet + ) { + + QuerySearchResult queryResult = searchContext.queryResult(); + StreamManager streamManager = searchContext.streamManager(); + if (streamManager == null) { + throw new RuntimeException("StreamManager not setup"); + } + final boolean[] isCancelled = { false }; + StreamTicket ticket = streamManager.registerStream(new StreamProducer() { + + @Override + public void close() { + isCancelled[0] = true; + } + + @Override + public BatchedJob createJob(BufferAllocator allocator) { + return new BatchedJob() { + + @Override + public void run(VectorSchemaRoot root, StreamProducer.FlushSignal flushSignal) { + try { + Collector collector = QueryCollectorContext.createQueryCollector(collectors); + final ArrowDocIdCollector arrowDocIdCollector = new ArrowDocIdCollector(collector, root, flushSignal, 1000); + try { + searcher.addQueryCancellation(() -> { + if (isCancelled[0] == true) { + throw new OpenSearchException("Stream for query results cancelled."); + } + }); + searcher.search(query, arrowDocIdCollector); + } catch (EarlyTerminatingCollector.EarlyTerminationException e) { + // EarlyTerminationException is not caught in ContextIndexSearcher to allow force termination of + // collection. Postcollection + // still needs to be processed for Aggregations when early termination takes place. + searchContext.bucketCollectorProcessor().processPostCollection(arrowDocIdCollector); + queryResult.terminatedEarly(true); + } + if (searchContext.isSearchTimedOut()) { + assert timeoutSet : "TimeExceededException thrown even though timeout wasn't set"; + if (searchContext.request().allowPartialSearchResults() == false) { + throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Time exceeded"); + } + queryResult.searchTimedOut(true); + } + if (searchContext.terminateAfter() != SearchContext.DEFAULT_TERMINATE_AFTER + && queryResult.terminatedEarly() == null) { + queryResult.terminatedEarly(false); + } + + for (QueryCollectorContext ctx : collectors) { + ctx.postProcess(queryResult); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void onCancel() { + isCancelled[0] = true; + } + + @Override + public boolean isCancelled() { + return searchContext.isCancelled() || isCancelled(); + } + }; + } + + @Override + public TimeValue getJobDeadline() { + return TimeValue.timeValueMinutes(5); + } + + @Override + public VectorSchemaRoot createRoot(BufferAllocator allocator) { + IntVector docIDVector = new IntVector("docID", allocator); + FieldVector[] vectors = new FieldVector[] { docIDVector }; + return new VectorSchemaRoot(Arrays.asList(vectors)); + } + + @Override + public int estimatedRowCount() { + return searcher.getIndexReader().numDocs(); + } + + @Override + public String getAction() { + return searchContext.getTask().getAction(); + } + }, searchContext.getTask().getParentTaskId()); + StreamSearchResult streamSearchResult = searchContext.streamSearchResult(); + streamSearchResult.flights(List.of(new OSTicket(ticket.toBytes()))); + return false; + } + } +} diff --git a/server/src/main/java/org/opensearch/search/stream/OSTicket.java b/server/src/main/java/org/opensearch/search/stream/OSTicket.java new file mode 100644 index 0000000000000..4024b0b6410e7 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/stream/OSTicket.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.stream; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +/** + * A ticket for a stream. + */ +@ExperimentalApi +public class OSTicket implements Writeable, ToXContentFragment { + + private final byte[] bytes; + + public OSTicket(byte[] bytes) { + this.bytes = bytes; + } + + public OSTicket(StreamInput in) throws IOException { + bytes = in.readByteArray(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.value(new String(bytes, StandardCharsets.UTF_8)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeByteArray(bytes); + } + + @Override + public String toString() { + return "OSTicket{" + new String(bytes, StandardCharsets.UTF_8) + "}"; + } +} diff --git a/server/src/main/java/org/opensearch/search/stream/StreamSearchResult.java b/server/src/main/java/org/opensearch/search/stream/StreamSearchResult.java new file mode 100644 index 0000000000000..2e186e1897974 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/stream/StreamSearchResult.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.stream; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.search.SearchPhaseResult; +import org.opensearch.search.SearchShardTarget; +import org.opensearch.search.internal.ShardSearchContextId; +import org.opensearch.search.internal.ShardSearchRequest; +import org.opensearch.search.query.QuerySearchResult; + +import java.io.IOException; +import java.util.List; + +/** + * A result of stream search execution. + */ +@ExperimentalApi +public class StreamSearchResult extends SearchPhaseResult { + private List flightTickets; + private final QuerySearchResult queryResult; + + public StreamSearchResult() { + super(); + this.queryResult = QuerySearchResult.nullInstance(); + } + + public StreamSearchResult(StreamInput in) throws IOException { + super(in); + contextId = new ShardSearchContextId(in); + setShardSearchRequest(in.readOptionalWriteable(ShardSearchRequest::new)); + if (in.readOptionalBoolean()) { + flightTickets = in.readList(OSTicket::new); + } + queryResult = new QuerySearchResult(contextId, getSearchShardTarget(), getShardSearchRequest()); + setSearchShardTarget(getSearchShardTarget()); + } + + public StreamSearchResult(ShardSearchContextId id, SearchShardTarget shardTarget, ShardSearchRequest searchRequest) { + this.contextId = id; + queryResult = new QuerySearchResult(id, shardTarget, searchRequest); + setSearchShardTarget(shardTarget); + setShardSearchRequest(searchRequest); + } + + public void flights(List flightTickets) { + this.flightTickets = flightTickets; + } + + @Override + public void setSearchShardTarget(SearchShardTarget shardTarget) { + super.setSearchShardTarget(shardTarget); + queryResult.setSearchShardTarget(shardTarget); + } + + @Override + public void setShardIndex(int shardIndex) { + super.setShardIndex(shardIndex); + queryResult.setShardIndex(shardIndex); + } + + @Override + public QuerySearchResult queryResult() { + return queryResult; + } + + public List getFlightTickets() { + return flightTickets; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + contextId.writeTo(out); + out.writeOptionalWriteable(getShardSearchRequest()); + if (flightTickets != null) { + out.writeOptionalBoolean(true); + out.writeList(flightTickets); + } + } +} diff --git a/server/src/main/java/org/opensearch/search/stream/package-info.java b/server/src/main/java/org/opensearch/search/stream/package-info.java new file mode 100644 index 0000000000000..dda450c26d3b8 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/stream/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Search module for all things streaming search. + */ +package org.opensearch.search.stream; diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy index 22e445f7d9022..b3a04b4248b0d 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/security.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy @@ -92,9 +92,22 @@ grant codeBase "${codebase.reactor-core}" { permission java.net.SocketPermission "*", "connect,resolve"; }; +grant codeBase "${codebase.opensearch-arrow-memory-shaded}" { + permission java.lang.RuntimePermission "modifyThreadGroup"; + permission java.lang.RuntimePermission "modifyThread"; + + // Reflection access needed by Arrow + permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + + // Memory access + permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; +}; + //// Everything else: grant { + // needed by vendored Guice permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.vm.annotation"; @@ -194,4 +207,7 @@ grant { permission java.io.FilePermission "/sys/fs/cgroup/cpuacct/-", "read"; permission java.io.FilePermission "/sys/fs/cgroup/memory", "read"; permission java.io.FilePermission "/sys/fs/cgroup/memory/-", "read"; + + // Needed for netty based arrow flight server for netty configs related to buffer allocator + permission java.security.AllPermission "modifyThreadGroup"; }; diff --git a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java index 55b30d5068daa..ab6f3cf5baa9c 100644 --- a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java @@ -232,7 +232,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - Collections.emptyList() + Collections.emptyList(), + null ); contextWithoutScroll.from(300); contextWithoutScroll.close(); @@ -276,7 +277,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - Collections.emptyList() + Collections.emptyList(), + null ); context1.from(300); exception = expectThrows(IllegalArgumentException.class, () -> context1.preProcess(false)); @@ -348,7 +350,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - Collections.emptyList() + Collections.emptyList(), + null ); SliceBuilder sliceBuilder = mock(SliceBuilder.class); @@ -389,7 +392,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - Collections.emptyList() + Collections.emptyList(), + null ); ParsedQuery parsedQuery = ParsedQuery.parsedMatchAllQuery(); context3.sliceBuilder(null).parsedQuery(parsedQuery).preProcess(false); @@ -426,7 +430,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - Collections.emptyList() + Collections.emptyList(), + null ); context4.sliceBuilder(new SliceBuilder(1, 2)).parsedQuery(parsedQuery).preProcess(false); Query query1 = context4.query(); @@ -458,7 +463,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - Collections.emptyList() + Collections.emptyList(), + null ); int numSlicesForPit = maxSlicesPerPit + randomIntBetween(1, 100); when(sliceBuilder.getMax()).thenReturn(numSlicesForPit); @@ -564,7 +570,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - Collections.emptyList() + Collections.emptyList(), + null ); assertThat(context.searcher().hasCancellations(), is(false)); context.searcher().addQueryCancellation(() -> {}); @@ -678,7 +685,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - Collections.emptyList() + Collections.emptyList(), + null ); // Case1: if sort is on timestamp field, non-concurrent path is used @@ -704,7 +712,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - Collections.emptyList() + Collections.emptyList(), + null ); context.sort( new SortAndFormats(new Sort(new SortField("test2", SortField.Type.INT)), new DocValueFormat[] { DocValueFormat.RAW }) @@ -732,7 +741,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - Collections.emptyList() + Collections.emptyList(), + null ); context.evaluateRequestShouldUseConcurrentSearch(); if (executor == null) { @@ -765,7 +775,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - Collections.emptyList() + Collections.emptyList(), + null ); context.evaluateRequestShouldUseConcurrentSearch(); assertFalse(context.shouldUseConcurrentSearch()); @@ -794,7 +805,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - Collections.emptyList() + Collections.emptyList(), + null ); context.evaluateRequestShouldUseConcurrentSearch(); assertFalse(context.shouldUseConcurrentSearch()); @@ -911,7 +923,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - Collections.emptyList() + Collections.emptyList(), + null ); // Case1: if there is no agg in the query, non-concurrent path is used @@ -939,7 +952,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - Collections.emptyList() + Collections.emptyList(), + null ); // add un-supported agg operation @@ -971,7 +985,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - Collections.emptyList() + Collections.emptyList(), + null ); // create a supported agg operation context.aggregations(mockAggregations); @@ -1029,7 +1044,8 @@ public Optional create(IndexSettings indexSettin false, executor, null, - concurrentSearchRequestDeciders + concurrentSearchRequestDeciders, + null ); // create a supported agg operation context.aggregations(mockAggregations); @@ -1067,7 +1083,8 @@ public Optional create(IndexSettings indexSettin false, executor, null, - concurrentSearchRequestDeciders + concurrentSearchRequestDeciders, + null ); // create a supported agg operation @@ -1109,7 +1126,8 @@ public Optional create(IndexSettings indexSettin false, executor, null, - concurrentSearchRequestDeciders + concurrentSearchRequestDeciders, + null ); // create a supported agg operation @@ -1152,7 +1170,8 @@ public Optional create(IndexSettings indexSettin false, executor, null, - concurrentSearchRequestDeciders + concurrentSearchRequestDeciders, + null ); // create a supported agg operation diff --git a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java index 20861e146d93d..9520bc491b715 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java @@ -550,6 +550,27 @@ public void testTerminateAfterEarlyTermination() throws Exception { dir.close(); } + public void testArrow() throws Exception { + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + final int numDocs = scaledRandomIntBetween(100, 200); + for (int i = 0; i < numDocs; ++i) { + Document doc = new Document(); + doc.add(new StringField("joinField", Integer.toString(i % 10), Store.NO)); + doc.add(new SortedSetDocValuesField("joinField", new BytesRef(Integer.toString(i % 10)))); + w.addDocument(doc); + } + w.close(); + + final IndexReader reader = DirectoryReader.open(dir); + TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader, executor)); + context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); + context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap())); + QueryPhase.executeInternal(context, queryPhaseSearcher); + assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); + } + public void testIndexSortingEarlyTermination() throws Exception { Directory dir = newDirectory(); final Sort sort = new Sort(new SortField("rank", SortField.Type.INT)); diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 493a3ed431e00..784512150e7f3 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -229,6 +229,7 @@ import org.opensearch.search.fetch.FetchPhase; import org.opensearch.search.pipeline.SearchPipelineService; import org.opensearch.search.query.QueryPhase; +import org.opensearch.search.query.StreamSearchPhase; import org.opensearch.snapshots.mockstore.MockEventuallyConsistentRepository; import org.opensearch.tasks.TaskResourceTrackingService; import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry; @@ -2333,11 +2334,13 @@ public void onFailure(final Exception e) { bigArrays, new QueryPhase(), new FetchPhase(Collections.emptyList()), + new StreamSearchPhase(), responseCollectorService, new NoneCircuitBreakerService(), null, new TaskResourceTrackingService(settings, clusterSettings, threadPool), - Collections.emptyList() + Collections.emptyList(), + null ); SearchPhaseController searchPhaseController = new SearchPhaseController( writableRegistry(), diff --git a/test/framework/build.gradle b/test/framework/build.gradle index 84a536fdf62c8..d44816490337f 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -39,6 +39,8 @@ dependencies { api project(":server") api project(":libs:opensearch-cli") api project(":test:telemetry") + api project(':libs:opensearch-arrow-spi') + api "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" api "junit:junit:${versions.junit}" api "org.hamcrest:hamcrest:${versions.hamcrest}" diff --git a/test/framework/src/main/java/org/opensearch/node/MockNode.java b/test/framework/src/main/java/org/opensearch/node/MockNode.java index 97c06962ca2e7..cd9b5bd8c3126 100644 --- a/test/framework/src/main/java/org/opensearch/node/MockNode.java +++ b/test/framework/src/main/java/org/opensearch/node/MockNode.java @@ -32,6 +32,7 @@ package org.opensearch.node; +import org.opensearch.arrow.spi.StreamManager; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.ClusterInfoService; import org.opensearch.cluster.MockInternalClusterInfoService; @@ -60,6 +61,7 @@ import org.opensearch.search.deciders.ConcurrentSearchRequestDecider; import org.opensearch.search.fetch.FetchPhase; import org.opensearch.search.query.QueryPhase; +import org.opensearch.search.query.StreamSearchPhase; import org.opensearch.tasks.TaskResourceTrackingService; import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.test.MockHttpTransport; @@ -154,11 +156,13 @@ protected SearchService newSearchService( BigArrays bigArrays, QueryPhase queryPhase, FetchPhase fetchPhase, + StreamSearchPhase streamSearchPhase, ResponseCollectorService responseCollectorService, CircuitBreakerService circuitBreakerService, Executor indexSearcherExecutor, TaskResourceTrackingService taskResourceTrackingService, - Collection concurrentSearchDeciderFactories + Collection concurrentSearchDeciderFactories, + StreamManager streamManager ) { if (getPluginsService().filterPlugins(MockSearchService.TestPlugin.class).isEmpty()) { return super.newSearchService( @@ -169,11 +173,13 @@ protected SearchService newSearchService( bigArrays, queryPhase, fetchPhase, + null, responseCollectorService, circuitBreakerService, indexSearcherExecutor, taskResourceTrackingService, - concurrentSearchDeciderFactories + concurrentSearchDeciderFactories, + streamManager ); } return new MockSearchService( @@ -186,7 +192,8 @@ protected SearchService newSearchService( fetchPhase, circuitBreakerService, indexSearcherExecutor, - taskResourceTrackingService + taskResourceTrackingService, + streamManager ); } diff --git a/test/framework/src/main/java/org/opensearch/search/MockSearchService.java b/test/framework/src/main/java/org/opensearch/search/MockSearchService.java index 28e202e783c4e..e8df31ad04f45 100644 --- a/test/framework/src/main/java/org/opensearch/search/MockSearchService.java +++ b/test/framework/src/main/java/org/opensearch/search/MockSearchService.java @@ -32,6 +32,7 @@ package org.opensearch.search; +import org.opensearch.arrow.spi.StreamManager; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.util.BigArrays; import org.opensearch.core.indices.breaker.CircuitBreakerService; @@ -42,6 +43,7 @@ import org.opensearch.search.fetch.FetchPhase; import org.opensearch.search.internal.ReaderContext; import org.opensearch.search.query.QueryPhase; +import org.opensearch.search.query.StreamSearchPhase; import org.opensearch.tasks.TaskResourceTrackingService; import org.opensearch.threadpool.ThreadPool; @@ -99,7 +101,8 @@ public MockSearchService( FetchPhase fetchPhase, CircuitBreakerService circuitBreakerService, Executor indexSearcherExecutor, - TaskResourceTrackingService taskResourceTrackingService + TaskResourceTrackingService taskResourceTrackingService, + StreamManager streamManager ) { super( clusterService, @@ -109,11 +112,13 @@ public MockSearchService( bigArrays, queryPhase, fetchPhase, + new StreamSearchPhase(), null, circuitBreakerService, indexSearcherExecutor, taskResourceTrackingService, - Collections.emptyList() + Collections.emptyList(), + streamManager ); } diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index 7b2c653e9bdb2..517e58523fb9e 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -1723,7 +1723,7 @@ public T getInstance(Class clazz) { return getInstance(clazz, nc -> true); } - private static T getInstanceFromNode(Class clazz, Node node) { + static T getInstanceFromNode(Class clazz, Node node) { return node.injector().getInstance(clazz); } diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index b180187303a60..6f2c2862fa88e 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -1768,7 +1768,7 @@ public static String getPortRange() { return getBasePort() + "-" + (getBasePort() + 99); // upper bound is inclusive } - protected static int getBasePort() { + protected static int getBasePort(int start) { // some tests use MockTransportService to do network based testing. Yet, we run tests in multiple JVMs that means // concurrent tests could claim port that another JVM just released and if that test tries to simulate a disconnect it might // be smart enough to re-connect depending on what is tested. To reduce the risk, since this is very hard to debug we use @@ -1792,7 +1792,11 @@ protected static int getBasePort() { startAt = (int) Math.floorMod(workerId - 1, 223L) + 1; } assert startAt >= 0 : "Unexpected test worker Id, resulting port range would be negative"; - return 10300 + (startAt * 100); + return start + (startAt * 100); + } + + protected static int getBasePort() { + return getBasePort(10300); } protected static InetAddress randomIp(boolean v4) { diff --git a/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java b/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java index eec890e289468..74e000a761a1f 100644 --- a/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java @@ -38,6 +38,7 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.action.search.SearchShardTask; import org.opensearch.action.search.SearchType; +import org.opensearch.arrow.spi.StreamManager; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.core.index.shard.ShardId; @@ -76,6 +77,7 @@ import org.opensearch.search.query.ReduceableSearchResult; import org.opensearch.search.rescore.RescoreContext; import org.opensearch.search.sort.SortAndFormats; +import org.opensearch.search.stream.StreamSearchResult; import org.opensearch.search.suggest.SuggestionSearchContext; import java.util.Collections; @@ -123,6 +125,8 @@ public class TestSearchContext extends SearchContext { private BucketCollectorProcessor bucketCollectorProcessor = NO_OP_BUCKET_COLLECTOR_PROCESSOR; private int maxSliceCount; + private StreamManager streamManager; + /** * Sets the concurrent segment search enabled field */ @@ -165,6 +169,16 @@ public TestSearchContext( IndexShard indexShard, ContextIndexSearcher searcher, ScrollContext scrollContext + ) { + this(queryShardContext, indexShard, searcher, scrollContext, null); + } + + public TestSearchContext( + QueryShardContext queryShardContext, + IndexShard indexShard, + ContextIndexSearcher searcher, + ScrollContext scrollContext, + StreamManager streamManager ) { this.bigArrays = null; this.indexService = null; @@ -175,6 +189,7 @@ public TestSearchContext( this.concurrentSegmentSearchEnabled = searcher != null; /* executor is always set */ this.maxSliceCount = randomIntBetween(0, 2); this.scrollContext = scrollContext; + this.streamManager = streamManager; } public void setSearcher(ContextIndexSearcher searcher) { @@ -344,6 +359,11 @@ public SimilarityService similarityService() { return null; } + @Override + public StreamManager streamManager() { + return streamManager; + } + @Override public BigArrays bigArrays() { return bigArrays; @@ -601,6 +621,11 @@ public QuerySearchResult queryResult() { return queryResult; } + @Override + public StreamSearchResult streamSearchResult() { + return null; + } + @Override public FetchSearchResult fetchResult() { return null;