GH-38915: [Java] Upgrade Arrow Java project to JPMS Java Platform Module System

java/algorithm/pom.xml

@@ -31,6 +31,7 @@
       <artifactId>arrow-vector</artifactId>
       <version>${project.version}</version>
       <type>test-jar</type>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.arrow</groupId>

java/c/pom.xml

@@ -53,6 +53,11 @@
             <artifactId>arrow-memory-unsafe</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-format</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>

java/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java

@@ -32,7 +32,6 @@
 import org.apache.arrow.dataset.file.DatasetFileWriter;
 import org.apache.arrow.dataset.file.FileFormat;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.util.ArrowTestDataUtil;
 import org.apache.arrow.vector.BigIntVector;
 import org.apache.arrow.vector.BitVector;
 import org.apache.arrow.vector.DateMilliVector;
@@ -76,6 +75,7 @@
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.arrow.vector.util.ArrowTestDataUtil;
 import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel;
 import org.apache.arrow.vector.util.Text;
 import org.junit.ClassRule;

java/flight/flight-core/pom.xml

@@ -54,10 +54,6 @@
       <groupId>io.grpc</groupId>
       <artifactId>grpc-core</artifactId>
     </dependency>
-    <dependency>
-      <groupId>io.grpc</groupId>
-      <artifactId>grpc-context</artifactId>
-    </dependency>
     <dependency>
       <groupId>io.grpc</groupId>
       <artifactId>grpc-protobuf</artifactId>
@@ -149,7 +145,13 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.1.1</version>
+        <!--
+          Downgrade maven-shade-plugin specifically for this module.
+          Using a newer version up to at least 3.5.1 will cause
+          issues in the arrow-tools tests looking up FlatBuffer
+          dependencies.
+         -->
+        <version>3.2.4</version>
         <executions>
           <execution>
             <id>shade-main</id>

java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java

@@ -24,7 +24,7 @@
 import java.util.List;
 import java.util.Random;
 
-import org.apache.arrow.util.ArrowTestDataUtil;
+import org.apache.arrow.vector.util.ArrowTestDataUtil;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.function.Executable;
 

java/flight/flight-grpc/pom.xml

@@ -48,12 +48,13 @@
     </dependency>
     <dependency>
       <groupId>io.grpc</groupId>
-      <artifactId>grpc-core</artifactId>
+      <artifactId>grpc-stub</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-inprocess</artifactId>
+      <scope>test</scope>
     </dependency>
-     <dependency>
-       <groupId>io.grpc</groupId>
-       <artifactId>grpc-stub</artifactId>
-     </dependency>
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-memory-core</artifactId>

java/flight/flight-sql-jdbc-driver/pom.xml

@@ -159,7 +159,6 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.4.1</version>
                 <executions>
                     <execution>
                         <phase>package</phase>

java/flight/flight-sql/pom.xml

@@ -53,6 +53,7 @@
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-jdbc</artifactId>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>
@@ -70,6 +71,7 @@
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.derby</groupId>

java/format/src/main/java/module-info.java

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module org.apache.arrow.format {
+  exports org.apache.arrow.flatbuf;
+  requires transitive flatbuffers.java;
+}

java/maven/module-info-compiler-maven-plugin/pom.xml

@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.arrow.maven.plugins</groupId>
+    <artifactId>arrow-maven-plugins</artifactId>
+    <version>15.0.0-SNAPSHOT</version>
+  </parent>
+  <artifactId>module-info-compiler-maven-plugin</artifactId>
+  <packaging>maven-plugin</packaging>
+
+  <name>Module Info Compiler Maven Plugin</name>
+
+  <url>https://arrow.apache.org</url>
+
+  <prerequisites>
+    <maven>${maven.version}</maven>
+  </prerequisites>
+
+  <properties>
+    <maven.version>3.3.9</maven.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.glavo</groupId>
+      <artifactId>module-info-compiler</artifactId>
+      <version>2.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.maven</groupId>
+      <artifactId>maven-plugin-api</artifactId>
+      <version>${maven.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.maven</groupId>
+      <artifactId>maven-core</artifactId>
+      <version>${maven.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.maven</groupId>
+      <artifactId>maven-artifact</artifactId>
+      <version>${maven.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.maven</groupId>
+      <artifactId>maven-model</artifactId>
+      <version>${maven.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.maven.plugin-tools</groupId>
+      <artifactId>maven-plugin-annotations</artifactId>
+      <version>3.6.0</version>
+      <scope>provided</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
+      <plugins>
+        <plugin>
+          <artifactId>maven-clean-plugin</artifactId>
+          <version>3.1.0</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-plugin-plugin</artifactId>
+          <version>3.6.0</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-jar-plugin</artifactId>
+          <version>3.0.2</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-install-plugin</artifactId>
+          <version>2.5.2</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-deploy-plugin</artifactId>
+          <version>2.8.2</version>
+        </plugin>
+        <plugin>
+          <artifactId>maven-invoker-plugin</artifactId>
+          <version>3.1.0</version>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-plugin-plugin</artifactId>
+        <version>3.6.0</version>
+        <configuration>
+          <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
+        </configuration>
+        <executions>
+          <execution>
+            <id>mojo-descriptor</id>
+            <goals>
+              <goal>descriptor</goal>
+            </goals>
+          </execution>
+          <execution>
+            <id>help-goal</id>
+            <goals>
+              <goal>helpmojo</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+
+</project>

java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.maven.plugins;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.Reader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import org.apache.maven.plugin.AbstractMojo;
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugins.annotations.LifecyclePhase;
+import org.apache.maven.plugins.annotations.Mojo;
+import org.apache.maven.plugins.annotations.Parameter;
+import org.apache.maven.project.MavenProject;
+import org.glavo.mic.ModuleInfoCompiler;
+
+/**
+ * Compiles the first module-info.java file in the project purely syntactically.
+ */
+@Mojo(name = "module-info-compile", defaultPhase = LifecyclePhase.COMPILE)
+public class ModuleInfoCompilerPlugin extends AbstractMojo {
+  /**
+   * Source directories.
+   */
+  @Parameter(defaultValue = "${project.compileSourceRoots}", property = "compileSourceRoots",
+      required = true)
+  private final List<String> compileSourceRoots = new ArrayList<>();
+
+  @Parameter(defaultValue = "false", property = "skip", required = false)
+  private boolean skip = false;
+
+  @Parameter(defaultValue = "${project}", readonly = true, required = true)
+  private MavenProject project;
+
+  @Override
+  public void execute() throws MojoExecutionException {
+    if (skip) {
+      getLog().info("Skipping module-info-compiler-maven-plugin");
+      return;
+    }
+
+    Optional<File> moduleInfoFile = findFirstModuleInfo(compileSourceRoots);
+    if (moduleInfoFile.isPresent()) {
+      // The compiled module-info.class file goes into target/classes/module-info/main
+      Path outputDir = Paths.get(project.getBuild().getOutputDirectory());
+
+      outputDir.toFile().mkdirs();
+      Path targetPath = outputDir.resolve("module-info.class");
+
+      // Invoke the compiler,
+      ModuleInfoCompiler compiler = new ModuleInfoCompiler();
+      try (Reader reader = new InputStreamReader(Files.newInputStream(moduleInfoFile.get().toPath()),
+          StandardCharsets.UTF_8);
+           OutputStream output = Files.newOutputStream(targetPath)) {
+        compiler.compile(reader, output);
+        getLog().info("Successfully wrote module-info.class file.");
+      } catch (IOException ex) {
+        throw new MojoExecutionException("Error compiling module-info.java", ex);
+      }
+    } else {
+      getLog().info("No module-info.java file found. module-info.class file was not generated.");
+    }
+  }
+
+  /**
+   * Finds the first module-info.java file in the set of source directories.
+   */
+  private Optional<File> findFirstModuleInfo(List<String> sourceDirectories) {
+    if (sourceDirectories == null) {
+      return Optional.empty();
+    }
+
+    return sourceDirectories.stream().map(Paths::get)
+        .map(sourcePath ->
+            sourcePath.toFile().listFiles(file ->
+                file.getName().equals("module-info.java")))
+        .filter(matchingFiles -> matchingFiles != null && matchingFiles.length != 0)
+        .map(matchingFiles -> matchingFiles[0])
+        .findAny();
+  }
+}