KAFKA-18773: Migrate the log4j1 config to log4j 2 for native image and README

build.gradle

@@ -1072,7 +1072,6 @@ project(':core') {
     implementation libs.argparse4j
     implementation libs.commonsValidator
     implementation libs.jacksonDatabind
-    implementation libs.jacksonModuleScala
     implementation libs.jacksonDataformatCsv
     implementation libs.jacksonJDK8Datatypes
     implementation libs.jacksonDatabindYaml

core/src/main/scala/kafka/docker/KafkaDockerWrapper.scala

@@ -110,7 +110,7 @@ object KafkaDockerWrapper extends Logging {
       case Some(str) => str
       case None => throw new RuntimeException("CLUSTER_ID environment variable is not set.")
     }
-    Array("format", "--cluster-id=" + clusterId, "-c", s"${configsPath.toString}/server.properties")
+    Array("format", "--cluster-id=" + clusterId, "-c", s"${configsPath.toString}/server.properties", "--standalone")
   }
 
   private def prepareConfigs(defaultConfigsPath: Path, mountedConfigsPath: Path, finalConfigsPath: Path): Unit = {
@@ -250,10 +250,6 @@ object KafkaDockerWrapper extends Logging {
     }
     val config = yaml.getConfiguration
 
-    if (config == null && loggerFromEnv.isEmpty && rootOption.isEmpty) {
-      return
-    }
-
     if (config == null) {
       generateDefaultLog4jConfig(loggerFromEnv, rootOption, filepath, mapper)
     } else {

core/src/test/scala/unit/kafka/docker/KafkaDockerWrapperTest.scala

@@ -133,7 +133,7 @@ class KafkaDockerWrapperTest {
     val configsPath = Paths.get("/path/to/configs")
     val envVars = Map("CLUSTER_ID" -> "MYwKGPhXQZidgd0qMv8Mkw")
 
-    val expected = Array("format", "--cluster-id=MYwKGPhXQZidgd0qMv8Mkw", "-c", "/path/to/configs/server.properties")
+    val expected = Array("format", "--cluster-id=MYwKGPhXQZidgd0qMv8Mkw", "-c", "/path/to/configs/server.properties", "--standalone")
     val actual = KafkaDockerWrapper.formatStorageCmd(configsPath, envVars)
 
     assertArrayEquals(expected.toArray[Object], actual.toArray[Object])

docker/examples/README.md

@@ -48,11 +48,11 @@ It is also possible to use the input file to have a common set of configurations
 
 - To provide configs to log4j property files, following points should be considered:
   - log4j properties provided via environment variables will be appended to the default properties file (log4j properties files bundled with Kafka).
-  - `KAFKA_LOG4J_ROOT_LOGLEVEL` can be provided to set the value of `log4j.rootLogger` in log4j.properties and `tools-log4j.properties`.
-  - log4j loggers can be added to log4j.properties by setting them in `KAFKA_LOG4J_LOGGERS` environment variable in a single comma separated string.
+  - `KAFKA_LOG4J_ROOT_LOGLEVEL` can be provided to set the value of `log4j.rootLogger` in `log4j2.yaml` and `tools-log4j2.yaml`.
+  - log4j loggers can be added to `log4j2.yaml` by setting them in `KAFKA_LOG4J_LOGGERS` environment variable in a single comma separated string.
       - Example:
           - Assuming that `KAFKA_LOG4J_LOGGERS='property1=value1,property2=value2'` environment variable is provided to Docker container.
-          - `log4j.logger.property1=value1` and `log4j.logger.property2=value2` will be added to the `log4j.properties` file inside Docker container.
+          - `log4j.logger.property1=value1` and `log4j.logger.property2=value2` will be added to the `log4j2.yaml` file inside Docker container.
 
 Running in SSL mode
 -------------------

docker/native/README.md

@@ -15,6 +15,11 @@ These metadata configs are provided in the `native-image-configs` directory.
 GraalVM provide an option to create these configs automatically by running the application normally with the [native-image agent](https://www.graalvm.org/latest/reference-manual/native-image/metadata/AutomaticMetadataCollection/) attached.
 The configs in the `native-image-configs` directory are generated by running existing Apache Kafka System Tests(using GraalVM JIT and attaching the native-image agent to the process) as they are quite exhaustive.
 
+To update reachability metadata, users can set following environment variable and run related jar files. New metadata configs will be generated in the `native-image-configs` directory.
+```bash
+export JAVA_TOOL_OPTIONS="-agentlib:native-image-agent=config-merge-dir=/path/to/kafka/docker/native/native-image-configs"
+```
+
 ## Limitations of Native Apache Kafka executable
 - Dynamic Features: For any new dynamic feature added or modified in Apache Kafka, the corresponding metadata configurations must be added or updated in the `native-image-configs`. As of now these configs are static.
 - Runtime Jars: Native Kafka Executable does not support capabilities that require a new runtime jar provided by the user, as the jar's information is not available during build time. In such cases, a new native Kafka binary must be built, including that jar in the classpath during the build.

docker/native/launch

@@ -41,12 +41,11 @@ fi
 result=$(/opt/kafka/kafka.Kafka setup \
       --default-configs-dir /etc/kafka/docker \
       --mounted-configs-dir /mnt/shared/config \
-      --final-configs-dir /opt/kafka/config \
-      -Dlog4j.configuration=file:/opt/kafka/config/tools-log4j.properties 2>&1) || \
+      --final-configs-dir /opt/kafka/config 2>&1) || \
       echo $result | grep -i "already formatted" || \
       { echo $result && (exit 1) }
 
 echo "WARNING: THIS IS AN EXPERIMENTAL DOCKER IMAGE RECOMMENDED FOR LOCAL TESTING AND DEVELOPMENT PURPOSES."
 
-KAFKA_LOG4J_CMD_OPTS="-Dkafka.logs.dir=/opt/kafka/logs/ -Dlog4j.configuration=file:/opt/kafka/config/log4j.properties"
+KAFKA_LOG4J_CMD_OPTS="-Dkafka.logs.dir=/opt/kafka/logs/ -Dlog4j2.configurationFile=file:/opt/kafka/config/log4j2.yaml"
 exec /opt/kafka/kafka.Kafka start --config /opt/kafka/config/server.properties $KAFKA_LOG4J_CMD_OPTS $KAFKA_JMX_OPTS ${KAFKA_OPTS-}

docker/native/native-image-configs/jni-config.json

@@ -24,12 +24,32 @@
 {
   "name":"java.lang.OutOfMemoryError"
 },
+{
+  "name":"java.lang.String",
+  "methods":[{"name":"lastIndexOf","parameterTypes":["int"] }, {"name":"substring","parameterTypes":["int"] }]
+},
+{
+  "name":"java.lang.System",
+  "methods":[{"name":"getProperty","parameterTypes":["java.lang.String"] }, {"name":"setProperty","parameterTypes":["java.lang.String","java.lang.String"] }]
+},
+{
+  "name":"java.lang.VersionProps",
+  "methods":[{"name":"print","parameterTypes":["boolean"] }]
+},
 {
   "name":"java.util.Arrays",
   "methods":[{"name":"asList","parameterTypes":["java.lang.Object[]"] }]
 },
+{
+  "name":"kafka.Kafka",
+  "methods":[{"name":"main","parameterTypes":["java.lang.String[]"] }]
+},
+{
+  "name":"kafka.tools.StorageTool",
+  "methods":[{"name":"main","parameterTypes":["java.lang.String[]"] }]
+},
 {
   "name":"sun.management.VMManagementImpl",
   "fields":[{"name":"compTimeMonitoringSupport"}, {"name":"currentThreadCpuTimeSupport"}, {"name":"objectMonitorUsageSupport"}, {"name":"otherThreadCpuTimeSupport"}, {"name":"remoteDiagnosticCommandsSupport"}, {"name":"synchronizerUsageSupport"}, {"name":"threadAllocatedMemorySupport"}, {"name":"threadContentionMonitoringSupport"}]
 }
-]
+]

docker/native/native-image-configs/predefined-classes-config.json

@@ -5,3 +5,4 @@
     ]
   }
 ]
+

docker/native/native-image-configs/proxy-config.json

@@ -2,4 +2,4 @@
   {
     "interfaces":["sun.misc.SignalHandler"]
   }
-]
+]