From aaf8c877ebdaa722054b66c3a36494278024fe1b Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 10 Sep 2024 17:08:59 +0800 Subject: [PATCH 1/4] init --- dev/deps/spark-deps-hadoop-3-hive-2.3 | 10 +++++----- pom.xml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 78da866076a5b..5bf1cb5c8a25f 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -184,11 +184,11 @@ lapack/3.0.3//lapack-3.0.3.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar libfb303/0.9.3//libfb303-0.9.3.jar libthrift/0.16.0//libthrift-0.16.0.jar -log4j-1.2-api/2.22.1//log4j-1.2-api-2.22.1.jar -log4j-api/2.22.1//log4j-api-2.22.1.jar -log4j-core/2.22.1//log4j-core-2.22.1.jar -log4j-layout-template-json/2.22.1//log4j-layout-template-json-2.22.1.jar -log4j-slf4j2-impl/2.22.1//log4j-slf4j2-impl-2.22.1.jar +log4j-1.2-api/2.24.0//log4j-1.2-api-2.24.0.jar +log4j-api/2.24.0//log4j-api-2.24.0.jar +log4j-core/2.24.0//log4j-core-2.24.0.jar +log4j-layout-template-json/2.24.0//log4j-layout-template-json-2.24.0.jar +log4j-slf4j2-impl/2.24.0//log4j-slf4j2-impl-2.24.0.jar logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar lz4-java/1.8.0//lz4-java-1.8.0.jar metrics-core/4.2.27//metrics-core-4.2.27.jar diff --git a/pom.xml b/pom.xml index 9780204a121bc..3d880522ed078 100644 --- a/pom.xml +++ b/pom.xml @@ -120,7 +120,7 @@ spark 9.7 2.0.16 - 2.22.1 + 2.24.0 3.4.0 From b6b26cfa3d4864ac50fe6a9a186ef802fd84791e Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 10 Sep 2024 21:57:42 +0800 Subject: [PATCH 2/4] workaround --- common/utils/src/test/resources/log4j2.component.properties | 1 + .../scala/org/apache/spark/util/StructuredLoggingSuite.scala | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) create mode 100644 common/utils/src/test/resources/log4j2.component.properties diff --git a/common/utils/src/test/resources/log4j2.component.properties b/common/utils/src/test/resources/log4j2.component.properties new file mode 100644 index 0000000000000..694ab80ad993a --- /dev/null +++ b/common/utils/src/test/resources/log4j2.component.properties @@ -0,0 +1 @@ +log4j2.garbagefree.threadContextMap = true \ No newline at end of file diff --git a/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala b/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala index 48951c2084f17..74db2128a9682 100644 --- a/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala +++ b/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala @@ -471,10 +471,6 @@ class StructuredLoggingSuite extends LoggingSuiteBase { org.slf4j.MDC.put(LogKeys.DATA.name, "some-data") val logOutputWithMDCSet = captureLogOutput(() => logInfo(msgWithMDC)) assert(mdcPattern.r.findFirstIn(logOutputWithMDCSet).isDefined) - - org.slf4j.MDC.remove(LogKeys.DATA.name) - val logOutputWithoutMDCSet = captureLogOutput(() => logInfo(msgWithMDC)) - assert(mdcPattern.r.findFirstIn(logOutputWithoutMDCSet).isEmpty) } } From 312a3a6a8c3df781efa5891180da59e9f7941c4f Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 10 Sep 2024 22:14:51 +0800 Subject: [PATCH 3/4] fix format --- common/utils/src/test/resources/log4j2.component.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/utils/src/test/resources/log4j2.component.properties b/common/utils/src/test/resources/log4j2.component.properties index 694ab80ad993a..5c622ab7e3c9c 100644 --- a/common/utils/src/test/resources/log4j2.component.properties +++ b/common/utils/src/test/resources/log4j2.component.properties @@ -1 +1 @@ -log4j2.garbagefree.threadContextMap = true \ No newline at end of file +log4j2.garbagefree.threadContextMap = true From a7280c0a82bca33d660b0eac0f3344ab5e5f5cbd Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 10 Sep 2024 22:15:54 +0800 Subject: [PATCH 4/4] revert --- .../scala/org/apache/spark/util/StructuredLoggingSuite.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala b/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala index 74db2128a9682..48951c2084f17 100644 --- a/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala +++ b/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala @@ -471,6 +471,10 @@ class StructuredLoggingSuite extends LoggingSuiteBase { org.slf4j.MDC.put(LogKeys.DATA.name, "some-data") val logOutputWithMDCSet = captureLogOutput(() => logInfo(msgWithMDC)) assert(mdcPattern.r.findFirstIn(logOutputWithMDCSet).isDefined) + + org.slf4j.MDC.remove(LogKeys.DATA.name) + val logOutputWithoutMDCSet = captureLogOutput(() => logInfo(msgWithMDC)) + assert(mdcPattern.r.findFirstIn(logOutputWithoutMDCSet).isEmpty) } }