From 5f49b29a88d8f3ddd8a9aba6e389180a6358d867 Mon Sep 17 00:00:00 2001 From: Jelena Furundzic Date: Thu, 13 Feb 2025 10:44:29 -0800 Subject: [PATCH 1/6] Parse out sensitive encryption material before logging json --- .../client/jdbc/SnowflakeFileTransferAgent.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java index a2eb191d9..229522f1f 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Strings; import com.google.common.io.ByteStreams; import com.google.common.io.CountingOutputStream; @@ -1338,12 +1339,20 @@ private static JsonNode parseCommandInGS(SFStatement statement, String command) } JsonNode jsonNode = (JsonNode) result; - logger.debug("Response: {}", jsonNode.toString()); + logger.debug("Response: {}", removeSensitiveJsonElementsForLogging(jsonNode)); SnowflakeUtil.checkErrorAndThrowException(jsonNode); return jsonNode; } + /** + * @param jsonNode A JsonNode that needs to have sensitive data removed before logging. + * @return A string value of the JSON without any sensitive data for logging. + */ + private static String removeSensitiveJsonElementsForLogging(JsonNode jsonNode) { + return ((ObjectNode) jsonNode.path("data")).remove("encryptionMaterial").toString(); + } + /** * @param rootNode JSON doc returned by GS * @throws SnowflakeSQLException Will be thrown if we fail to parse the stage credentials From 9690ef07c75e76c0b77aae27ef5d34c2166eb50b Mon Sep 17 00:00:00 2001 From: Jelena Furundzic Date: Thu, 13 Feb 2025 15:48:25 -0800 Subject: [PATCH 2/6] Fix test failure --- .../snowflake/client/jdbc/SnowflakeFileTransferAgent.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java index 229522f1f..6ba77d6e3 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java @@ -1339,6 +1339,7 @@ private static JsonNode parseCommandInGS(SFStatement statement, String command) } JsonNode jsonNode = (JsonNode) result; + logger.debug("Response: {}", removeSensitiveJsonElementsForLogging(jsonNode)); SnowflakeUtil.checkErrorAndThrowException(jsonNode); @@ -1349,8 +1350,10 @@ private static JsonNode parseCommandInGS(SFStatement statement, String command) * @param jsonNode A JsonNode that needs to have sensitive data removed before logging. * @return A string value of the JSON without any sensitive data for logging. */ - private static String removeSensitiveJsonElementsForLogging(JsonNode jsonNode) { - return ((ObjectNode) jsonNode.path("data")).remove("encryptionMaterial").toString(); + private static String removeSensitiveJsonElementsForLogging(JsonNode jsonNode) + throws SnowflakeSQLException { + JsonNode result = jsonNode.deepCopy(); + return ((ObjectNode) result.path("data")).remove("encryptionMaterial").toString(); } /** From 7f1ced412f6255420579c247b0677a49be2608f0 Mon Sep 17 00:00:00 2001 From: Jelena Furundzic Date: Wed, 26 Feb 2025 15:27:43 -0800 Subject: [PATCH 3/6] Changed implementation to use SecretDetector, added test --- .../jdbc/SnowflakeFileTransferAgent.java | 14 ++-------- .../snowflake/client/util/SecretDetector.java | 20 ++++++++++++++ .../client/util/SecretDetectorTest.java | 26 +++++++++++++++++++ 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java index 6ba77d6e3..42f91cb52 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java @@ -12,7 +12,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Strings; import com.google.common.io.ByteStreams; import com.google.common.io.CountingOutputStream; @@ -72,6 +71,7 @@ import net.snowflake.client.log.ArgSupplier; import net.snowflake.client.log.SFLogger; import net.snowflake.client.log.SFLoggerFactory; +import net.snowflake.client.util.SecretDetector; import net.snowflake.common.core.FileCompressionType; import net.snowflake.common.core.RemoteStoreFileEncryptionMaterial; import net.snowflake.common.core.SqlState; @@ -1340,22 +1340,12 @@ private static JsonNode parseCommandInGS(SFStatement statement, String command) JsonNode jsonNode = (JsonNode) result; - logger.debug("Response: {}", removeSensitiveJsonElementsForLogging(jsonNode)); + logger.debug("Response: {}", SecretDetector.filterEncryptionMaterial(jsonNode.toString())); SnowflakeUtil.checkErrorAndThrowException(jsonNode); return jsonNode; } - /** - * @param jsonNode A JsonNode that needs to have sensitive data removed before logging. - * @return A string value of the JSON without any sensitive data for logging. - */ - private static String removeSensitiveJsonElementsForLogging(JsonNode jsonNode) - throws SnowflakeSQLException { - JsonNode result = jsonNode.deepCopy(); - return ((ObjectNode) result.path("data")).remove("encryptionMaterial").toString(); - } - /** * @param rootNode JSON doc returned by GS * @throws SnowflakeSQLException Will be thrown if we fail to parse the stage credentials diff --git a/src/main/java/net/snowflake/client/util/SecretDetector.java b/src/main/java/net/snowflake/client/util/SecretDetector.java index 3c0727de7..2d2acd504 100644 --- a/src/main/java/net/snowflake/client/util/SecretDetector.java +++ b/src/main/java/net/snowflake/client/util/SecretDetector.java @@ -70,6 +70,9 @@ public class SecretDetector { "(token|assertion content)" + "(['\"\\s:=]+)" + "([a-z0-9=/_\\-+]{8,})", Pattern.CASE_INSENSITIVE); + private static final Pattern ENCRYPTION_MATERIAL_PATTERN = + Pattern.compile("\"encryptionMaterial\"\\s*:\\s*\\{.*?\\}", Pattern.CASE_INSENSITIVE); + // only attempt to find secrets in its leading 100Kb SNOW-30961 private static final int MAX_LENGTH = 100 * 1000; @@ -251,6 +254,23 @@ public static String filterAccessTokens(String message) { return message; } + /** + * Filter encryption material that may be buried inside a JSON string. + * + * @param message the message text which may contain encryption material + * @return Return filtered message + */ + public static String filterEncryptionMaterial(String message) { + Matcher matcher = + ENCRYPTION_MATERIAL_PATTERN.matcher( + message.length() <= MAX_LENGTH ? message : message.substring(0, MAX_LENGTH)); + + if (matcher.find()) { + return matcher.replaceAll("\"encryptionMaterial\" : ****"); + } + return message; + } + public static JSONObject maskJsonObject(JSONObject json) { for (Map.Entry entry : json.entrySet()) { if (entry.getValue() instanceof String) { diff --git a/src/test/java/net/snowflake/client/util/SecretDetectorTest.java b/src/test/java/net/snowflake/client/util/SecretDetectorTest.java index 1b936b929..870aea4fe 100644 --- a/src/test/java/net/snowflake/client/util/SecretDetectorTest.java +++ b/src/test/java/net/snowflake/client/util/SecretDetectorTest.java @@ -388,4 +388,30 @@ public void testMaskJacksonObject() { "Nested Jackson array node is not masked successfully", maskedNestedArrayStr.equals(SecretDetector.maskJacksonNode(objNode4).toString())); } + + /* + + */ + @Test + public void testEncryptionMaterialFilter() throws Exception { + String messageText = + "{\"data\":" + + "{\"autoCompress\":true," + + "\"overwrite\":false," + + "\"clientShowEncryptionParameter\":true," + + "\"encryptionMaterial\":{\"queryStageMasterKey\":\"asdfasdfasdfasdf==\",\"queryId\":\"01b6f5ba-0002-0181-0000-11111111da\",\"smkId\":1111}," + + "\"stageInfo\":{\"locationType\":\"AZURE\", \"region\":\"eastus2\"}"; + + String filteredMessageText = + "{\"data\":" + + "{\"autoCompress\":true," + + "\"overwrite\":false," + + "\"clientShowEncryptionParameter\":true," + + "\"encryptionMaterial\" : ****," + + "\"stageInfo\":{\"locationType\":\"AZURE\", \"region\":\"eastus2\"}"; + + String result = SecretDetector.filterEncryptionMaterial(messageText); + + assertEquals(filteredMessageText, result); + } } From 6868793603d11d06d6e76477ac5bdb485a4cc610 Mon Sep 17 00:00:00 2001 From: Jelena Furundzic Date: Wed, 26 Feb 2025 15:30:19 -0800 Subject: [PATCH 4/6] Fix typo in test --- .../java/net/snowflake/client/util/SecretDetectorTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test/java/net/snowflake/client/util/SecretDetectorTest.java b/src/test/java/net/snowflake/client/util/SecretDetectorTest.java index 870aea4fe..5ee9afbd8 100644 --- a/src/test/java/net/snowflake/client/util/SecretDetectorTest.java +++ b/src/test/java/net/snowflake/client/util/SecretDetectorTest.java @@ -389,9 +389,6 @@ public void testMaskJacksonObject() { maskedNestedArrayStr.equals(SecretDetector.maskJacksonNode(objNode4).toString())); } - /* - - */ @Test public void testEncryptionMaterialFilter() throws Exception { String messageText = From 57a3e6938f493719e9a1b3340189cc5bf44f9c3c Mon Sep 17 00:00:00 2001 From: Jelena Furundzic Date: Mon, 3 Mar 2025 09:04:24 -0800 Subject: [PATCH 5/6] Moving change to maskSecrets --- .../net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java | 2 +- src/main/java/net/snowflake/client/util/SecretDetector.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java index 42f91cb52..76902f8d3 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java @@ -1340,7 +1340,7 @@ private static JsonNode parseCommandInGS(SFStatement statement, String command) JsonNode jsonNode = (JsonNode) result; - logger.debug("Response: {}", SecretDetector.filterEncryptionMaterial(jsonNode.toString())); + logger.debug("Response: {}", SecretDetector.maskSecrets(jsonNode.toString())); SnowflakeUtil.checkErrorAndThrowException(jsonNode); return jsonNode; diff --git a/src/main/java/net/snowflake/client/util/SecretDetector.java b/src/main/java/net/snowflake/client/util/SecretDetector.java index 2d2acd504..a0cb5b5db 100644 --- a/src/main/java/net/snowflake/client/util/SecretDetector.java +++ b/src/main/java/net/snowflake/client/util/SecretDetector.java @@ -211,7 +211,8 @@ public static String maskSASToken(String text) { */ public static String maskSecrets(String text) { return filterAccessTokens( - filterConnectionTokens(filterPassword(filterSASTokens(filterAWSKeys(text))))); + filterConnectionTokens( + filterPassword(filterSASTokens(filterAWSKeys(filterEncryptionMaterial(text)))))); } /** From 65412a5206e6f24cc0f5c0988ff5fab1f4f0ddf7 Mon Sep 17 00:00:00 2001 From: Jelena Furundzic Date: Mon, 3 Mar 2025 09:29:58 -0800 Subject: [PATCH 6/6] fix checkstyle --- src/main/java/net/snowflake/client/util/SecretDetector.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/snowflake/client/util/SecretDetector.java b/src/main/java/net/snowflake/client/util/SecretDetector.java index 880817f91..e2ea5e8dc 100644 --- a/src/main/java/net/snowflake/client/util/SecretDetector.java +++ b/src/main/java/net/snowflake/client/util/SecretDetector.java @@ -225,7 +225,9 @@ public static String maskSASToken(String text) { public static String maskSecrets(String text) { return filterAccessTokens( filterConnectionTokens( - filterPassword(filterSASTokens(filterAWSKeys(filterOAuthTokens(filterEncryptionMaterial(text))))))); + filterPassword( + filterSASTokens( + filterAWSKeys(filterOAuthTokens(filterEncryptionMaterial(text))))))); } /**