Skip to content

Commit fcddeee

Browse files
sfc-gh-tzhangsfc-gh-tzhang
committed
Merge branch 'master' into tzhang-si-readme
2 parents 4d5ea58 + bcf6a77 commit fcddeee

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+541
-135
lines changed

‎deploy.sh

+33-1
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,45 @@ cat > $OSSRH_DEPLOY_SETTINGS_XML << SETTINGS.XML
3131
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
3232
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3333
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
34+
<profiles>
35+
<profile>
36+
<id>internal-maven</id>
37+
<repositories>
38+
<repository>
39+
<id>central</id>
40+
<name>Internal Maven Repository</name>
41+
<url>https://artifactory.int.snowflakecomputing.com/artifactory/development-maven-virtual</url>
42+
</repository>
43+
<repository>
44+
<id>deployment</id>
45+
<name>Internal Releases</name>
46+
<url>https://nexus.int.snowflakecomputing.com/repository/Releases/</url>
47+
</repository>
48+
</repositories>
49+
<pluginRepositories>
50+
<pluginRepository>
51+
<id>central</id>
52+
<name>Internal Maven Repository</name>
53+
<url>https://artifactory.int.snowflakecomputing.com/artifactory/development-maven-virtual</url>
54+
</pluginRepository>
55+
<pluginRepository>
56+
<id>deployment</id>
57+
<name>Internal Releases</name>
58+
<url>https://nexus.int.snowflakecomputing.com/repository/Releases/</url>
59+
</pluginRepository>
60+
</pluginRepositories>
61+
</profile>
62+
</profiles>
3463
<servers>
3564
<server>
3665
<id>$MVN_REPOSITORY_ID</id>
3766
<username>$SONATYPE_USER</username>
3867
<password>$SONATYPE_PWD</password>
3968
</server>
4069
</servers>
70+
<activeProfiles>
71+
<activeProfile>internal-maven</activeProfile>
72+
</activeProfiles>
4173
</settings>
4274
SETTINGS.XML
4375

@@ -50,7 +82,7 @@ echo "[Info] Sign package and deploy to staging area"
5082
project_version=$($THIS_DIR/scripts/get_project_info_from_pom.py $THIS_DIR/pom.xml version)
5183
$THIS_DIR/scripts/update_project_version.py public_pom.xml $project_version > generated_public_pom.xml
5284

53-
mvn deploy ${MVN_OPTIONS[@]} -Dossrh-deploy
85+
mvn deploy ${MVN_OPTIONS[@]} -Dossrh-deploy -Dmaven.wagon.http.pool=false -X -e
5486

5587
echo "[INFO] Close and Release"
5688
snowflake_repositories=$(mvn ${MVN_OPTIONS[@]} \

‎e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java

+8-2
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,10 @@ public class IngestTestUtils {
5252
private final Base64.Decoder base64Decoder = Base64.getDecoder();
5353

5454
public IngestTestUtils(String testName)
55-
throws SQLException, IOException, ClassNotFoundException, NoSuchAlgorithmException,
55+
throws SQLException,
56+
IOException,
57+
ClassNotFoundException,
58+
NoSuchAlgorithmException,
5659
InvalidKeySpecException {
5760
testId = String.format("%s_%s", testName, UUID.randomUUID().toString().replace("-", "_"));
5861
connection = getConnection();
@@ -108,7 +111,10 @@ private Properties loadProperties() throws IOException {
108111
}
109112

110113
private Connection getConnection()
111-
throws IOException, ClassNotFoundException, SQLException, NoSuchAlgorithmException,
114+
throws IOException,
115+
ClassNotFoundException,
116+
SQLException,
117+
NoSuchAlgorithmException,
112118
InvalidKeySpecException {
113119
Class.forName("net.snowflake.client.jdbc.SnowflakeDriver");
114120

‎e2e-jar-test/pom.xml

+5-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Copyright (c) 2025 Snowflake Computing Inc. All rights reserved.
4+
-->
25
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
36
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
47
<modelVersion>4.0.0</modelVersion>
@@ -29,13 +32,13 @@
2932
<dependency>
3033
<groupId>net.snowflake</groupId>
3134
<artifactId>snowflake-ingest-sdk</artifactId>
32-
<version>3.0.1</version>
35+
<version>3.1.1</version>
3336
</dependency>
3437

3538
<dependency>
3639
<groupId>net.snowflake</groupId>
3740
<artifactId>snowflake-jdbc-fips</artifactId>
38-
<version>3.20.0</version>
41+
<version>3.22.0</version>
3942
</dependency>
4043
<dependency>
4144
<groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>

‎format.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ set -euo pipefail
44
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
55
cd "$SCRIPT_DIR"
66

7-
GOOGLE_FORMAT_VERSION="1.10.0"
7+
GOOGLE_FORMAT_VERSION="1.20.0"
88
DOWNLOAD_URL="https://github.com/google/google-java-format/releases/download/v${GOOGLE_FORMAT_VERSION}/google-java-format-${GOOGLE_FORMAT_VERSION}-all-deps.jar"
99
JAR_FILE="./.cache/google-java-format-${GOOGLE_FORMAT_VERSION}-all-deps.jar"
1010

‎pom.xml

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<?xml version="1.0" encoding="UTF-8"?>
22
<!--
3-
~ Copyright (c) 2024 Snowflake Computing Inc. All rights reserved.
3+
~ Copyright (c) 2024-2025 Snowflake Computing Inc. All rights reserved.
44
-->
55
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
66
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -9,7 +9,7 @@
99
<!-- Arifact name and version information -->
1010
<groupId>net.snowflake</groupId>
1111
<artifactId>snowflake-ingest-sdk</artifactId>
12-
<version>3.0.1</version>
12+
<version>3.1.1</version>
1313
<packaging>jar</packaging>
1414
<name>Snowflake Ingest SDK</name>
1515
<description>Snowflake Ingest SDK</description>
@@ -62,7 +62,7 @@
6262
<maven.compiler.source>1.8</maven.compiler.source>
6363
<maven.compiler.target>1.8</maven.compiler.target>
6464
<net.minidev.version>2.4.9</net.minidev.version>
65-
<netty.version>4.1.115.Final</netty.version>
65+
<netty.version>4.1.118.Final</netty.version>
6666
<nimbusds.version>9.37.3</nimbusds.version>
6767
<objenesis.version>3.1</objenesis.version>
6868
<parquet.version>1.14.1</parquet.version>
@@ -72,7 +72,7 @@
7272
<shadeBase>net.snowflake.ingest.internal</shadeBase>
7373
<slf4j.version>1.7.36</slf4j.version>
7474
<snappy.version>1.1.10.5</snappy.version>
75-
<snowjdbc.version>3.20.0</snowjdbc.version>
75+
<snowjdbc.version>3.22.0</snowjdbc.version>
7676
<yetus.version>0.13.0</yetus.version>
7777
</properties>
7878

‎public_pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
<dependency>
4040
<groupId>net.snowflake</groupId>
4141
<artifactId>snowflake-jdbc</artifactId>
42-
<version>3.20.0</version>
42+
<version>3.22.0</version>
4343
<scope>compile</scope>
4444
</dependency>
4545
<dependency>

‎src/main/java/net/snowflake/ingest/connection/HistoryResponse.java

+1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ public class HistoryResponse {
2626
public HistoryResponse() {
2727
this.files = new ArrayList<>();
2828
}
29+
2930
/**
3031
* HistoryStats - The statistics reported back by the service about the currently loading set of
3132
* files

‎src/main/java/net/snowflake/ingest/connection/IngestResponseException.java

+1
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ public static class IngestExceptionBody {
7171

7272
// POJO constructor for mapper
7373
public IngestExceptionBody() {}
74+
7475
// When exception JSON does not match, store message as blob
7576
IngestExceptionBody(String blob) {
7677
messageBlob = blob;

‎src/main/java/net/snowflake/ingest/connection/IngestStatus.java

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ public enum IngestStatus {
2020
/** If ON_ERROR for pipe field is set and the file was loaded with a few errors */
2121
PARTIALLY_LOADED(4, "Partially loaded"),
2222
;
23+
2324
/**
2425
* Constructor, set id and description
2526
*

‎src/main/java/net/snowflake/ingest/connection/OAuthClient.java

+11-4
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
/*
2-
* Copyright (c) 2023 Snowflake Computing Inc. All rights reserved.
2+
* Copyright (c) 2023-2025 Snowflake Computing Inc. All rights reserved.
33
*/
44

55
package net.snowflake.ingest.connection;
66

7+
import static net.snowflake.ingest.utils.HttpUtil.NON_PROXY_HOSTS;
8+
79
import java.io.IOException;
810
import java.io.UnsupportedEncodingException;
11+
import java.net.URI;
912
import java.net.URLEncoder;
1013
import java.util.HashMap;
1114
import java.util.Map;
@@ -53,16 +56,20 @@ public class OAuthClient {
5356
/**
5457
* Creates an AuthClient for Snowflake OAuth given account, credential and base uri
5558
*
56-
* @param accountName - the snowflake account name of this user
5759
* @param oAuthCredential - the OAuth credential we're using to connect
5860
* @param baseURIBuilder - the uri builder with common scheme, host and port
5961
*/
60-
OAuthClient(String accountName, OAuthCredential oAuthCredential, URIBuilder baseURIBuilder) {
62+
OAuthClient(OAuthCredential oAuthCredential, URIBuilder baseURIBuilder) {
6163
this.oAuthCredential = new AtomicReference<>(oAuthCredential);
6264

6365
// build token request uri
6466
baseURIBuilder.setPath(TOKEN_REQUEST_ENDPOINT);
65-
this.httpClient = HttpUtil.getHttpClient(accountName);
67+
URI oAuthTokenEndpoint = oAuthCredential.getOAuthTokenEndpoint();
68+
this.httpClient =
69+
HttpUtil.initHttpClient(
70+
System.getProperty(NON_PROXY_HOSTS) != null
71+
&& HttpUtil.isInNonProxyHosts(
72+
oAuthTokenEndpoint == null ? "" : oAuthTokenEndpoint.getHost()));
6673
}
6774

6875
/** Get access token */

‎src/main/java/net/snowflake/ingest/connection/OAuthManager.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved.
2+
* Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
33
*/
44

55
package net.snowflake.ingest.connection;
@@ -89,7 +89,7 @@ public final class OAuthManager extends SecurityManager {
8989
throw new IllegalArgumentException("updateThresholdRatio should fall in (0, 1)");
9090
}
9191
this.updateThresholdRatio = updateThresholdRatio;
92-
this.oAuthClient = new OAuthClient(accountName, oAuthCredential, baseURIBuilder);
92+
this.oAuthClient = new OAuthClient(oAuthCredential, baseURIBuilder);
9393

9494
// generate our first token
9595
refreshToken();

‎src/main/java/net/snowflake/ingest/connection/RequestBuilder.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved.
2+
* Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
33
*/
44

55
package net.snowflake.ingest.connection;
@@ -110,7 +110,7 @@ public class RequestBuilder {
110110
// Don't change!
111111
public static final String CLIENT_NAME = "SnowpipeJavaSDK";
112112

113-
public static final String DEFAULT_VERSION = "3.0.1";
113+
public static final String DEFAULT_VERSION = "3.1.1";
114114

115115
public static final String JAVA_USER_AGENT = "JAVA";
116116

‎src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java

+1
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ public String getHttpMethod() {
5656
return httpMethod;
5757
}
5858
}
59+
5960
// the object mapper we use for deserialization
6061
static ObjectMapper mapper = new ObjectMapper();
6162

‎src/main/java/net/snowflake/ingest/example/IngestExampleHelper.java

+2
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
public class IngestExampleHelper {
3636
// a logger for all of our needs in this class
3737
private static final Logger LOGGER = LoggerFactory.getLogger(IngestExampleHelper.class.getName());
38+
3839
/**
3940
* Create directories if they don't exist under directoryPath.
4041
*
@@ -133,6 +134,7 @@ public static void doQuery(Connection conn, String query) {
133134
}
134135

135136
private static String ALGORITHM = "RSA";
137+
136138
/**
137139
* Generates an RSA keypair for use in this test
138140
*

‎src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java

+34-4
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import java.util.concurrent.CompletableFuture;
1010
import javax.annotation.Nullable;
1111
import net.snowflake.ingest.streaming.internal.ColumnProperties;
12+
import net.snowflake.ingest.utils.SFException;
1213

1314
/**
1415
* A logical partition that represents a connection to a single Snowflake table, data will be
@@ -66,14 +67,28 @@ public interface SnowflakeStreamingIngestChannel {
6667
*/
6768
String getFullyQualifiedTableName();
6869

69-
/** @return a boolean which indicates whether the channel is valid */
70+
/**
71+
* @return a boolean which indicates whether the channel is valid. Typically, this means whether
72+
* the current instance is the owner of the channel, i.e., if another Client opens a {@link
73+
* SnowflakeStreamingIngestChannel} of the same name then the current instance will be
74+
* considered "invalid" as its persisted epoch will have increased. If this returns false then
75+
* calling `insertRow(s)` on this channel instance will result in an {@link SFException} and
76+
* no further writes to the channel will be accepted.
77+
* <p>>Note: there may be a delay between server-side invalidations and the Client detecting
78+
* it so this may not immediately return false in the event of a server-side invalidation.
79+
*/
7080
boolean isValid();
7181

72-
/** @return a boolean which indicates whether the channel is closed */
82+
/**
83+
* @return a boolean which indicates whether the channel is closed. If true this means that the
84+
* current {@link SnowflakeStreamingIngestChannel} will not accept any additional rows on
85+
* calls to `insertRow(s)`
86+
*/
7387
boolean isClosed();
7488

7589
/**
76-
* Close the channel, this function will make sure all the data in this channel is committed
90+
* Close the channel. Closing entails draining any outstanding data in the Channel's buffer and
91+
* marking the Channel as no longer being able to accept writes via `insertRow(s)`*
7792
*
7893
* @return a completable future which will be completed when the channel is closed
7994
*/
@@ -202,7 +217,7 @@ public interface SnowflakeStreamingIngestChannel {
202217
* </li>
203218
*
204219
* </ul>
205-
*
220+
* <p>
206221
* For TIMESTAMP_LTZ and TIMESTAMP_TZ, all input without timezone will be by default interpreted in the timezone "America/Los_Angeles". This can be changed by calling {@link net.snowflake.ingest.streaming.OpenChannelRequest.OpenChannelRequestBuilder#setDefaultTimezone(ZoneId)}.
207222
* </td>
208223
* <tr>
@@ -244,6 +259,9 @@ public interface SnowflakeStreamingIngestChannel {
244259
* @param offsetToken offset of given row, used for replay in case of failures. It could be null
245260
* if you don't plan on replaying or can't replay
246261
* @return insert response that possibly contains errors because of insertion failures
262+
* @throws SFException if the channel is not considered "valid". Typically, this means that
263+
* another Client has claimed ownership of the Channel. Writes to this channel will be
264+
* rejected and result in this exception being thrown.
247265
*/
248266
InsertValidationResponse insertRow(Map<String, Object> row, @Nullable String offsetToken);
249267

@@ -258,6 +276,9 @@ public interface SnowflakeStreamingIngestChannel {
258276
* @param endOffsetToken end offset of the batch/row-set, used for replay in case of failures, *
259277
* It could be null if you don't plan on replaying or can't replay
260278
* @return insert response that possibly contains errors because of insertion failures
279+
* @throws SFException if the channel is not considered "valid". Typically, this means that
280+
* another Client has claimed ownership of the Channel. Writes to this channel will be
281+
* rejected and result in this exception being thrown.
261282
*/
262283
InsertValidationResponse insertRows(
263284
Iterable<Map<String, Object>> rows,
@@ -267,6 +288,10 @@ InsertValidationResponse insertRows(
267288
/**
268289
* Insert a batch of rows into the channel with the end offset token only, please see {@link
269290
* SnowflakeStreamingIngestChannel#insertRows(Iterable, String, String)} for more information.
291+
*
292+
* @throws SFException if the channel is not considered "valid". Typically, this means that
293+
* another Client has claimed ownership of the Channel. Writes to this channel will be
294+
* rejected and result in this exception being thrown.
270295
*/
271296
InsertValidationResponse insertRows(
272297
Iterable<Map<String, Object>> rows, @Nullable String offsetToken);
@@ -275,6 +300,11 @@ InsertValidationResponse insertRows(
275300
* Get the latest committed offset token from Snowflake
276301
*
277302
* @return the latest committed offset token
303+
* @throws SFException if Snowflake returns an invalid response code from its `status` endpoint
304+
* which typically indicates that the channel needs to be re-opened. This is evaluated with
305+
* respect to the current Client epoch, i.e. if another Client opens the channel with the same
306+
* name this will throw an exception as the current instant is not the active version of the
307+
* channel.
278308
*/
279309
@Nullable
280310
String getLatestCommittedOffsetToken();

0 commit comments

Comments
 (0)