SNOW-1342953: read lob size from server parameters (#735)

Author: sfc-gh-ext-simba-hx

include/snowflake/client.h

@@ -59,7 +59,7 @@ extern "C" {
 /**
  * The maximum object size
  */
-#define SF_MAX_OBJECT_SIZE 16777216
+#define SF_MAX_OBJECT_SIZE SF_MACRO_DEPRECATED_WARNING("SF_MAX_OBJECT_SIZE is deprecated, please use snowflake_get_attribute() instead to retrieve the max LOB size.") 16777216 
 
 /**
  * Login timeout in seconds
@@ -259,6 +259,9 @@ typedef enum SF_ATTRIBUTE {
     SF_CON_INCLUDE_RETRY_REASON,
     SF_CON_RETRY_TIMEOUT,
     SF_CON_MAX_RETRY,
+    SF_CON_MAX_VARCHAR_SIZE,
+    SF_CON_MAX_BINARY_SIZE,
+    SF_CON_MAX_VARIANT_SIZE,
     SF_DIR_QUERY_URL,
     SF_DIR_QUERY_URL_PARAM,
     SF_DIR_QUERY_TOKEN,
@@ -385,6 +388,11 @@ typedef struct SF_CONNECT {
 
     // Error
     SF_ERROR_STRUCT error;
+
+    // max lob size
+    uint64 max_varchar_size;
+    uint64 max_binary_size;
+    uint64 max_variant_size;
 } SF_CONNECT;
 
 /**

lib/client.c

@@ -175,6 +175,13 @@ static SF_STATUS STDCALL _reset_connection_parameters(
                 sf->qcc_capacity = snowflake_cJSON_GetUint64Value(value);
                 qcc_set_capacity(sf, sf->qcc_capacity);
             }
+            else if (strcmp(name->valuestring, "VARCHAR_AND_BINARY_MAX_SIZE_IN_RESULT") == 0) {
+                sf->max_varchar_size = snowflake_cJSON_GetUint64Value(value);
+                sf->max_binary_size = sf->max_varchar_size / 2;
+            }
+            else if (strcmp(name->valuestring, "VARIANT_MAX_SIZE_IN_RESULT") == 0) {
+                sf->max_variant_size = snowflake_cJSON_GetUint64Value(value);
+            }
         }
     }
     SF_STATUS ret = SF_STATUS_ERROR_GENERAL;
@@ -712,6 +719,10 @@ SF_CONNECT *STDCALL snowflake_init() {
         sf->qcc_capacity = SF_QCC_CAPACITY_DEF;
         sf->qcc_disable = SF_BOOLEAN_FALSE;
         sf->qcc = NULL;
+
+        sf->max_varchar_size = SF_DEFAULT_MAX_OBJECT_SIZE;
+        sf->max_binary_size = SF_DEFAULT_MAX_OBJECT_SIZE / 2;
+        sf->max_variant_size = SF_DEFAULT_MAX_OBJECT_SIZE;
     }
 
     return sf;
@@ -1263,6 +1274,15 @@ SF_STATUS STDCALL snowflake_get_attribute(
         case SF_CON_INCLUDE_RETRY_REASON:
             *value = &sf->include_retry_reason;
             break;
+        case SF_CON_MAX_VARCHAR_SIZE:
+            *value = &sf->max_varchar_size;
+            break;
+        case SF_CON_MAX_BINARY_SIZE:
+            *value = &sf->max_binary_size;
+            break;
+        case SF_CON_MAX_VARIANT_SIZE:
+            *value = &sf->max_variant_size;
+            break;
         default:
             SET_SNOWFLAKE_ERROR(&sf->error, SF_STATUS_ERROR_BAD_ATTRIBUTE_TYPE,
                                 "Invalid attribute type",

lib/client_int.h

@@ -23,6 +23,8 @@
 
 #define DEFAULT_SNOWFLAKE_REQUEST_TIMEOUT 60
 
+#define SF_DEFAULT_MAX_OBJECT_SIZE 16777216
+
 #define SESSION_URL "/session/v1/login-request"
 #define QUERY_URL "/queries/v1/query-request"
 #define RENEW_SESSION_URL "/session/token-request"

tests/test_crud.c

@@ -16,6 +16,8 @@ void _fetch_data(SF_STMT *sfstmt, int64 expected_sum) {
     assert_int_equal(num_fields, 2);
 
     SF_COLUMN_DESC *descs = snowflake_desc(sfstmt);
+    uint64* max_varchar_size_p = NULL;
+    snowflake_get_attribute(sfstmt->connection, SF_CON_MAX_VARCHAR_SIZE, (void**)&max_varchar_size_p);
     int i;
     for (i = 0; i < num_fields; ++i) {
         switch (i) {
@@ -35,8 +37,8 @@ void _fetch_data(SF_STMT *sfstmt, int64 expected_sum) {
                 assert_int_equal(descs[i].idx, 2);
                 assert_int_equal(descs[i].type, SF_DB_TYPE_TEXT);
                 assert_int_equal(descs[i].c_type, SF_C_TYPE_STRING);
-                assert_int_equal(descs[i].byte_size, 16777216);
-                assert_int_equal(descs[i].internal_size, 16777216);
+                assert_int_equal(descs[i].byte_size, *max_varchar_size_p);
+                assert_int_equal(descs[i].internal_size, *max_varchar_size_p);
                 assert_int_equal(descs[i].precision, 0);
                 assert_int_equal(descs[i].scale, 0);
                 assert_int_equal(descs[i].null_ok, 1);

tests/test_unit_file_metadata_init.cpp

@@ -6,8 +6,6 @@
 #include <FileMetadataInitializer.hpp>
 #include "FileMetadata.hpp"
 #include "FileCompressionType.hpp"
-#include "utils/test_setup.h"
-#include "utils/TestSetup.hpp"
 #include "snowflake/platform.h"
 #include <unordered_set>
 #include <iostream>
@@ -16,6 +14,8 @@
 #include <memory>
 #include "snowflake/IStatementPutGet.hpp"
 #include "StatementPutGet.hpp"
+#include "utils/test_setup.h"
+#include "utils/TestSetup.hpp"
 
 #define FILES_IN_DIR "file1.csv", "file2.csv", "file3.csv", "file4.csv", "file1.gz"
 

tests/test_unit_jwt.cpp

@@ -2,11 +2,11 @@
  * Copyright (c) 2018-2019 Snowflake Computing, Inc. All rights reserved.
  */
 
-#include "utils/TestSetup.hpp"
-#include "utils/test_setup.h"
-#include <jwt/Jwt.hpp>
 #include "openssl/rsa.h"
 #include <openssl/pem.h>
+#include <jwt/Jwt.hpp>
+#include "utils/TestSetup.hpp"
+#include "utils/test_setup.h"
 
 using Snowflake::Client::Jwt::IHeader;
 using Snowflake::Client::Jwt::IClaimSet;

tests/test_unit_set_get_attributes.cpp

@@ -4,10 +4,11 @@
 
 #include <cassert>
 #include <string>
-#include "utils/test_setup.h"
-#include "utils/TestSetup.hpp"
 #include "memory.h"
+#include "../lib/client_int.h"
 #include <vector>
+#include "utils/test_setup.h"
+#include "utils/TestSetup.hpp"
 
 typedef struct sf_string_attributes {
     SF_ATTRIBUTE type;
@@ -72,13 +73,15 @@ std::vector<sf_int_attributes> intAttributes = {
     { SF_CON_MAX_RETRY, 6 },
     { SF_CON_RETRY_TIMEOUT, 0 },
     { SF_CON_MAX_RETRY, 0 },
+    { SF_CON_MAX_VARCHAR_SIZE, SF_DEFAULT_MAX_OBJECT_SIZE },
+    { SF_CON_MAX_BINARY_SIZE, SF_DEFAULT_MAX_OBJECT_SIZE / 2 },
+    { SF_CON_MAX_VARIANT_SIZE, SF_DEFAULT_MAX_OBJECT_SIZE },
 };
 
 // unit test for snowflake_set_attribute and snowflake_get_attribute for all SF_ATTRIBUTE
 void test_set_get_all_attributes(void **unused)
 {
-    SF_CONNECT *sf = (SF_CONNECT *)SF_CALLOC(1, sizeof(SF_CONNECT));
-    memset(sf, 0, sizeof(SF_CONNECT));
+    SF_CONNECT *sf = snowflake_init();
 
     // Connection parameters that cannot be set by user
     sf->service_name = (char*)"test_service_name";
@@ -152,12 +155,17 @@ void test_set_get_all_attributes(void **unused)
     // set and get int attributes
     for (sf_int_attributes attr : intAttributes)
     {
-        status = snowflake_set_attribute(sf, attr.type, &attr.value);
-        if (status != SF_STATUS_SUCCESS)
+        if ((attr.type != SF_CON_MAX_VARCHAR_SIZE) &&
+            (attr.type != SF_CON_MAX_BINARY_SIZE) &&
+            (attr.type != SF_CON_MAX_VARIANT_SIZE))
         {
-            dump_error(&(sf->error));
+            status = snowflake_set_attribute(sf, attr.type, &attr.value);
+            if (status != SF_STATUS_SUCCESS)
+            {
+                dump_error(&(sf->error));
+            }
+            assert_int_equal(status, SF_STATUS_SUCCESS);
         }
-        assert_int_equal(status, SF_STATUS_SUCCESS);
 
         status = snowflake_get_attribute(sf, attr.type, &value);
         if (status != SF_STATUS_SUCCESS)

tests/test_unit_snowflake_types_to_string.cpp

@@ -4,10 +4,10 @@
 
 #include <cassert>
 #include <string>
-#include "utils/test_setup.h"
-#include "utils/TestSetup.hpp"
 #include "memory.h"
 #include <vector>
+#include "utils/test_setup.h"
+#include "utils/TestSetup.hpp"
 
 typedef struct sf_db_types {
     SF_DB_TYPE sf_dbType;