SNOW-1437443: use random db to allow test in parallel (#722)

Co-authored-by: sfc-gh-ext-simba-hx <[email protected]>

Author: SimbaGithub

tests/test_parallel_upload_download.cpp

@@ -5,6 +5,8 @@
 #include <aws/core/Aws.h>
 #include <snowflake/client.h>
 #include <fstream>
+#include <stdlib.h>
+#include <time.h>
 #include "utils/test_setup.h"
 #include "utils/TestSetup.hpp"
 #include "snowflake/IStatementPutGet.hpp"
@@ -66,12 +68,15 @@ void test_parallel_upload_download_core(int fileNumber, int numberOfRows = 1)
   assert_int_equal(SF_STATUS_SUCCESS, ret);
 
   std::string dataDir = TestSetup::getDataDir();
-  std::string testDir = dataDir + "test_parallel_upload";
-  testDir += PATH_SEP;
+  time_t t;
+  srand((unsigned) time(&t));
+  std::string randStr = std::string("_") + std::to_string(rand());
+  std::string uploadDir = dataDir + "test_parallel_upload" + randStr;
+  uploadDir += PATH_SEP;
 
-  populateDataInTestDir(testDir, fileNumber, numberOfRows);
+  populateDataInTestDir(uploadDir, fileNumber, numberOfRows);
 
-  std::string files = testDir + "*";
+  std::string files = uploadDir + "*";
   std::string putCommand = "put file://" + files + " @%test_parallel_upload_download auto_compress=false";
 
   std::unique_ptr<IStatementPutGet> stmtPutGet = std::unique_ptr
@@ -101,7 +106,8 @@ void test_parallel_upload_download_core(int fileNumber, int numberOfRows = 1)
   }
   assert_int_equal(status, SF_STATUS_EOF);
 
-  std::string getCommand = "get @%test_parallel_upload_download file://" + dataDir + "test_parallel_download";
+  std::string downloadDir = dataDir + "test_parallel_download" + randStr;
+  std::string getCommand = "get @%test_parallel_upload_download file://" + downloadDir;
   result = agent.execute(&getCommand);
   assert_int_equal(fileNumber, result->getResultSize());
 
@@ -113,9 +119,9 @@ void test_parallel_upload_download_core(int fileNumber, int numberOfRows = 1)
   }
 
 #ifdef _WIN32
-  std::string compCmd = "FC " + dataDir + "test_parallel_upload\\* " + dataDir + "test_parallel_download\\*";
+  std::string compCmd = "FC " + uploadDir + "* " + downloadDir + "\\*";
 #else
-  std::string compCmd = "diff " + dataDir + "test_parallel_upload/ " + dataDir + "test_parallel_download/";
+  std::string compCmd = "diff " + uploadDir + " " + downloadDir + "/";
 #endif
   int res = system(compCmd.c_str());
   assert_int_equal(0, res);
@@ -124,6 +130,9 @@ void test_parallel_upload_download_core(int fileNumber, int numberOfRows = 1)
 
   /* close and term */
   snowflake_term(sf); // purge snowflake context
+
+  sf_delete_directory_if_exists(uploadDir.c_str());
+  sf_delete_directory_if_exists(downloadDir.c_str());
 }
 
 static int teardown(void **unused)
@@ -141,12 +150,6 @@ static int teardown(void **unused)
   snowflake_stmt_term(sfstmt);
   snowflake_term(sf);
 
-  std::string dataDir = TestSetup::getDataDir();
-  std::string uploadDir = dataDir + "test_parallel_upload";
-  std::string downloadDir = dataDir + "test_parallel_download";
-
-  sf_delete_directory_if_exists(uploadDir.c_str());
-  sf_delete_directory_if_exists(downloadDir.c_str());
   return 0;
 }
 
@@ -162,17 +165,31 @@ void test_large_file_multipart_upload(void **unused)
 
 void test_large_file_concurrent_upload_download(void **unused)
 {
+// Jenkins node on Mac has issue with large file.
+#ifdef __APPLE__
+  char* jobname = getenv("JOB_NAME");
+  if (jobname && (strlen(jobname) > 0))
+  {
+    return;
+  }
+#endif
+
   test_parallel_upload_download_core(10, 200000);
 }
 
 static int gr_setup(void **unused)
 {
   initialize_test(SF_BOOLEAN_FALSE);
+  if(!setup_random_database()) {
+    std::cout << "Failed to setup random database, fallback to use regular one." << std::endl;
+  }
+
   return 0;
 }
 
 static int gr_teardown(void **unused)
 {
+  drop_random_database();
   snowflake_global_term();
   return 0;
 }

tests/test_simple_put.cpp

@@ -856,6 +856,10 @@ static int gr_setup(void **unused)
 {
   initialize_test(SF_BOOLEAN_FALSE);
 
+  if(!setup_random_database()) {
+    std::cout << "Failed to setup random database, fallback to use regular one." << std::endl;
+  }
+
   // create large 2GB file
   char *githubenv = getenv("GITHUB_ACTIONS");
   if (githubenv && strlen(githubenv) > 0)
@@ -865,6 +869,14 @@ static int gr_setup(void **unused)
       return 0;
     }
   }
+// Jenkins node on Mac has issue with large file.
+#ifdef __APPLE__
+  char* jobname = getenv("JOB_NAME");
+  if (jobname && (strlen(jobname) > 0))
+  {
+    return 0;
+  }
+#endif
 
   std::string file2GB = TestSetup::getDataDir() + FILE_NAME_2GB;
   std::ofstream ofs(file2GB, std::ios::binary | std::ios::out);
@@ -877,6 +889,7 @@ static int gr_setup(void **unused)
 
 static int gr_teardown(void **unused)
 {
+  drop_random_database();
   snowflake_global_term();
   std::string file2GB = TestSetup::getDataDir() + FILE_NAME_2GB;
   remove(file2GB.c_str());
@@ -1225,6 +1238,15 @@ void test_2GBlarge_put(void **unused)
     return;
   }
 
+// Jenkins node on Mac has issue with large file.
+#ifdef __APPLE__
+  char* jobname = getenv("JOB_NAME");
+  if (jobname && (strlen(jobname) > 0))
+  {
+    return;
+  }
+#endif
+
   test_simple_put_core(FILE_NAME_2GB, // filename
                        "none", //source compression
                        false,   // auto compress
@@ -1255,6 +1277,15 @@ void test_2GBlarge_get(void **unused)
     return;
   }
 
+// Jenkins node on Mac has issue with large file.
+#ifdef __APPLE__
+  char* jobname = getenv("JOB_NAME");
+  if (jobname && (strlen(jobname) > 0))
+  {
+    return;
+  }
+#endif
+
   std::string getcmd = std::string("get @%test_small_put/") + FILE_NAME_2GB +
                        " file://" + TestSetup::getDataDir();
   test_simple_get_data(getcmd.c_str(), std::to_string(FILE_SIZE_2GB).c_str());

tests/utils/test_setup.c

@@ -5,10 +5,15 @@
 #include <snowflake/logger.h>
 #include <string.h>
 #include "test_setup.h"
+#include <stdlib.h>
+#include <time.h>
 
 // Long path space
 char PERFORMANCE_TEST_RESULTS_PATH[5000];
 
+// random database namespace
+char RANDOM_DATABASE_NAME[100];
+
 void initialize_test(sf_bool debug) {
     // default location and the maximum logging
     snowflake_global_init(NULL, SF_LOG_TRACE, NULL);
@@ -127,3 +132,89 @@ void process_results(struct timespec begin, struct timespec end, int num_iterati
     //printf("%s, %lf, %i\n", label, time_elapsed, num_iterations);
     fclose(results_file);
 }
+
+int setup_random_database()
+{
+    SF_STATUS status;
+    SF_CONNECT *sf;
+    SF_STMT *sfstmt;
+    char random_db_name[100];
+    char query[200];
+    time_t t;
+
+    if (strlen(RANDOM_DATABASE_NAME) > 0)
+    {
+        // already setup
+        return 1;
+    }
+
+    srand((unsigned) time(&t));
+    sprintf(random_db_name, "random_test_db_%d", rand());
+
+    sf = setup_snowflake_connection();
+    status = snowflake_connect(sf);
+    if (status != SF_STATUS_SUCCESS) {
+        return 0;
+    }
+    sfstmt = snowflake_stmt(sf);
+
+    sprintf(query, "create or replace database %s", random_db_name);
+    status = snowflake_query(
+      sfstmt,
+      query,
+      0);
+    if (status != SF_STATUS_SUCCESS) {
+      // ingore in case the test account doesn't have privilege to create database
+      snowflake_stmt_term(sfstmt);
+      snowflake_term(sf);
+	  return 0;
+    }
+
+    sprintf(query, "create or replace schema %s", getenv("SNOWFLAKE_TEST_SCHEMA"));
+    status = snowflake_query(
+      sfstmt,
+      query,
+      0);
+    if (status != SF_STATUS_SUCCESS) {
+      // ingore in case the test account doesn't have privilege to create database
+      snowflake_stmt_term(sfstmt);
+      snowflake_term(sf);
+	  return 0;
+    }
+
+    sf_setenv("SNOWFLAKE_TEST_DATABASE", random_db_name);
+    strcpy(RANDOM_DATABASE_NAME, random_db_name);
+    snowflake_stmt_term(sfstmt);
+    snowflake_term(sf);
+    return 1;
+}
+
+void drop_random_database()
+{
+    SF_STATUS status;
+    SF_CONNECT *sf;
+    SF_STMT *sfstmt;
+    char query[200];
+
+    if (strlen(RANDOM_DATABASE_NAME) == 0)
+    {
+        // not setup
+        return;
+    }
+
+    sf = setup_snowflake_connection();
+    status = snowflake_connect(sf);
+    if (status != SF_STATUS_SUCCESS) {
+        dump_error(&(sf->error));
+    }
+    assert_int_equal(status, SF_STATUS_SUCCESS);
+    sfstmt = snowflake_stmt(sf);
+
+    sprintf(query, "drop database %s", RANDOM_DATABASE_NAME);
+    status = snowflake_query(
+      sfstmt,
+      query,
+      0);
+    snowflake_stmt_term(sfstmt);
+    snowflake_term(sf);
+}

tests/utils/test_setup.h

@@ -42,6 +42,11 @@ void setup_and_run_query(SF_CONNECT **sfp, SF_STMT **sfstmtp, const char *query)
 
 void process_results(struct timespec begin, struct timespec end, int num_iterations, const char *label);
 
+// Setup database with random name for test, return 1 if succeeded, 0 otherwise.
+int setup_random_database();
+
+void drop_random_database();
+
 /**
  * Dump error
  * @param error SF_ERROR_STRUCT