Upgrade vcpkg (#7)

* Try using recent vcpkg

* Try disabling http proxy flag

* Add debug flag

* One more try...

* Turn on debugging

* Cleanup client init

* Add libcurl dependency

* Fix version

* Fix baseline

* Try fixing cache issue

* Remove unneeded include

* Fix HTTP 1.1

* Disable region variable

* Change client init

* 1.11.255

* Initialize endpoint correctly

* 1.11.352

* 1.11.285

* 1.11.255 with traces

* Try 1.11.412 with vcpkg overlay

* Remove invalid patch

* Typo

* Try http2 + cert verification

* Cleanup

* Restore checkout depth

---------

Co-authored-by: Stephane Gouache <[email protected]>

.github/workflows/hosted-ninja-vcpkg_submod.yml

@@ -58,6 +58,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: true
+          fetch-depth: 0
 
       - uses: lukka/[email protected]
 
@@ -105,6 +106,8 @@ jobs:
 
       - name: Run CMake+vcpkg+Ninja+CTest to build packages and generate/build/test the code.
         uses: lukka/run-cmake@v10
+        env:
+          S3_DRIVER_LOGLEVEL: debug
         with:
           configurePreset: 'ninja-multi-vcpkg'
           configurePresetAdditionalArgs: "['-DCMAKE_BUILD_TYPE=Release']"

.github/workflows/hosted-pure-workflow.yml

@@ -66,6 +66,8 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: true
+          fetch-depth: 0
+
       - name: "Create directory '${{ env.VCPKG_DEFAULT_BINARY_CACHE }}'"
         run: mkdir -p $VCPKG_DEFAULT_BINARY_CACHE
         shell: bash
@@ -135,6 +137,10 @@ jobs:
         run: |
           cmake --build --preset ninja-vcpkg-release
 
+      #- name: Run single test (with logs)
+      #  run: |
+      #    builds/ninja-multi-vcpkg/bin/basic_test --gtest_filter=S3DriverTest.GetFileSize
+
       # Test the whole project with CTest, again Release configuration only.
       - name: Test (Release configuration)
         run: |

CMakeLists.txt

@@ -72,6 +72,7 @@ target_link_options(khiopsdriver_file_s3 PRIVATE $<$<CONFIG:RELEASE>:-s>
 )# stripping
 target_link_libraries(khiopsdriver_file_s3 PRIVATE ${AWSSDK_LIBRARIES}
                                                    ZLIB::ZLIB spdlog::spdlog)
+
 target_compile_options(
   khiopsdriver_file_s3
   PRIVATE $<$<CXX_COMPILER_ID:MSVC>:-Wall>

src/s3plugin.cpp

@@ -36,14 +36,17 @@
 #include <memory>
 #include <sstream>
 #include <string>
+#include <aws/core/utils/logging/DefaultLogSystem.h>
+#include <aws/core/utils/logging/ConsoleLogSystem.h>
 
+using namespace Aws::Utils::Logging;
 using namespace s3plugin;
 
 using S3Object = Aws::S3::Model::Object;
 
 int bIsConnected = false;
 
-constexpr const char* S3EndpointProvider = "KHIOPS_ML_S3";
+constexpr const char* KHIOPS_S3 = "KHIOPS_S3";
 
 Aws::SDKOptions options;
 Aws::UniquePtr<Aws::S3::S3Client> client;
@@ -603,7 +606,7 @@ Aws::S3::Model::UploadPartRequest MakeUploadPartRequest(Writer& writer,
 	Aws::S3::Model::UploadPartRequest request =
 	    MakeBaseUploadPartRequest<Aws::S3::Model::UploadPartRequest>(writer);
 
-	const auto body = Aws::MakeShared<Aws::IOStream>(S3EndpointProvider, &pre_buf);
+	const auto body = Aws::MakeShared<Aws::IOStream>(KHIOPS_S3, &pre_buf);
 	request.SetBody(body);
 	return request;
 }
@@ -676,6 +679,9 @@ int driver_connect()
 	Aws::String s3endpoint = "";
 	Aws::String s3region = "us-east-1";
 
+	// Note: this might be useless now since AWS SDK apparently allows setting 
+	// custom endpoints now...
+
 	// Load AWS configuration from file
 	Aws::Auth::AWSCredentials configCredentials;
 	Aws::String userHome = GetEnvironmentVariableOrDefault("HOME", "");
@@ -731,6 +737,10 @@ int driver_connect()
 	// Initialize variables from environment
 	// Both AWS_xxx standard variables and AutoML S3_xxx variables are supported
 	// If both are present, AWS_xxx variables will be given precedence
+
+	// Note: this behavior is normally the same as the one implemented by the SDK
+	// except for the "S3_*" variables that are kept to support legacy applications
+
 	globalBucketName = GetEnvironmentVariableOrDefault("S3_BUCKET_NAME", "");
 	s3endpoint = GetEnvironmentVariableOrDefault("S3_ENDPOINT", s3endpoint);
 	s3endpoint = GetEnvironmentVariableOrDefault("AWS_ENDPOINT_URL", s3endpoint);
@@ -746,17 +756,22 @@ int driver_connect()
 		return false;
 	}
 
-	//options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
+	if (!GetEnvironmentVariableOrDefault("AWS_DEBUG_HTTP_LOGS", "").empty()) {
+		options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
+		options.loggingOptions.logger_create_fn = [] { return std::make_shared<ConsoleLogSystem>(LogLevel::Debug); };
+	}
 
 	// Initialisation du SDK AWS
 	Aws::InitAPI(options);
 
-	Aws::Client::ClientConfiguration clientConfig;
-	clientConfig.allowSystemProxy = true;
-	clientConfig.verifySSL = false;
+	Aws::Client::ClientConfiguration clientConfig(true, "legacy", true);
+	clientConfig.allowSystemProxy = getenv("http_proxy") != NULL || getenv("https_proxy") != NULL ||
+		getenv("HTTP_PROXY") != NULL || getenv("HTTPS_PROXY") != NULL || getenv("S3_ALLOW_SYSTEM_PROXY");
+	clientConfig.verifySSL = true;
+	clientConfig.version = Aws::Http::Version::HTTP_VERSION_2TLS;
 	if (s3endpoint != "")
 	{
-		clientConfig.endpointOverride = s3endpoint;
+		clientConfig.endpointOverride = std::move(s3endpoint);
 	}
 	if (s3region != "")
 	{
@@ -768,8 +783,8 @@ int driver_connect()
 		configCredentials = Aws::Auth::AWSCredentials(s3accessKey, s3secretKey);
 	}
 
-	client = Aws::MakeUnique<Aws::S3::S3Client>(S3EndpointProvider, configCredentials,
-						    Aws::MakeShared<Aws::S3::S3EndpointProvider>(S3EndpointProvider),
+	client = Aws::MakeUnique<Aws::S3::S3Client>(KHIOPS_S3, configCredentials, 
+							Aws::MakeShared<Aws::S3::S3EndpointProvider>(KHIOPS_S3),
 						    clientConfig);
 
 	bIsConnected = true;
@@ -818,7 +833,8 @@ int driver_disconnect()
 	active_reader_handles.clear();
 
 	client.reset();
-
+
+	//Aws::Utils::Logging::ShutdownAWSLogging();
 	ShutdownAPI(options);
 
 	bIsConnected = kFalse;
@@ -1035,7 +1051,7 @@ SimpleOutcome<ReaderPtr> MakeReaderPtr(Aws::String bucketname, Aws::String objec
 		Aws::Vector<Aws::String> objectnames(1, objectname);
 		Aws::Vector<tOffset> sizes(1, size);
 
-		return Aws::MakeUnique<Reader>(S3EndpointProvider, std::move(bucketname), std::move(objectname), 0, 0,
+		return Aws::MakeUnique<Reader>(KHIOPS_S3, std::move(bucketname), std::move(objectname), 0, 0,
 					       std::move(objectnames), std::move(sizes));
 	}
 
@@ -1097,7 +1113,7 @@ SimpleOutcome<ReaderPtr> MakeReaderPtr(Aws::String bucketname, Aws::String objec
 	}
 
 	// construct the result
-	return Aws::MakeUnique<Reader>(S3EndpointProvider, std::move(bucketname), std::move(objectname), 0,
+	return Aws::MakeUnique<Reader>(KHIOPS_S3, std::move(bucketname), std::move(objectname), 0,
 				       common_header_length, std::move(filenames), std::move(cumulative_size));
 }
 
@@ -1108,7 +1124,7 @@ SimpleOutcome<WriterPtr> MakeWriterPtr(Aws::String bucket, Aws::String object)
 	request.SetKey(std::move(object));
 	auto outcome = client->CreateMultipartUpload(request);
 	RETURN_OUTCOME_ON_ERROR(outcome);
-	return Aws::MakeUnique<Writer>(S3EndpointProvider, outcome.GetResultWithOwnership());
+	return Aws::MakeUnique<Writer>(KHIOPS_S3, outcome.GetResultWithOwnership());
 }
 
 // This template is only here to get specialized

test/CMakeLists.txt

@@ -39,6 +39,9 @@ target_link_libraries(basic_test PRIVATE ${AWSSDK_LIBRARIES})
 target_link_libraries(basic_test PRIVATE GTest::gtest GTest::gmock
                                          GTest::gmock_main khiopsdriver_file_s3)
 
+if (WIN32)
+  target_link_libraries(basic_test PRIVATE ws2_32 Wininet winhttp)
+endif()
 gtest_discover_tests(basic_test)
 
 add_executable(plugin_test plugin_test.cpp path_helper.cpp)

vcpkg-configuration.json

@@ -0,0 +1,5 @@
+{
+	"overlay-ports": [
+		"./vcpkg_overlay"
+	]
+}

vcpkg.json

@@ -25,5 +25,12 @@
     {
       "name": "zlib"
     }
+  ],
+  "builtin-baseline": "91f002cae2281636da5155efc5a11d67efa72415",
+  "overrides": [
+    {
+        "name": "aws-sdk-cpp",
+        "version": "1.11.412"
+    }
   ]
 }

vcpkg_overlay/aws-c-auth/portfile.cmake

@@ -0,0 +1,32 @@
+vcpkg_from_github(
+    OUT_SOURCE_PATH SOURCE_PATH
+    REPO awslabs/aws-c-auth
+    REF "v${VERSION}"
+    SHA512 1847b7790590cf079785c4fb98fbcde9988cbc68254d18904a8676802bf96168e7038799642d4b67b834c6c772b7a35c6e7f7d35b93438cb9c0ceac9130e31ad
+    HEAD_REF master
+)
+
+vcpkg_cmake_configure(
+    SOURCE_PATH "${SOURCE_PATH}"
+    OPTIONS
+        "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files
+        -DBUILD_TESTING=FALSE
+)
+
+vcpkg_cmake_install()
+
+string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}")
+vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH)
+vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake")
+vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/")
+
+file(REMOVE_RECURSE
+    "${CURRENT_PACKAGES_DIR}/debug/include"
+    "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}"
+    "${CURRENT_PACKAGES_DIR}/debug/share"
+    "${CURRENT_PACKAGES_DIR}/lib/${PORT}"
+)
+
+vcpkg_copy_pdbs()
+
+vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE")

vcpkg_overlay/aws-c-auth/vcpkg.json

@@ -0,0 +1,25 @@
+{
+  "name": "aws-c-auth",
+  "version": "0.7.31",
+  "description": "C99 library implementation of AWS client-side authentication: standard credentials providers and signing.",
+  "homepage": "https://github.com/awslabs/aws-c-auth",
+  "license": "Apache-2.0",
+  "supports": "!(windows & arm) & !uwp",
+  "dependencies": [
+    "aws-c-cal",
+    "aws-c-http",
+    "aws-c-sdkutils",
+    {
+      "name": "s2n",
+      "platform": "!uwp & !windows"
+    },
+    {
+      "name": "vcpkg-cmake",
+      "host": true
+    },
+    {
+      "name": "vcpkg-cmake-config",
+      "host": true
+    }
+  ]
+}

vcpkg_overlay/aws-c-cal/portfile.cmake

@@ -0,0 +1,33 @@
+vcpkg_from_github(
+    OUT_SOURCE_PATH SOURCE_PATH
+    REPO awslabs/aws-c-cal
+    REF "v${VERSION}"
+    SHA512 7c0fa1fe976e7b432b5126792bd1fad94cf5bacffb4e9a5e374e173f617722431bea880b6026c2c664372399635cbf9f129b6ad48c9aca7c087fb6f94fb81837
+    HEAD_REF master
+    PATCHES remove-libcrypto-messages.patch
+)
+
+vcpkg_cmake_configure(
+    SOURCE_PATH "${SOURCE_PATH}"
+    OPTIONS
+        "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files
+        -DBUILD_TESTING=FALSE
+)
+
+vcpkg_cmake_install()
+
+string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}")
+vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH)
+vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake")
+vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/")
+
+file(REMOVE_RECURSE
+    "${CURRENT_PACKAGES_DIR}/debug/include"
+    "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}"
+    "${CURRENT_PACKAGES_DIR}/debug/share"
+    "${CURRENT_PACKAGES_DIR}/lib/${PORT}"
+)
+
+vcpkg_copy_pdbs()
+
+vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE")

vcpkg_overlay/aws-c-cal/remove-libcrypto-messages.patch

@@ -0,0 +1,14 @@
+diff --git a/cmake/modules/Findcrypto.cmake b/cmake/modules/Findcrypto.cmake
+index fed83bb..9c1ae28 100644
+--- a/cmake/modules/Findcrypto.cmake
++++ b/cmake/modules/Findcrypto.cmake
+@@ -105,9 +105,6 @@ else()
+         set(CRYPTO_FOUND true)
+         set(crypto_FOUND true)
+
+-        message(STATUS "LibCrypto Include Dir: ${crypto_INCLUDE_DIR}")
+-        message(STATUS "LibCrypto Shared Lib:  ${crypto_SHARED_LIBRARY}")
+-        message(STATUS "LibCrypto Static Lib:  ${crypto_STATIC_LIBRARY}")
+         if (NOT TARGET AWS::crypto AND
+             (EXISTS "${crypto_LIBRARY}")
+             )

vcpkg_overlay/aws-c-cal/vcpkg.json

@@ -0,0 +1,23 @@
+{
+  "name": "aws-c-cal",
+  "version": "0.7.4",
+  "description": "C99 wrapper for cryptography primitives.",
+  "homepage": "https://github.com/awslabs/aws-c-cal",
+  "license": "Apache-2.0",
+  "supports": "!(windows & arm) & !uwp",
+  "dependencies": [
+    "aws-c-common",
+    {
+      "name": "openssl",
+      "platform": "!windows & !osx"
+    },
+    {
+      "name": "vcpkg-cmake",
+      "host": true
+    },
+    {
+      "name": "vcpkg-cmake-config",
+      "host": true
+    }
+  ]
+}

vcpkg_overlay/aws-c-common/portfile.cmake

@@ -0,0 +1,33 @@
+vcpkg_from_github(
+    OUT_SOURCE_PATH SOURCE_PATH
+    REPO awslabs/aws-c-common
+    REF "v${VERSION}"
+    SHA512 25da9356e36c87210bcdd95b007824288f36fd3ae4bdd757a1d3e88ef3cc8b65a0c1a31cbe338147949257c8e908c1721fc6297aeb0cbfe7cb89b4d7727dc2ad
+    HEAD_REF master
+)
+
+vcpkg_cmake_configure(
+    SOURCE_PATH "${SOURCE_PATH}"
+    OPTIONS
+        -DBUILD_TESTING=FALSE
+)
+
+vcpkg_cmake_install()
+
+vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake) # central macros
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share")
+
+string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}")
+vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH)
+vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake")
+vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/")
+
+file(REMOVE_RECURSE
+    "${CURRENT_PACKAGES_DIR}/debug/include"
+    "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}"
+    "${CURRENT_PACKAGES_DIR}/lib/${PORT}"
+)
+
+vcpkg_copy_pdbs()
+
+vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE")

vcpkg_overlay/aws-c-common/vcpkg.json

@@ -0,0 +1,18 @@
+{
+  "name": "aws-c-common",
+  "version": "0.9.28",
+  "description": "AWS common library for C",
+  "homepage": "https://github.com/awslabs/aws-c-common",
+  "license": "Apache-2.0",
+  "supports": "!(windows & arm) & !uwp",
+  "dependencies": [
+    {
+      "name": "vcpkg-cmake",
+      "host": true
+    },
+    {
+      "name": "vcpkg-cmake-config",
+      "host": true
+    }
+  ]
+}