diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e2d8c7..7dfe61e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,6 +65,6 @@ endif() # ------------------------------------------------- # Local Development Sandboxes (not included in source control) # ------------------------------------------------- -if (EXISTS ".local") - add_subdirectory(.local) +if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.local") + add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/.local") endif() diff --git a/Src/IACore/imp/cpp/DataOps.cpp b/Src/IACore/imp/cpp/DataOps.cpp index 30f43e5..85a94c4 100644 --- a/Src/IACore/imp/cpp/DataOps.cpp +++ b/Src/IACore/imp/cpp/DataOps.cpp @@ -21,6 +21,52 @@ namespace IACore { + // FNV-1a 32-bit Constants + constexpr uint32_t FNV1A_32_PRIME = 0x01000193; // 16777619 + constexpr uint32_t FNV1A_32_OFFSET = 0x811c9dc5; // 2166136261 + + UINT32 DataOps::Hash(IN CONST String &string) + { + uint32_t hash = FNV1A_32_OFFSET; + for (char c : string) + { + hash ^= static_cast(c); + hash *= FNV1A_32_PRIME; + } + return hash; + } + + UINT32 DataOps::Hash(IN Span data) + { + uint32_t hash = FNV1A_32_OFFSET; + const uint8_t *ptr = static_cast(data.data()); + + for (size_t i = 0; i < data.size(); ++i) + { + hash ^= ptr[i]; + hash *= FNV1A_32_PRIME; + } + return hash; + } + + DataOps::CompressionType DataOps::DetectCompression(IN Span data) + { + if (data.size() < 2) + return CompressionType::None; + + // Check for GZIP Magic Number (0x1F 0x8B) + if (data[0] == 0x1F && data[1] == 0x8B) + return CompressionType::Gzip; + + // Check for ZLIB Magic Number (starts with 0x78) + // 0x78 = Deflate compression with 32k window size + // Valid second bytes: 0x01 (Fastest), 0x9C (Default), 0xDA (Best) + if (data[0] == 0x78 && (data[1] == 0x01 || data[1] == 0x9C || data[1] == 0xDA)) + return CompressionType::Zlib; + + return CompressionType::None; + } + Expected, String> DataOps::ZlibInflate(IN Span data) { z_stream zs{}; @@ -28,6 +74,7 @@ namespace IACore zs.zfree = Z_NULL; zs.opaque = Z_NULL; + // 15 + 32 = Auto-detect Gzip or Zlib if (inflateInit2(&zs, 15 + 32) != Z_OK) return MakeUnexpected("Failed to initialize zlib inflate"); @@ -35,18 +82,28 @@ namespace IACore zs.avail_in = static_cast(data.size()); Vector outBuffer; - outBuffer.resize(data.size() * 2); + // Start with 2x input size. + // Small packets compress well, so maybe 4x for very small inputs. + size_t guessSize = data.size() < 1024 ? data.size() * 4 : data.size() * 2; + outBuffer.resize(guessSize); + + zs.next_out = reinterpret_cast(outBuffer.data()); + zs.avail_out = static_cast(outBuffer.size()); int ret; do { - if (zs.total_out >= outBuffer.size()) + if (zs.avail_out == 0) { - outBuffer.resize(outBuffer.size() * 2); - } + size_t currentPos = zs.total_out; - zs.next_out = reinterpret_cast(outBuffer.data() + zs.total_out); - zs.avail_out = static_cast(outBuffer.size() - zs.total_out); + size_t newSize = outBuffer.size() * 2; + outBuffer.resize(newSize); + + zs.next_out = reinterpret_cast(outBuffer.data() + currentPos); + + zs.avail_out = static_cast(newSize - currentPos); + } ret = inflate(&zs, Z_NO_FLUSH); @@ -55,9 +112,10 @@ namespace IACore inflateEnd(&zs); if (ret != Z_STREAM_END) - return MakeUnexpected("Failed to inflate, corrupt or incomplete data"); + return MakeUnexpected("Failed to inflate: corrupt data or stream error"); outBuffer.resize(zs.total_out); + return outBuffer; } @@ -165,4 +223,46 @@ namespace IACore outBuffer.resize(compressedSize); return outBuffer; } + + Expected, String> DataOps::GZipDeflate(IN Span data) + { + z_stream zs{}; + zs.zalloc = Z_NULL; + zs.zfree = Z_NULL; + zs.opaque = Z_NULL; + + // WindowBits = 15 + 16 (31) -> This forces GZIP encoding + // MemLevel = 8 (default) + // Strategy = Z_DEFAULT_STRATEGY + if (deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY) != Z_OK) + return MakeUnexpected("Failed to initialize gzip deflate"); + + zs.next_in = const_cast(data.data()); + zs.avail_in = static_cast(data.size()); + + Vector outBuffer; + + outBuffer.resize(deflateBound(&zs, data.size()) + 1024); // Additional 1KB buffer for safety + + zs.next_out = reinterpret_cast(outBuffer.data()); + zs.avail_out = static_cast(outBuffer.size()); + + int ret = deflate(&zs, Z_FINISH); + + if (ret != Z_STREAM_END) + { + deflateEnd(&zs); + return MakeUnexpected("Failed to deflate"); + } + + outBuffer.resize(zs.total_out); + + deflateEnd(&zs); + return outBuffer; + } + + Expected, String> DataOps::GZipInflate(IN Span data) + { + return ZlibInflate(data); + } } // namespace IACore \ No newline at end of file diff --git a/Src/IACore/imp/cpp/HttpClient.cpp b/Src/IACore/imp/cpp/HttpClient.cpp index 6c66ace..422848e 100644 --- a/Src/IACore/imp/cpp/HttpClient.cpp +++ b/Src/IACore/imp/cpp/HttpClient.cpp @@ -15,6 +15,7 @@ // along with this program. If not, see . #include +#include #include @@ -48,19 +49,47 @@ namespace IACore { } + String HttpClient::PreprocessResponse(IN CONST String &response) + { + const auto responseBytes = Span{(PCUINT8) response.data(), response.size()}; + const auto compression = DataOps::DetectCompression(responseBytes); + switch (compression) + { + case DataOps::CompressionType::Gzip: { + const auto data = DataOps::GZipInflate(responseBytes); + if (!data) + return response; + return String((PCCHAR) data->data(), data->size()); + } + + case DataOps::CompressionType::Zlib: { + const auto data = DataOps::ZlibInflate(responseBytes); + if (!data) + return response; + return String((PCCHAR) data->data(), data->size()); + } + + case DataOps::CompressionType::None: + default: + break; + } + return response; + } + Expected HttpClient::RawGet(IN CONST String &path, IN Span headers, IN PCCHAR defaultContentType) { auto httpHeaders = BuildHeaders(headers, defaultContentType); static_cast(m_client)->enable_server_certificate_verification(false); - auto res = static_cast(m_client)->Get(path.c_str(), httpHeaders); + auto res = static_cast(m_client)->Get( + (!path.empty() && path[0] != '/') ? ('/' + path).c_str() : path.c_str(), httpHeaders); if (res) { m_lastResponseCode = static_cast(res->status); if (res->status >= 200 && res->status < 300) - return res->body; + return PreprocessResponse(res->body); else return MakeUnexpected(std::format("HTTP Error {} : {}", res->status, res->body)); } @@ -81,14 +110,17 @@ namespace IACore httpHeaders.erase(t); } + static_cast(m_client)->set_keep_alive(true); static_cast(m_client)->enable_server_certificate_verification(false); - auto res = static_cast(m_client)->Post(path.c_str(), httpHeaders, body, contentType.c_str()); + auto res = static_cast(m_client)->Post( + (!path.empty() && path[0] != '/') ? ('/' + path).c_str() : path.c_str(), httpHeaders, body, + contentType.c_str()); if (res) { m_lastResponseCode = static_cast(res->status); if (res->status >= 200 && res->status < 300) - return res->body; + return PreprocessResponse(res->body); else return MakeUnexpected(std::format("HTTP Error {} : {}", res->status, res->body)); } diff --git a/Src/IACore/inc/IACore/DataOps.hpp b/Src/IACore/inc/IACore/DataOps.hpp index b0d5050..47d9da4 100644 --- a/Src/IACore/inc/IACore/DataOps.hpp +++ b/Src/IACore/inc/IACore/DataOps.hpp @@ -22,11 +22,27 @@ namespace IACore { class DataOps { - public: + public: + enum class CompressionType + { + None, + Gzip, + Zlib + }; + + public: + STATIC UINT32 Hash(IN CONST String &string); + STATIC UINT32 Hash(IN Span data); + + STATIC CompressionType DetectCompression(IN Span data); + + STATIC Expected, String> GZipInflate(IN Span data); + STATIC Expected, String> GZipDeflate(IN Span data); + STATIC Expected, String> ZlibInflate(IN Span data); STATIC Expected, String> ZlibDeflate(IN Span data); STATIC Expected, String> ZstdInflate(IN Span data); STATIC Expected, String> ZstdDeflate(IN Span data); }; -} \ No newline at end of file +} // namespace IACore \ No newline at end of file diff --git a/Src/IACore/inc/IACore/HttpClient.hpp b/Src/IACore/inc/IACore/HttpClient.hpp index 6174794..ad87cf6 100644 --- a/Src/IACore/inc/IACore/HttpClient.hpp +++ b/Src/IACore/inc/IACore/HttpClient.hpp @@ -135,12 +135,8 @@ namespace IACore ~HttpClient(); public: - // Automatically adds the following headers, if not present: - // 1) EHeaderType::CONTENT_TYPE = defaultContentType Expected RawGet(IN CONST String &path, IN Span headers, IN PCCHAR defaultContentType = "application/x-www-form-urlencoded"); - // Automatically adds the following headers, if not present: - // 1) EHeaderType::CONTENT_TYPE = defaultContentType Expected RawPost(IN CONST String &path, IN Span headers, IN CONST String &body, IN PCCHAR defaultContentType = "application/x-www-form-urlencoded"); @@ -169,6 +165,9 @@ namespace IACore private: PVOID m_client{}; EResponseCode m_lastResponseCode; + + private: + String PreprocessResponse(IN CONST String& response); }; template