Add Intel Quickassist Technology (QAT) gzip codec to unified compression API

CMake/FindQAT.cmake

@@ -0,0 +1,97 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+include_guard(GLOBAL)
+
+include(ExternalProject)
+
+macro(build_qatzip)
+  message(STATUS "Building QATzip from source")
+  set(QATZIP_BUILD_VERSION "v1.1.2")
+  set(QATZIP_BUILD_SHA256_CHECKSUM
+      "31419fa4b42d217b3e55a70a34545582cbf401a4f4d44738d21b4a3944b1e1ef")
+  set(QATZIP_SOURCE_URL
+      "https://github.com/intel/QATzip/archive/refs/tags/${QATZIP_BUILD_VERSION}.tar.gz"
+  )
+  set(QATZIP_LIB_NAME "qatzip")
+
+  set(QATZIP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/qatzip_ep-install")
+  set(QATZIP_SOURCE_DIR "${QATZIP_PREFIX}/src/qatzip_ep")
+  set(QATZIP_INCLUDE_DIR "${QATZIP_SOURCE_DIR}/include")
+  set(QATZIP_STATIC_LIB_NAME
+      "${CMAKE_STATIC_LIBRARY_PREFIX}${QATZIP_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
+  )
+  set(QATZIP_STATIC_LIB_TARGETS
+      "${QATZIP_SOURCE_DIR}/src/.libs/${QATZIP_STATIC_LIB_NAME}")
+  set(QATZIP_CONFIGURE_ARGS "--prefix=${QATZIP_PREFIX}" "--with-pic"
+                            "--with-ICP_ROOT=${ICP_ROOT}")
+
+  ExternalProject_Add(
+    qatzip_ep
+    PREFIX ${QATZIP_PREFIX}
+    URL ${QATZIP_SOURCE_URL}
+    URL_HASH "SHA256=${QATZIP_BUILD_SHA256_CHECKSUM}"
+    SOURCE_DIR ${QATZIP_SOURCE_DIR}
+    CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env QZ_ROOT=${QATZIP_SOURCE_DIR}
+                      ./configure ${QATZIP_CONFIGURE_ARGS}
+    BUILD_COMMAND ${MAKE_PROGRAM} all
+    BUILD_BYPRODUCTS ${QATZIP_STATIC_LIB_TARGETS}
+    BUILD_IN_SOURCE 1)
+
+  ExternalProject_Add_Step(
+    qatzip_ep pre-configure
+    COMMAND ./autogen.sh
+    DEPENDEES download
+    DEPENDERS configure
+    WORKING_DIRECTORY ${QATZIP_SOURCE_DIR})
+
+  # The include directory must exist before it is referenced by a target.
+  file(MAKE_DIRECTORY "${QATZIP_INCLUDE_DIR}")
+
+  set(QATZIP_LINK_LIBRARIES
+      ZLIB::ZLIB lz4::lz4 "${UDEV_LIBRARY}" "${USDM_DRV_LIBRARY}"
+      "${QAT_S_LIBRARY}" Threads::Threads)
+
+  add_library(qatzip::qatzip STATIC IMPORTED)
+  set_target_properties(
+    qatzip::qatzip
+    PROPERTIES IMPORTED_LOCATION "${QATZIP_STATIC_LIB_TARGETS}"
+               INTERFACE_INCLUDE_DIRECTORIES "${QATZIP_INCLUDE_DIR}"
+               INTERFACE_LINK_LIBRARIES "${QATZIP_LINK_LIBRARIES}")
+
+  add_dependencies(qatzip::qatzip qatzip_ep)
+endmacro()
+
+set(ICP_ROOT $ENV{ICP_ROOT})
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+
+find_package(Threads REQUIRED)
+find_program(MAKE_PROGRAM make REQUIRED)
+
+find_library(UDEV_LIBRARY REQUIRED NAMES udev)
+find_library(
+  USDM_DRV_LIBRARY REQUIRED
+  NAMES usdm_drv_s
+  PATHS "${ICP_ROOT}/build"
+  NO_DEFAULT_PATH)
+find_library(
+  QAT_S_LIBRARY REQUIRED
+  NAMES qat_s
+  PATHS "${ICP_ROOT}/build"
+  NO_DEFAULT_PATH)
+
+message(STATUS "Found udev: ${UDEV_LIBRARY}")
+message(STATUS "Found usdm_drv: ${USDM_DRV_LIBRARY}")
+message(STATUS "Found qat_s: ${QAT_S_LIBRARY}")
+
+build_qatzip()

CMakeLists.txt

@@ -98,6 +98,7 @@ option(VELOX_ENABLE_PARQUET "Enable Parquet support" OFF)
 option(VELOX_ENABLE_ARROW "Enable Arrow support" OFF)
 option(VELOX_ENABLE_REMOTE_FUNCTIONS "Enable remote function support" OFF)
 option(VELOX_ENABLE_CCACHE "Use ccache if installed." ON)
+option(VELOX_ENABLE_QAT "Enable Intel QuickAssist Technology support" OFF)
 
 option(VELOX_BUILD_TEST_UTILS "Builds Velox test utilities" OFF)
 option(VELOX_BUILD_PYTHON_PACKAGE "Builds Velox Python bindings" OFF)
@@ -257,6 +258,11 @@ if(VELOX_ENABLE_PARQUET)
   set(VELOX_ENABLE_ARROW ON)
 endif()
 
+if(VELOX_ENABLE_QAT)
+  find_package(QAT REQUIRED)
+  add_definitions(-DVELOX_ENABLE_QAT)
+endif()
+
 if(VELOX_ENABLE_REMOTE_FUNCTIONS)
   # TODO: Move this to use resolve_dependency(). For some reason, FBThrift
   # requires clients to explicitly install fizz and wangle.
@@ -424,20 +430,18 @@ resolve_dependency(glog)
 set_source(fmt)
 resolve_dependency(fmt)
 
-if(NOT ${VELOX_BUILD_MINIMAL})
-  find_package(ZLIB REQUIRED)
-  find_package(lz4 REQUIRED)
-  find_package(lzo2 REQUIRED)
-  find_package(zstd REQUIRED)
-  find_package(Snappy REQUIRED)
-  if(NOT TARGET zstd::zstd)
-    if(TARGET zstd::libzstd_static)
-      set(ZSTD_TYPE static)
-    else()
-      set(ZSTD_TYPE shared)
-    endif()
-    add_library(zstd::zstd ALIAS zstd::libzstd_${ZSTD_TYPE})
+find_package(ZLIB REQUIRED)
+find_package(lz4 REQUIRED)
+find_package(lzo2 REQUIRED)
+find_package(zstd REQUIRED)
+find_package(Snappy REQUIRED)
+if(NOT TARGET zstd::zstd)
+  if(TARGET zstd::libzstd_static)
+    set(ZSTD_TYPE static)
+  else()
+    set(ZSTD_TYPE shared)
   endif()
+  add_library(zstd::zstd ALIAS zstd::libzstd_${ZSTD_TYPE})
 endif()
 
 set_source(re2)

velox/CMakeLists.txt

@@ -79,3 +79,7 @@ endif()
 if(${VELOX_ENABLE_SUBSTRAIT})
   add_subdirectory(substrait)
 endif()
+
+if(${VELOX_ENABLE_QAT})
+  add_subdirectory(common/compression/v2/qat)
+endif()

velox/common/CMakeLists.txt

@@ -14,6 +14,7 @@
 add_subdirectory(base)
 add_subdirectory(caching)
 add_subdirectory(compression)
+add_subdirectory(compression/v2)
 add_subdirectory(config)
 add_subdirectory(encode)
 add_subdirectory(file)

velox/common/compression/Compression.cpp

@@ -76,6 +76,12 @@ std::string compressionKindToString(CompressionKind kind) {
       return "lz4";
     case CompressionKind_GZIP:
       return "gzip";
+    case CompressionKind_LZ4RAW:
+      return "lz4_raw";
+    case CompressionKind_LZ4HADOOP:
+      return "lz4_hadoop";
+    case CompressionKind_LZOHADOOP:
+      return "lzo_hadoop";
   }
   return folly::to<std::string>("unknown - ", kind);
 }
@@ -89,7 +95,10 @@ CompressionKind stringToCompressionKind(const std::string& kind) {
           {"lzo", CompressionKind_LZO},
           {"zstd", CompressionKind_ZSTD},
           {"lz4", CompressionKind_LZ4},
-          {"gzip", CompressionKind_GZIP}};
+          {"gzip", CompressionKind_GZIP},
+          {"lz4_raw", CompressionKind_LZ4RAW},
+          {"lz4_hadoop", CompressionKind_LZ4HADOOP},
+          {"lzo_hadoop", CompressionKind_LZOHADOOP}};
   auto iter = stringToCompressionKindMap.find(kind);
   if (iter != stringToCompressionKindMap.end()) {
     return iter->second;

velox/common/compression/Compression.h

@@ -29,6 +29,9 @@ enum CompressionKind {
   CompressionKind_ZSTD = 4,
   CompressionKind_LZ4 = 5,
   CompressionKind_GZIP = 6,
+  CompressionKind_LZ4RAW = 7,
+  CompressionKind_LZ4HADOOP = 8,
+  CompressionKind_LZOHADOOP = 9,
   CompressionKind_MAX = INT64_MAX
 };
 

velox/common/compression/tests/CompressionTest.cpp

@@ -31,6 +31,9 @@ TEST_F(CompressionTest, testCompressionNames) {
   EXPECT_EQ("lzo", compressionKindToString(CompressionKind_LZO));
   EXPECT_EQ("lz4", compressionKindToString(CompressionKind_LZ4));
   EXPECT_EQ("zstd", compressionKindToString(CompressionKind_ZSTD));
+  EXPECT_EQ("gzip", compressionKindToString(CompressionKind_GZIP));
+  EXPECT_EQ("lz4_raw", compressionKindToString(CompressionKind_LZ4RAW));
+  EXPECT_EQ("lz4_hadoop", compressionKindToString(CompressionKind_LZ4HADOOP));
   EXPECT_EQ(
       "unknown - 99",
       compressionKindToString(static_cast<CompressionKind>(99)));
@@ -56,6 +59,8 @@ TEST_F(CompressionTest, stringToCompressionKind) {
   EXPECT_EQ(stringToCompressionKind("lz4"), CompressionKind_LZ4);
   EXPECT_EQ(stringToCompressionKind("zstd"), CompressionKind_ZSTD);
   EXPECT_EQ(stringToCompressionKind("gzip"), CompressionKind_GZIP);
+  EXPECT_EQ(stringToCompressionKind("lz4_raw"), CompressionKind_LZ4RAW);
+  EXPECT_EQ(stringToCompressionKind("lz4_hadoop"), CompressionKind_LZ4HADOOP);
   VELOX_ASSERT_THROW(
       stringToCompressionKind("bz2"), "Not support compression kind bz2");
 }

velox/common/compression/v2/CMakeLists.txt

@@ -0,0 +1,37 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if(${VELOX_BUILD_TESTING})
+  add_subdirectory(tests)
+endif()
+
+add_library(
+  velox_common_compression_v2
+  Compression.cpp
+  GzipCompression.cpp
+  HadoopCompressionFormat.cpp
+  Lz4Compression.cpp
+  LzoCompression.cpp
+  SnappyCompression.cpp
+  ZstdCompression.cpp)
+
+target_link_libraries(
+  velox_common_compression_v2
+  velox_common_compression
+  velox_common_base
+  Folly::folly
+  Snappy::snappy
+  zstd::zstd
+  ZLIB::ZLIB
+  lz4::lz4)