From 03fba38bd19af78e8484cbfb3b050701bfedb377 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 28 Sep 2022 20:47:42 -0700 Subject: [PATCH] Switch the build system to CMake By popular request, use CMake instead of a plain Makefile. Some parts of this commit are based on the pull request https://github.com/ebiggers/libdeflate/pull/101, which was written by Benjamin Sergeant and SpaceIm <30052553+SpaceIm@users.noreply.github.com>. --- .cirrus.yml | 6 +- .github/workflows/android.yml | 104 ----- .github/workflows/ci.yml | 74 +++- .gitignore | 22 +- CMakeLists.txt | 245 ++++++++++++ Makefile | 431 --------------------- Makefile.msc | 65 ---- README.md | 173 ++------- common_defs.h | 9 - libdeflate-config.cmake.in | 3 + libdeflate.pc.in | 6 +- programs/CMakeLists.txt | 95 +++++ programs/config.h.in | 22 ++ programs/gzip.c | 8 +- scripts/afl-fuzz/.gitignore | 1 - scripts/afl-fuzz/Makefile | 11 - scripts/afl-fuzz/fuzz.sh | 13 +- scripts/android_build.sh | 35 +- scripts/android_tests.sh | 9 +- scripts/benchmark.sh | 10 + scripts/check.sh | 18 - scripts/checksum.sh | 10 + scripts/checksum_benchmarks.sh | 14 +- scripts/cmake-helper.sh | 15 + scripts/deflate_benchmarks.sh | 8 +- scripts/detect.sh | 63 --- scripts/exec_tests.sh | 4 + scripts/make-windows-releases.sh | 15 +- scripts/pgo_build.sh | 23 -- scripts/run_tests.sh | 74 ++-- scripts/toolchain-i686-w64-mingw32.cmake | 8 + scripts/toolchain-x86_64-w64-mingw32.cmake | 8 + 32 files changed, 620 insertions(+), 982 deletions(-) delete mode 100644 .github/workflows/android.yml create mode 100644 CMakeLists.txt delete mode 100644 Makefile delete mode 100644 Makefile.msc create mode 100644 libdeflate-config.cmake.in create mode 100644 programs/CMakeLists.txt create mode 100644 programs/config.h.in delete mode 100644 scripts/afl-fuzz/.gitignore delete mode 100644 scripts/afl-fuzz/Makefile create mode 100755 scripts/benchmark.sh delete mode 100755 scripts/check.sh create mode 100755 scripts/checksum.sh create mode 100755 scripts/cmake-helper.sh delete mode 100755 scripts/detect.sh delete mode 100755 scripts/pgo_build.sh create mode 100644 scripts/toolchain-i686-w64-mingw32.cmake create mode 100644 scripts/toolchain-x86_64-w64-mingw32.cmake diff --git a/.cirrus.yml b/.cirrus.yml index c175120b..a4f5cad5 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -3,6 +3,8 @@ task: matrix: - image_family: freebsd-12-3 - image_family: freebsd-13-0 - install_script: pkg install -y gmake + install_script: pkg install -y cmake script: - - gmake check + - cmake -B build -DLIBDEFLATE_BUILD_TESTS=1 + - cmake --build build + - ctest --test-dir build diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml deleted file mode 100644 index f02729b3..00000000 --- a/.github/workflows/android.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: cross compile with android-ndk - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - cross-compile-for-android: - strategy: - fail-fast: false - matrix: - os: [ubuntu-18.04, ubuntu-20.04, macos-10.15, macos-11] - target-abi: [armeabi-v7a, arm64-v8a, x86, x86_64] - - runs-on: ${{ matrix.os }} - - steps: - - run: brew install tree file - - - run: tree --version - - run: file --version - - run: make --version - - - uses: actions/checkout@v2 - - - name: build for ${{ matrix.target-abi }} - run: | - COLOR_GREEN='\033[0;32m' # Green - COLOR_PURPLE='\033[0;35m' # Purple - COLOR_OFF='\033[0m' # Reset - - run() { - printf '%b\n' "$COLOR_PURPLE==>$COLOR_OFF $COLOR_GREEN$*$COLOR_OFF" - eval "$*" - } - - BUILD_MACHINE_OS_TYPE=$(uname | tr A-Z a-z) - BUILD_MACHINE_OS_ARCH=$(uname -m) - - export ANDROID_NDK_HOME="$ANDROID_NDK_LATEST_HOME" - - env | sed -n '/^ANDROID_/p' - - ANDROID_NDK_VERS=$(grep "Pkg.Revision" "$ANDROID_NDK_HOME/source.properties" | cut -d " " -f3) - ANDROID_NDK_BASE=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/$BUILD_MACHINE_OS_TYPE-$BUILD_MACHINE_OS_ARCH - ANDROID_NDK_BIND=$ANDROID_NDK_BASE/bin - SYSROOT=$ANDROID_NDK_BASE/sysroot - - printf '%s = %s\n' ANDROID_NDK_HOME "$ANDROID_NDK_HOME" - printf '%s = %s\n' ANDROID_NDK_BIND "$ANDROID_NDK_BIND" - printf '%s = %s\n' ANDROID_NDK_VERS "$ANDROID_NDK_VERS" - printf '%s\n' - - export CPPFLAGS="--sysroot $SYSROOT" - export CFLAGS="--sysroot $SYSROOT -fpic" - export LDFLAGS="--sysroot $SYSROOT" - - case ${{ matrix.target-abi }} in - armeabi-v7a) CC=armv7a-linux-androideabi21-clang ;; - arm64-v8a) CC=aarch64-linux-android21-clang ;; - x86) CC=i686-linux-android21-clang ;; - x86_64) CC=x86_64-linux-android21-clang ;; - esac - - AR=llvm-ar - - export PATH="$ANDROID_NDK_BIND:$PATH" - - printf '%b\n' "$COLOR_PURPLE==>$COLOR_OFF ${COLOR_GREEN}show PATH$COLOR_OFF" - printf '%s\n' "$PATH" | tr ':' '\n' - printf '%s\n' - - run make clean install PREFIX=$PWD/install.d CC="$CC" CPPFLAGS="'$CPPFLAGS'" CFLAGS="'$CFLAGS'" AR="$AR" V=1 - - - run: tree install.d - - - run: | - file install.d/lib/libdeflate.so - case ${{ matrix.target-abi }} in - armeabi-v7a) file install.d/lib/libdeflate.so | grep 'ELF 32-bit LSB shared object, ARM, EABI5' ;; - arm64-v8a) file install.d/lib/libdeflate.so | grep 'ELF 64-bit LSB shared object, ARM aarch64,' ;; - x86) file install.d/lib/libdeflate.so | grep 'ELF 32-bit LSB shared object, Intel 80386,' ;; - x86_64) file install.d/lib/libdeflate.so | grep 'ELF 64-bit LSB shared object, x86-64,' ;; - esac - - - run: | - file install.d/bin/libdeflate-gzip - case ${{ matrix.target-abi }} in - armeabi-v7a) file install.d/bin/libdeflate-gzip | grep 'ELF 32-bit LSB pie executable, ARM, EABI5' ;; - arm64-v8a) file install.d/bin/libdeflate-gzip | grep 'ELF 64-bit LSB pie executable, ARM aarch64,' ;; - x86) file install.d/bin/libdeflate-gzip | grep 'ELF 32-bit LSB pie executable, Intel 80386,' ;; - x86_64) file install.d/bin/libdeflate-gzip | grep 'ELF 64-bit LSB pie executable, x86-64,' ;; - esac - - - run: | - file install.d/bin/libdeflate-gunzip - case ${{ matrix.target-abi }} in - armeabi-v7a) file install.d/bin/libdeflate-gunzip | grep 'ELF 32-bit LSB pie executable, ARM, EABI5' ;; - arm64-v8a) file install.d/bin/libdeflate-gunzip | grep 'ELF 64-bit LSB pie executable, ARM aarch64,' ;; - x86) file install.d/bin/libdeflate-gunzip | grep 'ELF 32-bit LSB pie executable, Intel 80386,' ;; - x86_64) file install.d/bin/libdeflate-gunzip | grep 'ELF 64-bit LSB pie executable, x86-64,' ;; - esac diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b28f66ef..c9ab9bb6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,5 @@ name: CI on: [pull_request] -env: - CFLAGS: -Werror -DLIBDEFLATE_ENABLE_ASSERTIONS jobs: x86_64-build-and-test: @@ -13,6 +11,7 @@ jobs: runs-on: ${{ matrix.os }} env: CC: ${{ matrix.compiler }} + CFLAGS: -Werror -DLIBDEFLATE_ENABLE_ASSERTIONS steps: - uses: actions/checkout@v2 - name: Install dependencies @@ -28,6 +27,8 @@ jobs: arch: [armv6, armv7, aarch64, s390x, ppc64le] compiler: [gcc, clang] runs-on: ubuntu-latest + env: + CFLAGS: -Werror -DLIBDEFLATE_ENABLE_ASSERTIONS steps: - uses: actions/checkout@v2 - uses: uraimo/run-on-arch-action@v2.2.0 @@ -37,7 +38,7 @@ jobs: githubToken: ${{ github.token }} install: | apt-get update - apt-get install -y build-essential clang llvm libz-dev valgrind + apt-get install -y build-essential cmake clang llvm libz-dev valgrind run: | # Valgrind and ASAN crash on at least s390x, ppc64le, and aarch64 # here. (It's probably something related to the QEMU user-mode @@ -52,9 +53,11 @@ jobs: export SKIP_FREESTANDING=1 ;; aarch64) - # "ldd: exited with unknown exit code (139)" if [ ${{ matrix.compiler }} = clang ]; then + # "ldd: exited with unknown exit code (139)" export SKIP_SHARED_LIB=1 + # "archive has no index; run ranlib to add one" + export SKIP_CFI=1 fi ;; esac @@ -65,9 +68,13 @@ jobs: macos-build-and-test: name: Build and test (macOS) runs-on: macos-latest + env: + CFLAGS: -Werror -DLIBDEFLATE_ENABLE_ASSERTIONS steps: - uses: actions/checkout@v2 - - run: make all check + - run: cmake -B build -DLIBDEFLATE_BUILD_TESTS=1 + - run: cmake --build build --verbose + - run: ctest --test-dir build windows-msys2-build-and-test: name: Build and test (Windows, MSYS2, ${{matrix.sys}}) @@ -80,22 +87,32 @@ jobs: defaults: run: shell: msys2 {0} + env: + CFLAGS: -Werror -DLIBDEFLATE_ENABLE_ASSERTIONS steps: - uses: actions/checkout@v2 - uses: msys2/setup-msys2@v2 with: msystem: ${{matrix.sys}} update: true - install: make mingw-w64-${{matrix.env}}-cc mingw-w64-${{matrix.env}}-zlib - - run: make all check + install: > + make + mingw-w64-${{matrix.env}}-cc + mingw-w64-${{matrix.env}}-cmake + mingw-w64-${{matrix.env}}-ninja + mingw-w64-${{matrix.env}}-zlib + - run: cmake -B build -G Ninja -DLIBDEFLATE_BUILD_TESTS=1 + - run: cmake --build build --verbose + - run: ctest --test-dir build windows-msvc-build: name: Build (Windows, Visual Studio) runs-on: windows-latest steps: - uses: actions/checkout@v2 - - uses: ilammy/msvc-dev-cmd@v1 - - run: nmake /f Makefile.msc + - uses: microsoft/setup-msbuild@v1.1 + - run: cmake -B build -G "Visual Studio 17 2022" + - run: cmake --build build --verbose run-clang-static-analyzer: name: Run clang static analyzer @@ -106,8 +123,8 @@ jobs: run: | sudo apt-get update sudo apt-get install -y clang-tools - - name: Run clang static analyzer - run: make scan-build + - run: scan-build cmake -B build -DLIBDEFLATE_BUILD_TESTS=1 + - run: scan-build cmake --build build --verbose run-shellcheck: name: Run shellcheck @@ -119,18 +136,47 @@ jobs: sudo apt-get update sudo apt-get install -y shellcheck - name: Run shellcheck - run: make shellcheck + run: shellcheck scripts/*.sh cross-compile-for-windows: name: Cross compile for Windows runs-on: ubuntu-latest + env: + CFLAGS: -Werror -DLIBDEFLATE_ENABLE_ASSERTIONS steps: - uses: actions/checkout@v2 - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 libz-mingw-w64-dev + # Unfortunately Ubuntu doesn't have {i686,x86_64}-w64-mingw32-cmake like + # some distros have, so we have to provide our own toolchain files here. - name: 32-bit build - run: make CC=i686-w64-mingw32-gcc all test_programs + run: | + scripts/cmake-helper.sh -DLIBDEFLATE_BUILD_TESTS=1 \ + -DCMAKE_TOOLCHAIN_FILE=scripts/toolchain-i686-w64-mingw32.cmake + cmake --build build --verbose - name: 64-bit build - run: make CC=x86_64-w64-mingw32-gcc all test_programs + run: | + scripts/cmake-helper.sh -DLIBDEFLATE_BUILD_TESTS=1 \ + -DCMAKE_TOOLCHAIN_FILE=scripts/toolchain-x86_64-w64-mingw32.cmake + cmake --build build --verbose + + cross-compile-for-android: + name: Cross compile for ${{matrix.abi}} Android on ${{matrix.os}} + strategy: + matrix: + os: [ubuntu-18.04, ubuntu-20.04, macos-latest] + abi: [armeabi-v7a, arm64-v8a, x86, x86_64] + runs-on: ${{matrix.os}} + env: + CFLAGS: -Werror -DLIBDEFLATE_ENABLE_ASSERTIONS + steps: + - uses: actions/checkout@v2 + - run: | + scripts/cmake-helper.sh \ + -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK_LATEST_HOME"/build/cmake/android.toolchain.cmake \ + -DANDROID_ABI=${{matrix.abi}} \ + -DANDROID_PLATFORM=28 \ + -DLIBDEFLATE_BUILD_TESTS=1 + cmake --build build --verbose diff --git a/.gitignore b/.gitignore index f0b08611..3a696efc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,21 +1,3 @@ -*.a -*.def -*.dll -*.dllobj -*.dylib -*.exe -*.exp -*.lib -*.o -*.obj -*.so -*.so.* -/.build-config -/programs/config.h -/benchmark -/checksum -/gzip -/gunzip -/test_* -tags +/build* cscope* +tags diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..6d4f84ee --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,245 @@ +cmake_minimum_required(VERSION 3.7) + +# Default to a release build. +if(NOT CMAKE_BUILD_TYPE) + message(STATUS "No build type selected; defaulting to Release") + set(CMAKE_BUILD_TYPE Release) +endif() + +# Extract the version string from libdeflate.h so that it doesn't have to be +# duplicated here. +set(VERSION_REGEX "#define LIBDEFLATE_VERSION_STRING[ \t]+\"([0-9\\.]+)\"") +file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/libdeflate.h VERSION_STRING REGEX ${VERSION_REGEX}) +string(REGEX REPLACE ${VERSION_REGEX} "\\1" VERSION_STRING "${VERSION_STRING}") + +# Declare the project. +project(libdeflate + LANGUAGES C + VERSION ${VERSION_STRING}) + +# Include the CMake modules required by the top-level directory. +include(CMakePackageConfigHelpers) +include(CheckCCompilerFlag) +include(GNUInstallDirs) + +# Declare the options, which can be overridden via 'cmake -DOPTION=VALUE'. +option(LIBDEFLATE_BUILD_STATIC_LIB "Build the static library" ON) +option(LIBDEFLATE_BUILD_SHARED_LIB "Build the shared library" ON) +option(LIBDEFLATE_COMPRESSION_SUPPORT "Support compression" ON) +option(LIBDEFLATE_DECOMPRESSION_SUPPORT "Support decompression" ON) +option(LIBDEFLATE_ZLIB_SUPPORT "Support the zlib format" ON) +option(LIBDEFLATE_GZIP_SUPPORT "Support the gzip format" ON) +option(LIBDEFLATE_FREESTANDING + "Build a freestanding library, i.e. a library that doesn't link to any + libc functions like malloc(), free(), and memcpy(). Library users will + need to call libdeflate_set_memory_allocator()." OFF) +option(LIBDEFLATE_BUILD_GZIP "Build the libdeflate-gzip program" ON) +option(LIBDEFLATE_BUILD_TESTS "Build the test programs" OFF) +option(LIBDEFLATE_USE_SHARED_LIB + "Link the libdeflate-gzip and test programs to the shared library instead + of the static library" OFF) + +if(LIBDEFLATE_BUILD_TESTS) + enable_testing() +endif() + +# The gzip program can't be built if any library feature it needs is disabled. +if(NOT LIBDEFLATE_COMPRESSION_SUPPORT OR NOT LIBDEFLATE_DECOMPRESSION_SUPPORT + OR NOT LIBDEFLATE_GZIP_SUPPORT) + set(LIBDEFLATE_BUILD_GZIP OFF) +endif() + +# If the static library isn't being built, we have to link to the shared one. +if(NOT LIBDEFLATE_BUILD_STATIC_LIB) + set(LIBDEFLATE_USE_SHARED_LIB ON) +endif() + +# Set common C compiler flags for all targets (the library and the programs). +set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG") +set(CMAKE_C_STANDARD 99) +if(NOT MSVC) + check_c_compiler_flag(-Wdeclaration-after-statement HAVE_WDECLARATION_AFTER_STATEMENT) + check_c_compiler_flag(-Wimplicit-fallthrough HAVE_WIMPLICIT_FALLTHROUGH) + check_c_compiler_flag(-Wmissing-prototypes HAVE_WMISSING_PROTOTYPES) + check_c_compiler_flag(-Wpedantic HAVE_WPEDANTIC) + check_c_compiler_flag(-Wshadow HAVE_WSHADOW) + check_c_compiler_flag(-Wstrict-prototypes HAVE_WSTRICT_PROTOTYPES) + check_c_compiler_flag(-Wundef HAVE_WUNDEF) + check_c_compiler_flag(-Wvla HAVE_WVLA) + add_compile_options( + -Wall + $<$:-Wdeclaration-after-statement> + $<$:-Wimplicit-fallthrough> + $<$:-Wmissing-prototypes> + $<$:-Wpedantic> + $<$:-Wshadow> + $<$:-Wstrict-prototypes> + $<$:-Wundef> + $<$:-Wvla> + ) +endif() +if(LIBDEFLATE_FREESTANDING) + add_definitions(-DFREESTANDING) +endif() + +# Determine the list of source files and the list of compiler options that will +# be used for both the static library and the shared library. + +set(LIB_SOURCES + lib/arm/cpu_features.c + lib/arm/cpu_features.h + lib/cpu_features_common.h + lib/deflate_constants.h + lib/lib_common.h + lib/utils.c + lib/x86/cpu_features.c + lib/x86/cpu_features.h +) +if(LIBDEFLATE_COMPRESSION_SUPPORT) + list(APPEND LIB_SOURCES + lib/arm/matchfinder_impl.h + lib/bt_matchfinder.h + lib/deflate_compress.c + lib/deflate_compress.h + lib/hc_matchfinder.h + lib/ht_matchfinder.h + lib/matchfinder_common.h + lib/x86/matchfinder_impl.h + ) +endif() +if(LIBDEFLATE_DECOMPRESSION_SUPPORT) + list(APPEND LIB_SOURCES + lib/decompress_template.h + lib/deflate_decompress.c + lib/x86/decompress_impl.h + ) +endif() +if(LIBDEFLATE_ZLIB_SUPPORT) + list(APPEND LIB_SOURCES + lib/adler32.c + lib/adler32_vec_template.h + lib/arm/adler32_impl.h + lib/x86/adler32_impl.h + lib/zlib_constants.h + ) + if(LIBDEFLATE_COMPRESSION_SUPPORT) + list(APPEND LIB_SOURCES lib/zlib_compress.c) + endif() + if(LIBDEFLATE_DECOMPRESSION_SUPPORT) + list(APPEND LIB_SOURCES lib/zlib_decompress.c) + endif() +endif() +if(LIBDEFLATE_GZIP_SUPPORT) + list(APPEND LIB_SOURCES + lib/arm/crc32_impl.h + lib/arm/crc32_pmull_helpers.h + lib/arm/crc32_pmull_wide.h + lib/crc32.c + lib/crc32_multipliers.h + lib/crc32_tables.h + lib/gzip_constants.h + lib/x86/crc32_impl.h + lib/x86/crc32_pclmul_template.h + ) + if(LIBDEFLATE_COMPRESSION_SUPPORT) + list(APPEND LIB_SOURCES lib/gzip_compress.c) + endif() + if(LIBDEFLATE_DECOMPRESSION_SUPPORT) + list(APPEND LIB_SOURCES lib/gzip_decompress.c) + endif() +endif() + +if(NOT MSVC) + list(APPEND LIB_COMPILE_DEFINITIONS _ANSI_SOURCE) +endif() + +if(LIBDEFLATE_FREESTANDING) + list(APPEND LIB_COMPILE_OPTIONS -ffreestanding -nostdlib) + list(APPEND LIB_LINK_LIBRARIES -ffreestanding -nostdlib) +endif() + +set(LIB_INCLUDE_DIRS + $ + $) + +# Build the static library. +if(LIBDEFLATE_BUILD_STATIC_LIB) + add_library(libdeflate_static STATIC ${LIB_SOURCES}) + + # This alias allows third-party usage of the library with CMake to work the + # same way with add_subdirectory() as with other ways. + add_library(libdeflate::libdeflate_static ALIAS libdeflate_static) + + set_target_properties(libdeflate_static PROPERTIES + OUTPUT_NAME deflate + PUBLIC_HEADER libdeflate.h) + target_include_directories(libdeflate_static PUBLIC ${LIB_INCLUDE_DIRS}) + target_compile_definitions(libdeflate_static PUBLIC LIBDEFLATE_STATIC) + target_compile_definitions(libdeflate_static PRIVATE ${LIB_COMPILE_DEFINITIONS}) + target_compile_options(libdeflate_static PRIVATE ${LIB_COMPILE_OPTIONS}) + list(APPEND LIB_TARGETS libdeflate_static) +endif() + +# Build the shared library. +if(LIBDEFLATE_BUILD_SHARED_LIB) + add_library(libdeflate_shared SHARED ${LIB_SOURCES}) + + # This alias allows third-party usage of the library with CMake to work the + # same way with add_subdirectory() as with other ways. + add_library(libdeflate::libdeflate_shared ALIAS libdeflate_shared) + + set_target_properties(libdeflate_shared PROPERTIES + OUTPUT_NAME deflate + PUBLIC_HEADER libdeflate.h + C_VISIBILITY_PRESET hidden + SOVERSION 0) + target_include_directories(libdeflate_shared PUBLIC ${LIB_INCLUDE_DIRS}) + target_compile_definitions(libdeflate_shared PRIVATE ${LIB_COMPILE_DEFINITIONS}) + target_compile_options(libdeflate_shared PRIVATE ${LIB_COMPILE_OPTIONS}) + target_link_libraries(libdeflate_shared PRIVATE ${LIB_LINK_LIBRARIES}) + list(APPEND LIB_TARGETS libdeflate_shared) +endif() + +# Install the static and/or shared library. +install(TARGETS ${LIB_TARGETS} + EXPORT libdeflate_exported_targets + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + +# Generate and install the pkg-config file. (Don't confuse this with the CMake +# package config file, which is CMake-specific.) +configure_file(libdeflate.pc.in libdeflate.pc) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libdeflate.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + +# Generate a "libdeflate-targets.cmake" file in the build tree that can be +# included by outside projects to import targets from the build tree. +export(EXPORT libdeflate_exported_targets + NAMESPACE libdeflate:: + FILE libdeflate-targets.cmake) + +# Generate and install a separate "libdeflate-targets.cmake" file that can be +# included by outside projects to import targets from the installation tree. +install(EXPORT libdeflate_exported_targets + NAMESPACE libdeflate:: + FILE libdeflate-targets.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libdeflate) + +# Generate and install the CMake package version and config files. +write_basic_package_version_file(libdeflate-config-version.cmake + VERSION ${PROJECT_VERSION} + COMPATIBILITY AnyNewerVersion) +configure_package_config_file( + ${CMAKE_CURRENT_SOURCE_DIR}/libdeflate-config.cmake.in + libdeflate-config.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libdeflate) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libdeflate-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/libdeflate-config-version.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libdeflate) + +# Build the programs subdirectory if needed. +if(LIBDEFLATE_BUILD_GZIP OR LIBDEFLATE_BUILD_TESTS) + add_subdirectory(programs) +endif() diff --git a/Makefile b/Makefile deleted file mode 100644 index 1114baed..00000000 --- a/Makefile +++ /dev/null @@ -1,431 +0,0 @@ -# -# Use 'make help' to list available targets. -# -# Define V=1 to enable "verbose" mode, showing all executed commands. -# -# Define USE_SHARED_LIB to link the binaries to the shared library version of -# libdeflate rather than to the static library version. -# -# Define DISABLE_SHARED to disable building and installing of shared library. -# Defining this variable undefines USE_SHARED_LIB. -# -# Define DECOMPRESSION_ONLY to omit all compression code, building a -# decompression-only library. If doing this, you must also build a specific -# library target such as 'libdeflate.a', as the programs will no longer compile. -# -# Define DISABLE_GZIP to disable support for the gzip wrapper format. -# -# Define DISABLE_ZLIB to disable support for the zlib wrapper format. -# -# Define PREFIX to override the installation prefix, like './configure --prefix' -# in autotools-based projects (default: /usr/local) -# -# Define BINDIR to override where to install binaries, like './configure -# --bindir' in autotools-based projects (default: PREFIX/bin) -# -# Define INCDIR to override where to install headers, like './configure -# --includedir' in autotools-based projects (default: PREFIX/include) -# -# Define LIBDIR to override where to install libraries, like './configure -# --libdir' in autotools-based projects (default: PREFIX/lib) -# -# Define DESTDIR to override the installation destination directory -# (default: empty string) -# -# Define FREESTANDING to build a freestanding library, i.e. a library that -# doesn't link to any libc functions like malloc(), free(), and memcpy(). -# All users will need to call libdeflate_set_memory_allocator(). -# -# You can also specify a custom CC, CFLAGS, CPPFLAGS, and/or LDFLAGS. -# -############################################################################## - -VERSION = $(shell sed -n 's/\#define LIBDEFLATE_VERSION_STRING.*"\(.*\)"/\1/p' \ - libdeflate.h) - -#### Common compiler flags. You can add additional flags by defining CFLAGS -#### in the environment or on the 'make' command line. -#### -#### The default optimization flags can be overridden, e.g. via CFLAGS="-O3" or -#### CFLAGS="-O0 -fno-omit-frame-pointer". But this usually isn't recommended; -#### you're unlikely to get significantly better performance even with -O3. - -cc-option = $(shell if $(CC) $(1) -c -x c /dev/null -o /dev/null \ - 1>&2 2>/dev/null; then echo $(1); fi) - -override CFLAGS := \ - -O2 -fomit-frame-pointer -std=c99 -I. -Wall -Wundef \ - $(call cc-option,-Wdeclaration-after-statement) \ - $(call cc-option,-Wimplicit-fallthrough) \ - $(call cc-option,-Wmissing-prototypes) \ - $(call cc-option,-Wpedantic) \ - $(call cc-option,-Wshadow) \ - $(call cc-option,-Wstrict-prototypes) \ - $(call cc-option,-Wvla) \ - $(CFLAGS) - -FREESTANDING := -ifdef FREESTANDING -override CPPFLAGS += -DFREESTANDING -LIB_CFLAGS += -ffreestanding -nostdlib -endif - -# Don't use this option except for testing; it isn't a stable interface. -TEST_SUPPORT__DO_NOT_USE := -ifdef TEST_SUPPORT__DO_NOT_USE -override CPPFLAGS += -DTEST_SUPPORT__DO_NOT_USE -endif - -############################################################################## - -PREFIX ?= /usr/local -BINDIR ?= $(PREFIX)/bin -INCDIR ?= $(PREFIX)/include -LIBDIR ?= $(PREFIX)/lib - -SOVERSION := 0 - -IMPLIB := -PROG_SUFFIX := -PROG_CFLAGS := -HARD_LINKS := 1 - -TARGET_MACHINE := $(shell $(CC) -dumpmachine 2>/dev/null) - -# Compiling for Windows with MinGW or LLVM? -ifneq ($(findstring -mingw,$(TARGET_MACHINE))$(findstring -windows-gnu,$(TARGET_MACHINE)),) - SHARED_LIB := libdeflate.dll - SHARED_LIB_SYMLINK := - SHARED_LIB_CFLAGS := - SHARED_LIB_LDFLAGS := -Wl,--out-implib,libdeflate.dll.a \ - -Wl,--output-def,libdeflate.def - IMPLIB := libdeflate.dll.a - PROG_SUFFIX := .exe - PROG_CFLAGS := -static -municode - HARD_LINKS := - override CFLAGS := $(CFLAGS) $(call cc-option,-Wno-pedantic-ms-format) - - # If AR was not already overridden, then derive it from $(CC). - # Note that CC may take different forms, e.g. "cc", "gcc", "clang", - # "x86_64-w64-mingw32-gcc", or "x86_64-w64-mingw32-gcc-6.3.1", or - # "x86_64-w64-mingw32-clang". - # On Windows it may also have a .exe extension. - ifeq ($(AR),ar) - AR := $(shell echo $(CC) | \ - sed -E 's/(g?cc|clang)(-?[0-9]+(\.[0-9]+)*)?(\.exe)?$$/ar\4/') - endif - -# Compiling for macOS? -else ifneq ($(findstring -darwin,$(TARGET_MACHINE)),) - SHARED_LIB := libdeflate.$(SOVERSION).dylib - SHARED_LIB_SYMLINK := libdeflate.dylib - SHARED_LIB_CFLAGS := -fPIC - SHARED_LIB_LDFLAGS := -install_name $(LIBDIR)/$(SHARED_LIB) - -# Compiling for Android? -else ifneq ($(findstring -android,$(TARGET_MACHINE)),) - SHARED_LIB := libdeflate.so - SHARED_LIB_SYMLINK := - SHARED_LIB_CFLAGS := -fPIC - SHARED_LIB_LDFLAGS := -Wl,-soname=$(SHARED_LIB) - -# Compiling for Haiku? -else ifneq ($(findstring -haiku,$(TARGET_MACHINE)),) - SHARED_LIB := libdeflate.so.$(SOVERSION) - SHARED_LIB_SYMLINK := libdeflate.so - SHARED_LIB_CFLAGS := -fPIC - SHARED_LIB_LDFLAGS := -Wl,-soname=$(SHARED_LIB) - HARD_LINKS := - -# Compiling for Linux, FreeBSD, etc. -else - SHARED_LIB := libdeflate.so.$(SOVERSION) - SHARED_LIB_SYMLINK := libdeflate.so - SHARED_LIB_CFLAGS := -fPIC - SHARED_LIB_LDFLAGS := -Wl,-soname=$(SHARED_LIB) -endif - -############################################################################## - -#### Quiet make is enabled by default. Define V=1 to disable. - -ifneq ($(findstring s,$(MAKEFLAGS)),s) -ifneq ($(V),1) - QUIET_CC = @echo ' CC ' $@; - QUIET_CCLD = @echo ' CCLD ' $@; - QUIET_AR = @echo ' AR ' $@; - QUIET_LN = @echo ' LN ' $@; - QUIET_CP = @echo ' CP ' $@; - QUIET_GEN = @echo ' GEN ' $@; -endif -endif - -############################################################################## - -# Rebuild if a user-specified setting that affects the build changed. -.build-config: FORCE - @flags=$$( \ - echo 'USE_SHARED_LIB=$(USE_SHARED_LIB)'; \ - echo 'DISABLE_SHARED=$(DISABLE_SHARED)'; \ - echo 'DECOMPRESSION_ONLY=$(DECOMPRESSION_ONLY)'; \ - echo 'DISABLE_GZIP=$(DISABLE_GZIP)'; \ - echo 'DISABLE_ZLIB=$(DISABLE_ZLIB)'; \ - echo 'FREESTANDING=$(FREESTANDING)'; \ - echo 'CC=$(CC)'; \ - echo 'CFLAGS=$(CFLAGS)'; \ - echo 'CPPFLAGS=$(CPPFLAGS)'; \ - echo 'LDFLAGS=$(LDFLAGS)'; \ - ); \ - if [ "$$flags" != "`cat $@ 2>/dev/null`" ]; then \ - [ -e $@ ] && echo "Rebuilding due to new settings"; \ - echo "$$flags" > $@; \ - fi - -############################################################################## - -COMMON_HEADERS := libdeflate.h common_defs.h -DEFAULT_TARGETS := - -#### Library - -STATIC_LIB := libdeflate.a - -LIB_CFLAGS += $(CFLAGS) -fvisibility=hidden -D_ANSI_SOURCE - -LIB_HEADERS := $(wildcard lib/*.h) $(wildcard lib/*/*.h) - -LIB_SRC := lib/deflate_decompress.c lib/utils.c \ - $(wildcard lib/*/cpu_features.c) - -DECOMPRESSION_ONLY := -ifndef DECOMPRESSION_ONLY - LIB_SRC += lib/deflate_compress.c -endif - -DISABLE_ZLIB := -ifndef DISABLE_ZLIB - LIB_SRC += lib/adler32.c lib/zlib_decompress.c - ifndef DECOMPRESSION_ONLY - LIB_SRC += lib/zlib_compress.c - endif -endif - -DISABLE_GZIP := -ifndef DISABLE_GZIP - LIB_SRC += lib/crc32.c lib/gzip_decompress.c - ifndef DECOMPRESSION_ONLY - LIB_SRC += lib/gzip_compress.c - endif -endif - -STATIC_LIB_OBJ := $(LIB_SRC:.c=.o) -SHARED_LIB_OBJ := $(LIB_SRC:.c=.shlib.o) - -# Compile static library object files -$(STATIC_LIB_OBJ): %.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .build-config - $(QUIET_CC) $(CC) -o $@ -c $(CPPFLAGS) $(LIB_CFLAGS) \ - -DLIBDEFLATE_STATIC $< - -# Compile shared library object files -$(SHARED_LIB_OBJ): %.shlib.o: %.c $(LIB_HEADERS) $(COMMON_HEADERS) .build-config - $(QUIET_CC) $(CC) -o $@ -c $(CPPFLAGS) $(LIB_CFLAGS) \ - $(SHARED_LIB_CFLAGS) $< - -# Create static library -$(STATIC_LIB):$(STATIC_LIB_OBJ) - $(QUIET_AR) $(AR) cr $@ $+ - -DEFAULT_TARGETS += $(STATIC_LIB) - -# Create shared library -$(SHARED_LIB):$(SHARED_LIB_OBJ) - $(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(LIB_CFLAGS) \ - $(SHARED_LIB_LDFLAGS) -shared $+ - -ifdef DISABLE_SHARED -undefine SHARED_LIB_SYMLINK -else -DEFAULT_TARGETS += $(SHARED_LIB) -endif - -ifdef SHARED_LIB_SYMLINK -# Create the symlink libdeflate.so => libdeflate.so.$SOVERSION -$(SHARED_LIB_SYMLINK):$(SHARED_LIB) - $(QUIET_LN) ln -sf $+ $@ -DEFAULT_TARGETS += $(SHARED_LIB_SYMLINK) -endif - -############################################################################## - -#### Programs - -PROG_CFLAGS += $(CFLAGS) \ - -D_POSIX_C_SOURCE=200809L \ - -D_FILE_OFFSET_BITS=64 \ - -DHAVE_CONFIG_H - -ifndef USE_SHARED_LIB -PROG_CFLAGS += -DLIBDEFLATE_STATIC -endif - -ALL_PROG_COMMON_HEADERS := programs/config.h \ - programs/prog_util.h \ - programs/test_util.h -PROG_COMMON_SRC := programs/prog_util.c \ - programs/tgetopt.c -NONTEST_PROG_SRC := programs/gzip.c -TEST_PROG_COMMON_SRC := programs/test_util.c -TEST_PROG_SRC := programs/benchmark.c \ - programs/checksum.c \ - $(filter-out $(TEST_PROG_COMMON_SRC),$(wildcard programs/test_*.c)) - -NONTEST_PROGRAMS := $(NONTEST_PROG_SRC:programs/%.c=%$(PROG_SUFFIX)) -DEFAULT_TARGETS += $(NONTEST_PROGRAMS) -TEST_PROGRAMS := $(TEST_PROG_SRC:programs/%.c=%$(PROG_SUFFIX)) - -PROG_COMMON_OBJ := $(PROG_COMMON_SRC:%.c=%.o) -NONTEST_PROG_OBJ := $(NONTEST_PROG_SRC:%.c=%.o) -TEST_PROG_COMMON_OBJ := $(TEST_PROG_COMMON_SRC:%.c=%.o) -TEST_PROG_OBJ := $(TEST_PROG_SRC:%.c=%.o) - -ALL_PROG_OBJ := $(PROG_COMMON_OBJ) $(NONTEST_PROG_OBJ) \ - $(TEST_PROG_COMMON_OBJ) $(TEST_PROG_OBJ) - -# Generate autodetected configuration header -programs/config.h:scripts/detect.sh .build-config - $(QUIET_GEN) CC="$(CC)" CFLAGS="$(PROG_CFLAGS)" $< > $@ - -# Compile program object files -$(ALL_PROG_OBJ): %.o: %.c $(ALL_PROG_COMMON_HEADERS) $(COMMON_HEADERS) \ - .build-config - $(QUIET_CC) $(CC) -o $@ -c $(CPPFLAGS) $(PROG_CFLAGS) $< - -# Link the programs. -# -# Note: the test programs are not compiled by default. One reason is that the -# test programs must be linked with zlib for doing comparisons. - -ifdef USE_SHARED_LIB -LIB := $(SHARED_LIB) -else -LIB := $(STATIC_LIB) -endif - -$(NONTEST_PROGRAMS): %$(PROG_SUFFIX): programs/%.o $(PROG_COMMON_OBJ) $(LIB) - $(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(PROG_CFLAGS) $+ - -$(TEST_PROGRAMS): %$(PROG_SUFFIX): programs/%.o $(PROG_COMMON_OBJ) \ - $(TEST_PROG_COMMON_OBJ) $(LIB) - $(QUIET_CCLD) $(CC) -o $@ $(LDFLAGS) $(PROG_CFLAGS) $+ -lz - -ifdef HARD_LINKS -# Hard link gunzip to gzip -gunzip$(PROG_SUFFIX):gzip$(PROG_SUFFIX) - $(QUIET_LN) ln -f $< $@ -else -# No hard links; copy gzip to gunzip -gunzip$(PROG_SUFFIX):gzip$(PROG_SUFFIX) - $(QUIET_CP) cp -f $< $@ -endif - -DEFAULT_TARGETS += gunzip$(PROG_SUFFIX) - -############################################################################## - -all:$(DEFAULT_TARGETS) - -# We don't generate the pkg-config file until install time, as needed -# definitions might not exist until then. -PKGCONFBASE := libdeflate.pc.in - -# Install the files. Note: not all versions of the 'install' program have the -# '-D' and '-t' options, so don't use them; use portable commands only. -install:all $(PKGCONFBASE) - install -d $(DESTDIR)$(LIBDIR)/pkgconfig $(DESTDIR)$(INCDIR) \ - $(DESTDIR)$(BINDIR) - install -m644 $(STATIC_LIB) $(DESTDIR)$(LIBDIR) - if [ -z "$(DISABLE_SHARED)" ]; then \ - if [ -n "$(IMPLIB)" ]; then \ - install -m755 $(SHARED_LIB) $(DESTDIR)$(BINDIR);\ - install -m644 $(IMPLIB) $(DESTDIR)$(LIBDIR); \ - else \ - install -m755 $(SHARED_LIB) $(DESTDIR)$(LIBDIR);\ - fi \ - fi - sed -e "s|@LIBDIR@|$(LIBDIR)|" \ - -e "s|@INCDIR@|$(INCDIR)|" \ - -e "s|@VERSION@|$(VERSION)|" \ - $(PKGCONFBASE) > $(DESTDIR)$(LIBDIR)/pkgconfig/libdeflate.pc - chmod 644 $(DESTDIR)$(LIBDIR)/pkgconfig/libdeflate.pc - install -m644 libdeflate.h $(DESTDIR)$(INCDIR) - install -m755 gzip$(PROG_SUFFIX) \ - $(DESTDIR)$(BINDIR)/libdeflate-gzip$(PROG_SUFFIX) - if [ -n "$(HARD_LINKS)" ]; then \ - ln -f $(DESTDIR)$(BINDIR)/libdeflate-gzip$(PROG_SUFFIX) \ - $(DESTDIR)$(BINDIR)/libdeflate-gunzip$(PROG_SUFFIX); \ - else \ - ln -sf libdeflate-gzip$(PROG_SUFFIX) \ - $(DESTDIR)$(BINDIR)/libdeflate-gunzip$(PROG_SUFFIX); \ - fi - if [ -n "$(SHARED_LIB_SYMLINK)" ]; then \ - ln -sf $(SHARED_LIB) \ - $(DESTDIR)$(LIBDIR)/$(SHARED_LIB_SYMLINK); \ - fi - -uninstall: - rm -f $(DESTDIR)$(LIBDIR)/$(STATIC_LIB) \ - $(DESTDIR)$(LIBDIR)/pkgconfig/libdeflate.pc \ - $(DESTDIR)$(INCDIR)/libdeflate.h \ - $(DESTDIR)$(BINDIR)/libdeflate-gzip$(PROG_SUFFIX) \ - $(DESTDIR)$(BINDIR)/libdeflate-gunzip$(PROG_SUFFIX) - if [ -n "$(IMPLIB)" ]; then \ - rm -f $(DESTDIR)$(BINDIR)/$(SHARED_LIB) \ - $(DESTDIR)$(LIBDIR)/$(IMPLIB); \ - else \ - rm -f $(DESTDIR)$(LIBDIR)/$(SHARED_LIB); \ - fi - if [ -n "$(SHARED_LIB_SYMLINK)" ]; then \ - rm -f $(DESTDIR)$(LIBDIR)/$(SHARED_LIB_SYMLINK); \ - fi - -test_programs:$(TEST_PROGRAMS) - -# A minimal 'make check' target. This only runs some quick tests; -# use scripts/run_tests.sh if you want to run the full tests. -check:test_programs - ./scripts/check.sh - -# Run the clang static analyzer. -scan-build: - scan-build --status-bugs make all test_programs - -# Run shellcheck on all shell scripts. -shellcheck: - shellcheck scripts/*.sh - -help: - @echo "Available targets:" - @echo "------------------" - @for target in $(DEFAULT_TARGETS) $(TEST_PROGRAMS); do \ - echo -e "$$target"; \ - done - -clean: - rm -f *.a *.dll *.exe *.exp *.dylib *.so \ - lib/*.o lib/*/*.o \ - lib/*.obj lib/*/*.obj \ - lib/*.dllobj lib/*/*.dllobj \ - programs/*.o programs/*.obj \ - $(DEFAULT_TARGETS) $(TEST_PROGRAMS) programs/config.h \ - libdeflate.lib libdeflate.def libdeflatestatic.lib \ - .build-config - -realclean: clean - rm -f tags cscope* - -FORCE: - -.PHONY: all install uninstall test_programs check scan-build shellcheck help \ - clean realclean - -.DEFAULT_GOAL = all diff --git a/Makefile.msc b/Makefile.msc deleted file mode 100644 index 3ec8c1f1..00000000 --- a/Makefile.msc +++ /dev/null @@ -1,65 +0,0 @@ -# -# Makefile for the Microsoft toolchain -# -# Usage: -# nmake /f Makefile.msc -# - -.SUFFIXES: .c .obj .dllobj - -CC = cl -LD = link -AR = lib -CFLAGS = /MD /O2 -I. -LDFLAGS = - -STATIC_LIB = libdeflatestatic.lib -SHARED_LIB = libdeflate.dll -IMPORT_LIB = libdeflate.lib - -STATIC_LIB_OBJ = \ - lib/adler32.obj \ - lib/crc32.obj \ - lib/deflate_compress.obj \ - lib/deflate_decompress.obj \ - lib/gzip_compress.obj \ - lib/gzip_decompress.obj \ - lib/utils.obj \ - lib/x86/cpu_features.obj \ - lib/zlib_compress.obj \ - lib/zlib_decompress.obj - -SHARED_LIB_OBJ = $(STATIC_LIB_OBJ:.obj=.dllobj) - -PROG_COMMON_OBJ = programs/prog_util.obj \ - programs/tgetopt.obj \ - $(STATIC_LIB) - -PROG_CFLAGS = $(CFLAGS) -Iprograms - -all: $(STATIC_LIB) $(SHARED_LIB) $(IMPORT_LIB) gzip.exe gunzip.exe - -.c.obj: - $(CC) -c /Fo$@ $(CFLAGS) /DLIBDEFLATE_STATIC $** - -.c.dllobj: - $(CC) -c /Fo$@ $(CFLAGS) $** - -$(STATIC_LIB): $(STATIC_LIB_OBJ) - $(AR) $(ARFLAGS) -out:$@ $(STATIC_LIB_OBJ) - -$(SHARED_LIB): $(SHARED_LIB_OBJ) - $(LD) $(LDFLAGS) -out:$@ -dll -implib:$(IMPORT_LIB) $(SHARED_LIB_OBJ) - -$(IMPORT_LIB): $(SHARED_LIB) - -gzip.exe:programs/gzip.obj $(PROG_COMMON_OBJ) - $(LD) $(LDFLAGS) -out:$@ $** - -gunzip.exe:gzip.exe - copy $** $@ - -clean: - -del *.dll *.exe *.exp libdeflate.lib libdeflatestatic.lib gzip.lib \ - lib\*.obj lib\x86\*.obj lib\*.dllobj lib\x86\*.dllobj \ - programs\*.obj 2>nul diff --git a/README.md b/README.md index b2e27c54..583c1620 100644 --- a/README.md +++ b/README.md @@ -14,26 +14,29 @@ library, both for compression and decompression, and especially on x86 processors. In addition, libdeflate provides optional high compression modes that provide a better compression ratio than the zlib's "level 9". -libdeflate itself is a library, but the following command-line programs which -use this library are also provided: +libdeflate itself is a library. The following command-line programs which use +this library are also included: -* gzip (or gunzip), a program which mostly behaves like the standard equivalent, - except that it does not yet have good streaming support and therefore does not - yet support very large files -* benchmark, a program for benchmarking in-memory compression and decompression +* `libdeflate-gzip`, a program which can be a drop-in replacement for standard + `gzip` under some circumstances. Note that `libdeflate-gzip` has some + limitations; it is provided for convenience and is **not** meant to be the + main use case of libdeflate. It needs a lot of memory to process large files, + and it omits support for some infrequently-used options of GNU gzip. + +* `benchmark`, a test program that does round-trip compression and decompression + of the provided data, and measures the compression and decompression speed. + It can use libdeflate, zlib, or a combination of the two. + +* `checksum`, a test program that checksums the provided data with Adler-32 or + CRC-32, and optionally measures the speed. It can use libdeflate or zlib. For the release notes, see the [NEWS file](NEWS.md). ## Table of Contents - [Building](#building) - - [Using the Makefile](#using-the-makefile) - - [For UNIX](#for-unix) - - [For macOS](#for-macos) - - [For Windows](#for-windows) - - [Using Cygwin](#using-cygwin) - - [Using MSYS2](#using-msys2) - - [Using a custom build system](#using-a-custom-build-system) + - [Using CMake](#using-cmake) + - [Directly integrating the library sources](#directly-integrating-the-library-sources) - [API](#api) - [Bindings for other programming languages](#bindings-for-other-programming-languages) - [DEFLATE vs. zlib vs. gzip](#deflate-vs-zlib-vs-gzip) @@ -41,142 +44,34 @@ For the release notes, see the [NEWS file](NEWS.md). - [Motivation](#motivation) - [License](#license) - # Building -libdeflate and the provided programs like `gzip` can be built using the provided -Makefile. If only the library is needed, it can alternatively be easily -integrated into applications and built using any build system; see [Using a -custom build system](#using-a-custom-build-system). - -## Using the Makefile - -### For UNIX +## Using CMake -Just run `make`, then (if desired) `make install`. You need GNU Make and either -GCC or Clang. GCC is recommended because it builds slightly faster binaries. +libdeflate uses [CMake](https://cmake.org/). It can be built just like any +other CMake project, e.g. with: -By default, the following targets are built: the static library `libdeflate.a`, -the shared library `libdeflate.so`, the `gzip` program, and the `gunzip` program -(which is actually just a hard link to `gzip`). Benchmarking and test programs -such as `benchmark` are not built by default. You can run `make help` to -display the available build targets. + cmake -B build && cmake --build build -There are also many options which can be set on the `make` command line, e.g. to -omit library features or to customize the directories into which `make install` -installs files. See the Makefile for details. +By default the following targets are built: -### For macOS +- The static library (normally called `libdeflate.a`) +- The shared library (normally called `libdeflate.so`) +- The `libdeflate-gzip` program, including its alias `libdeflate-gunzip` -Prebuilt macOS binaries can be installed with [Homebrew](https://brew.sh): - - brew install libdeflate - -But if you need to build the binaries yourself, see the section for UNIX above. - -### For Windows +Besides the standard CMake build and installation options, there are some +libdeflate-specific build options. See `CMakeLists.txt` for the list of these +options. To set an option, add `-DOPTION=VALUE` to the `cmake` command. Prebuilt Windows binaries can be downloaded from -https://github.com/ebiggers/libdeflate/releases. But if you need to build the -binaries yourself, MinGW (gcc) is the recommended compiler to use. If you're -performing the build *on* Windows (as opposed to cross-compiling for Windows on -Linux, for example), you'll need to follow the directions in **one** of the two -sections below to set up a minimal UNIX-compatible environment using either -Cygwin or MSYS2, then do the build. (Other MinGW distributions may not work, as -they often omit basic UNIX tools such as `sh`.) - -Alternatively, libdeflate may be built using the Visual Studio toolchain by -running `nmake /f Makefile.msc`. However, while this is supported in the sense -that it will produce working binaries, it is not recommended because the -binaries built with MinGW will be significantly faster. - -Also note that 64-bit binaries are faster than 32-bit binaries and should be -preferred whenever possible. - -#### Using Cygwin - -Run the Cygwin installer, available from https://cygwin.com/setup-x86_64.exe. -When you get to the package selection screen, choose the following additional -packages from category "Devel": - -- git -- make -- mingw64-i686-binutils -- mingw64-i686-gcc-g++ -- mingw64-x86_64-binutils -- mingw64-x86_64-gcc-g++ - -(You may skip the mingw64-i686 packages if you don't need to build 32-bit -binaries.) - -After the installation finishes, open a Cygwin terminal. Then download -libdeflate's source code (if you haven't already) and `cd` into its directory: - - git clone https://github.com/ebiggers/libdeflate - cd libdeflate - -(Note that it's not required to use `git`; an alternative is to extract a .zip -or .tar.gz archive of the source code downloaded from the releases page. -Also, in case you need to find it in the file browser, note that your home -directory in Cygwin is usually located at `C:\cygwin64\home\`.) - -Then, to build 64-bit binaries: - - make CC=x86_64-w64-mingw32-gcc - -or to build 32-bit binaries: - - make CC=i686-w64-mingw32-gcc - -#### Using MSYS2 - -Run the MSYS2 installer, available from http://www.msys2.org/. After -installing, open an MSYS2 shell and run: - - pacman -Syu - -Say `y`, then when it's finished, close the shell window and open a new one. -Then run the same command again: - - pacman -Syu - -Then, install the packages needed to build libdeflate: - - pacman -S git \ - make \ - mingw-w64-i686-binutils \ - mingw-w64-i686-gcc \ - mingw-w64-x86_64-binutils \ - mingw-w64-x86_64-gcc - -(You may skip the mingw-w64-i686 packages if you don't need to build 32-bit -binaries.) - -Then download libdeflate's source code (if you haven't already): - - git clone https://github.com/ebiggers/libdeflate - -(Note that it's not required to use `git`; an alternative is to extract a .zip -or .tar.gz archive of the source code downloaded from the releases page. -Also, in case you need to find it in the file browser, note that your home -directory in MSYS2 is usually located at `C:\msys64\home\`.) - -Then, to build 64-bit binaries, open "MSYS2 MinGW 64-bit" from the Start menu -and run the following commands: - - cd libdeflate - make clean - make - -Or to build 32-bit binaries, do the same but use "MSYS2 MinGW 32-bit" instead. +https://github.com/ebiggers/libdeflate/releases. -## Using a custom build system +## Directly integrating the library sources -The source files of the library are designed to be compilable directly, without -any prerequisite step like running a `./configure` script. Therefore, as an -alternative to building the library using the provided Makefile, the library -source files can be easily integrated directly into your application and built -using any build system. +Although the official build system is CMake, care has been taken to keep the +library source files compilable directly, without a prerequisite configuration +step. Therefore, it is also fine to just add the library source files directly +to your application, without using CMake. You should compile both `lib/*.c` and `lib/*/*.c`. You don't need to worry about excluding irrelevant architecture-specific code, as this is already @@ -185,7 +80,7 @@ handled in the source files themselves using `#ifdef`s. It is **strongly** recommended to use either gcc or clang, and to use `-O2`. If you are doing a freestanding build with `-ffreestanding`, you must add -`-DFREESTANDING` as well, otherwise performance will suffer greatly. +`-DFREESTANDING` as well (matching what the `CMakeLists.txt` does). # API diff --git a/common_defs.h b/common_defs.h index cfe6fd62..4f6f3965 100644 --- a/common_defs.h +++ b/common_defs.h @@ -111,15 +111,6 @@ typedef size_t machine_word_t; # define __has_builtin(builtin) 0 #endif -/* LIBEXPORT - export a function from a shared library */ -#ifdef _WIN32 -# define LIBEXPORT __declspec(dllexport) -#elif defined(__GNUC__) -# define LIBEXPORT __attribute__((visibility("default"))) -#else -# define LIBEXPORT -#endif - /* inline - suggest that a function be inlined */ #ifdef _MSC_VER # define inline __inline diff --git a/libdeflate-config.cmake.in b/libdeflate-config.cmake.in new file mode 100644 index 00000000..747799df --- /dev/null +++ b/libdeflate-config.cmake.in @@ -0,0 +1,3 @@ +@PACKAGE_INIT@ + +include("${CMAKE_CURRENT_LIST_DIR}/libdeflate-targets.cmake") diff --git a/libdeflate.pc.in b/libdeflate.pc.in index 1647c9ee..a8b39c1b 100644 --- a/libdeflate.pc.in +++ b/libdeflate.pc.in @@ -1,6 +1,6 @@ Name: libdeflate Description: Fast implementation of DEFLATE, zlib, and gzip -Version: @VERSION@ -Libs: -L@LIBDIR@ -ldeflate -Cflags: -I@INCDIR@ +Version: ${PROJECT_VERSION} +Libs: -L${CMAKE_INSTALL_FULL_LIBDIR} -ldeflate +Cflags: -I${CMAKE_INSTALL_FULL_INCLUDEDIR} Cflags.private: -DLIBDEFLATE_STATIC diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt new file mode 100644 index 00000000..597cf1a4 --- /dev/null +++ b/programs/CMakeLists.txt @@ -0,0 +1,95 @@ +include(CheckSymbolExists) + +# Check for the availability of OS functionality and generate the config.h file. +check_symbol_exists(clock_gettime "time.h" HAVE_CLOCK_GETTIME) +check_symbol_exists(futimens "fcntl.h;sys/stat.h" HAVE_FUTIMENS) +check_symbol_exists(futimes "sys/time.h" HAVE_FUTIMES) +check_symbol_exists(posix_fadvise "fcntl.h" HAVE_POSIX_FADVISE) +check_symbol_exists(posix_madvise "sys/mman.h" HAVE_POSIX_MADVISE) +check_c_source_compiles("#include + #include + int main() { struct stat st; (void)st.st_atim; }" + HAVE_STAT_NANOSECOND_PRECISION) +configure_file(config.h.in config.h) + +# Build a utility library for the programs. This library is not installed. +add_library(libdeflate_prog_utils STATIC prog_util.c tgetopt.c) +set_target_properties(libdeflate_prog_utils PROPERTIES + OUTPUT_NAME deflate_prog_utils) +if(LIBDEFLATE_USE_SHARED_LIB) + target_link_libraries(libdeflate_prog_utils PUBLIC libdeflate_shared) +else() + target_link_libraries(libdeflate_prog_utils PUBLIC libdeflate_static) +endif() +target_include_directories(libdeflate_prog_utils PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) +target_compile_definitions(libdeflate_prog_utils PUBLIC HAVE_CONFIG_H) +if(WIN32) + if(MINGW) + target_compile_options(libdeflate_prog_utils PUBLIC -municode) + target_link_libraries(libdeflate_prog_utils PUBLIC -municode) + else() + target_compile_definitions(libdeflate_prog_utils PUBLIC UNICODE _UNICODE) + endif() +endif() +if(NOT MSVC) + target_compile_definitions(libdeflate_prog_utils PUBLIC + _POSIX_C_SOURCE=200809L _FILE_OFFSET_BITS=64) +endif() + +# Build and install libdeflate-gzip and its alias libdeflate-gunzip. +if(LIBDEFLATE_BUILD_GZIP) + add_executable(libdeflate-gzip gzip.c) + target_link_libraries(libdeflate-gzip PRIVATE libdeflate_prog_utils) + install(TARGETS libdeflate-gzip DESTINATION ${CMAKE_INSTALL_BINDIR}) + if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.14") + # Install libdeflate-gunzip as a hard link to libdeflate-gzip. + # Fall back to a copy if hard links are unsupported. + set(GZIP ${CMAKE_INSTALL_FULL_BINDIR}/libdeflate-gzip) + set(GUNZIP ${CMAKE_INSTALL_FULL_BINDIR}/libdeflate-gunzip) + install(CODE "message(\"-- Installing: \$ENV{DESTDIR}${GUNZIP}\")") + install(CODE "file(CREATE_LINK \$ENV{DESTDIR}${GZIP} + \$ENV{DESTDIR}${GUNZIP})" COPY_ON_ERROR) + else() + # The cmake version is too old to support file(CREATE_LINK). + # Just compile gzip.c again to build libdeflate-gunzip. + add_executable(libdeflate-gunzip gzip.c) + target_link_libraries(libdeflate-gunzip PRIVATE libdeflate_prog_utils) + install(TARGETS libdeflate-gunzip DESTINATION ${CMAKE_INSTALL_BINDIR}) + endif() +endif() + +# Build the test programs, if requested. +if(LIBDEFLATE_BUILD_TESTS) + + # The test programs depend on zlib for comparison tests. + find_package(ZLIB REQUIRED) + + # Build a utility library for the test programs. + add_library(libdeflate_test_utils STATIC test_util.c) + set_target_properties(libdeflate_test_utils PROPERTIES + OUTPUT_NAME deflate_test_utils) + target_link_libraries(libdeflate_test_utils PUBLIC + libdeflate_prog_utils ZLIB::ZLIB) + + # Build the benchmark and checksum programs. + add_executable(benchmark benchmark.c) + target_link_libraries(benchmark PRIVATE libdeflate_test_utils) + add_executable(checksum checksum.c) + target_link_libraries(checksum PRIVATE libdeflate_test_utils) + + # Build the unit test programs and register them with CTest. + set(UNIT_TEST_PROGS + test_checksums + test_custom_malloc + test_incomplete_codes + test_litrunlen_overflow + test_overread + test_slow_decompression + test_trailing_bytes + ) + foreach(PROG ${UNIT_TEST_PROGS}) + add_executable(${PROG} ${PROG}.c) + target_link_libraries(${PROG} PRIVATE libdeflate_test_utils) + add_test(NAME ${PROG} COMMAND ${PROG}) + endforeach() +endif() diff --git a/programs/config.h.in b/programs/config.h.in new file mode 100644 index 00000000..588aa8dc --- /dev/null +++ b/programs/config.h.in @@ -0,0 +1,22 @@ +#ifndef CONFIG_H +#define CONFIG_H + +/* Is the clock_gettime() function available? */ +#cmakedefine HAVE_CLOCK_GETTIME + +/* Is the futimens() function available? */ +#cmakedefine HAVE_FUTIMENS + +/* Is the futimes() function available? */ +#cmakedefine HAVE_FUTIMES + +/* Is the posix_fadvise() function available? */ +#cmakedefine HAVE_POSIX_FADVISE + +/* Is the posix_madvise() function available? */ +#cmakedefine HAVE_POSIX_MADVISE + +/* Does stat() provide nanosecond-precision timestamps? */ +#cmakedefine HAVE_STAT_NANOSECOND_PRECISION + +#endif /* CONFIG_H */ diff --git a/programs/gzip.c b/programs/gzip.c index a3d7ba63..932a3408 100644 --- a/programs/gzip.c +++ b/programs/gzip.c @@ -352,7 +352,13 @@ restore_timestamps(struct file_stream *out, const tchar *newpath, const stat_t *stbuf) { int ret; -#if defined(HAVE_FUTIMENS) && defined(HAVE_STAT_NANOSECOND_PRECISION) +#ifdef __APPLE__ + struct timespec times[2] = { + { stbuf->st_atime, stbuf->st_atimensec }, + { stbuf->st_mtime, stbuf->st_mtimensec }, + }; + ret = futimens(out->fd, times); +#elif defined(HAVE_FUTIMENS) && defined(HAVE_STAT_NANOSECOND_PRECISION) struct timespec times[2] = { stbuf->st_atim, stbuf->st_mtim, }; diff --git a/scripts/afl-fuzz/.gitignore b/scripts/afl-fuzz/.gitignore deleted file mode 100644 index 24c8d6ce..00000000 --- a/scripts/afl-fuzz/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*/fuzz diff --git a/scripts/afl-fuzz/Makefile b/scripts/afl-fuzz/Makefile deleted file mode 100644 index 0fc87d57..00000000 --- a/scripts/afl-fuzz/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -SRC := $(wildcard */*.c) -EXE := $(SRC:.c=) - -LDLIBS := -ldeflate -LDFLAGS := -L../.. -CPPFLAGS := -I../.. - -all:$(EXE) - -clean: - rm -f $(EXE) diff --git a/scripts/afl-fuzz/fuzz.sh b/scripts/afl-fuzz/fuzz.sh index f220d4c0..c78a2ca0 100755 --- a/scripts/afl-fuzz/fuzz.sh +++ b/scripts/afl-fuzz/fuzz.sh @@ -114,7 +114,7 @@ elif $ubsan; then else export AFL_HARDEN=1 export CFLAGS="-O2" - export CC=afl-clang-fast + export CC=afl-gcc fi CFLAGS+=" -DLIBDEFLATE_ENABLE_ASSERTIONS" @@ -123,13 +123,16 @@ if [ -e /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor ]; then sudo sh -c "echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor" fi -NPROC=$(getconf _NPROCESSORS_ONLN) - -make "-j$NPROC" -C ../../ libdeflate.a -make "-j$NPROC" -B +srcdir=../.. +builddir=$srcdir/build +$srcdir/scripts/cmake-helper.sh -G Ninja +cmake --build $builddir for dir in "${targets[@]}"; do cp -vaT "$dir" "/tmp/$dir" + # shellcheck disable=SC2086 # Intended word splitting of $CFLAGS + $CC $CFLAGS -Wall -I$srcdir "$dir"/fuzz.c $builddir/libdeflate.a \ + -o "/tmp/$dir/fuzz" indir=/tmp/$dir/inputs outdir=/tmp/$dir/outputs if [ -e "$outdir" ]; then diff --git a/scripts/android_build.sh b/scripts/android_build.sh index 204ead2d..f61e4522 100755 --- a/scripts/android_build.sh +++ b/scripts/android_build.sh @@ -2,12 +2,14 @@ set -eu -o pipefail +SCRIPTDIR="$(dirname "$0")" +BUILDDIR="$SCRIPTDIR/../build" API_LEVEL=28 ARCH=arm64 CFLAGS= ENABLE_CRC=false ENABLE_CRYPTO=false -NDKDIR=$HOME/android-ndk-r21d +NDKDIR=$HOME/android-ndk-r23b usage() { cat << EOF @@ -15,7 +17,7 @@ Usage: $0 [OPTION]... -- [MAKE_TARGET]... Build libdeflate for Android. --api-level=LEVEL Android API level to target (default: $API_LEVEL) - --arch=ARCH Architecture: arm32|arm64 (default: $ARCH) + --arch=ARCH Architecture: arm32|arm64|x86|x86_64 (default: $ARCH) --enable-crc Enable crc instructions --enable-crypto Enable crypto instructions --ndkdir=NDKDIR Android NDK directory (default: $NDKDIR) @@ -65,13 +67,11 @@ while [ $# -gt 0 ]; do shift done -BINDIR=$NDKDIR/toolchains/llvm/prebuilt/linux-x86_64/bin/ - case "$ARCH" in -arm|arm32|aarch32) - CC=$BINDIR/armv7a-linux-androideabi$API_LEVEL-clang +arm|arm32|aarch32|armeabi-v7a) + ANDROID_ABI=armeabi-v7a if $ENABLE_CRC || $ENABLE_CRYPTO; then - CFLAGS="-march=armv8-a" + export CFLAGS="-march=armv8-a" if $ENABLE_CRC; then CFLAGS+=" -mcrc" else @@ -84,8 +84,8 @@ arm|arm32|aarch32) fi fi ;; -arm64|aarch64) - CC=$BINDIR/aarch64-linux-android$API_LEVEL-clang +arm64|aarch64|arm64-v8a) + ANDROID_ABI=arm64-v8a features="" if $ENABLE_CRC; then features+="+crc" @@ -94,15 +94,24 @@ arm64|aarch64) features+="+crypto" fi if [ -n "$features" ]; then - CFLAGS="-march=armv8-a$features" + export CFLAGS="-march=armv8-a$features" fi ;; +x86) + ANDROID_ABI=x86 + ;; +x86_64) + ANDROID_ABI=x86_64 + ;; *) echo 1>&2 "Unknown architecture: \"$ARCH\"" usage 1>&2 exit 1 esac -cmd=(make "-j$(grep -c processor /proc/cpuinfo)" "CC=$CC" "CFLAGS=$CFLAGS" "$@") -echo "${cmd[*]}" -"${cmd[@]}" +"$SCRIPTDIR"/cmake-helper.sh -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE="$NDKDIR"/build/cmake/android.toolchain.cmake \ + -DANDROID_ABI="$ANDROID_ABI" \ + -DANDROID_PLATFORM="$API_LEVEL" \ + -DLIBDEFLATE_BUILD_TESTS=1 +cmake --build "$BUILDDIR" diff --git a/scripts/android_tests.sh b/scripts/android_tests.sh index fe70ce90..80413134 100755 --- a/scripts/android_tests.sh +++ b/scripts/android_tests.sh @@ -12,7 +12,7 @@ if [ $# -ne 0 ]; then fi # Use NDKDIR if specified in environment, else use default value. -: "${NDKDIR:=$HOME/android-ndk-r21d}" +: "${NDKDIR:=$HOME/android-ndk-r23b}" if [ ! -e "$NDKDIR" ]; then cat 1>&2 << EOF Android NDK was not found in NDKDIR=$NDKDIR! Set the @@ -45,10 +45,9 @@ CLEANUP_CMDS+=("rm -r '$TMPDIR'") android_build_and_test() { echo "Running Android tests with $*" - ./scripts/android_build.sh --ndkdir="$NDKDIR" "$@" \ - all test_programs > /dev/null - adb push "$TESTDATA" ./scripts/exec_tests.sh benchmark test_* \ - /data/local/tmp/ > /dev/null + ./scripts/android_build.sh --ndkdir="$NDKDIR" "$@" > /dev/null + adb push "$TESTDATA" ./scripts/exec_tests.sh \ + ./build/programs/{benchmark,test_*} /data/local/tmp/ > /dev/null # Note: adb shell always returns 0, even if the shell command fails... adb shell "cd /data/local/tmp && WRAPPER= TESTDATA=$(basename "$TESTDATA") sh exec_tests.sh" \ diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh new file mode 100755 index 00000000..aa933caf --- /dev/null +++ b/scripts/benchmark.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +SCRIPTDIR="$(dirname "$0")" +BUILDDIR="$SCRIPTDIR/../build" + +"$SCRIPTDIR"/cmake-helper.sh -DLIBDEFLATE_BUILD_TESTS=1 -G Ninja +ninja -C "$BUILDDIR" benchmark +"$BUILDDIR"/programs/benchmark "$@" diff --git a/scripts/check.sh b/scripts/check.sh deleted file mode 100755 index 100a1af4..00000000 --- a/scripts/check.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -# A minimal 'make check' target. This only runs some quick tests; -# use scripts/run_tests.sh if you want to run the full tests. - -set -e -u - -if [ "$(uname)" = Darwin ]; then - export DYLD_FALLBACK_LIBRARY_PATH=. -else - export LD_LIBRARY_PATH=. -fi -cat lib/*.c | ./benchmark > /dev/null -cat lib/*.c | ./benchmark -C libz > /dev/null -cat lib/*.c | ./benchmark -D libz > /dev/null -for prog in ./test_*; do - "$prog" -done diff --git a/scripts/checksum.sh b/scripts/checksum.sh new file mode 100755 index 00000000..35af5731 --- /dev/null +++ b/scripts/checksum.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +SCRIPTDIR="$(dirname "$0")" +BUILDDIR="$SCRIPTDIR/../build" + +"$SCRIPTDIR"/cmake-helper.sh -DLIBDEFLATE_BUILD_TESTS=1 -G Ninja +ninja -C "$BUILDDIR" checksum +"$BUILDDIR"/programs/checksum "$@" diff --git a/scripts/checksum_benchmarks.sh b/scripts/checksum_benchmarks.sh index 5aafa22f..83667608 100755 --- a/scripts/checksum_benchmarks.sh +++ b/scripts/checksum_benchmarks.sh @@ -19,11 +19,14 @@ have_cpu_feature() { make_and_test() { # Build the checksum program and tests. Set the special test support # flag to get support for LIBDEFLATE_DISABLE_CPU_FEATURES. - make "$@" TEST_SUPPORT__DO_NOT_USE=1 checksum test_checksums > /dev/null + rm -rf build + CFLAGS="$CFLAGS -DTEST_SUPPORT__DO_NOT_USE=1" \ + cmake -B build -G Ninja -DLIBDEFLATE_BUILD_TESTS=1 > /dev/null + cmake --build build > /dev/null # Run the checksum tests, for good measure. (This isn't actually part # of the benchmarking.) - ./test_checksums > /dev/null + ./build/programs/test_checksums > /dev/null } __do_benchmark() { @@ -31,7 +34,8 @@ __do_benchmark() { shift local flags=("$@") - speed=$(./checksum "${CKSUM_FLAGS[@]}" "${flags[@]}" -t "$FILE" | \ + speed=$(./build/programs/checksum "${CKSUM_FLAGS[@]}" \ + "${flags[@]}" -t "$FILE" | \ grep -o '[0-9]\+ MB/s' | grep -o '[0-9]\+') printf "%-45s%-10s\n" "$CKSUM_NAME ($impl)" "$speed" } @@ -42,10 +46,10 @@ do_benchmark() { if [ "$impl" = zlib ]; then __do_benchmark "$impl" "-Z" else - make_and_test CFLAGS="${EXTRA_CFLAGS[*]}" + CFLAGS="${EXTRA_CFLAGS[*]}" make_and_test __do_benchmark "libdeflate, $impl" if [ "$ARCH" = x86_64 ]; then - make_and_test CFLAGS="-m32 ${EXTRA_CFLAGS[*]}" + CFLAGS="-m32 ${EXTRA_CFLAGS[*]}" make_and_test __do_benchmark "libdeflate, $impl, 32-bit" fi fi diff --git a/scripts/cmake-helper.sh b/scripts/cmake-helper.sh new file mode 100755 index 00000000..0c67930b --- /dev/null +++ b/scripts/cmake-helper.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# This script ensures that the 'build' directory has been created and configured +# with the given CMake options and environment. + +TOPDIR="$(dirname "$0")"/.. +BUILDDIR="$TOPDIR"/build + +flags=$(env; echo "@CMAKEOPTS@=$*") +if [ "$flags" != "$(cat "$BUILDDIR"/.flags 2>/dev/null)" ]; then + rm -rf "$BUILDDIR"/CMakeCache.txt "$BUILDDIR"/CMakeFiles + mkdir -p "$BUILDDIR" + cmake -S "$TOPDIR" -B "$BUILDDIR" "$@" + echo "$flags" > "$BUILDDIR"/.flags +fi diff --git a/scripts/deflate_benchmarks.sh b/scripts/deflate_benchmarks.sh index 1d2f633d..5321cdc3 100755 --- a/scripts/deflate_benchmarks.sh +++ b/scripts/deflate_benchmarks.sh @@ -35,7 +35,8 @@ multifile() for file in "$@"; do echo -n "$(basename "$file")" results=() - cmd=("$topdir/benchmark" -s"$(stat -c "%s" "$file")" "$file") + cmd=("$topdir/build/programs/benchmark" + -s"$(stat -c "%s" "$file")" "$file") run_benchmark "${cmd[@]}" -Y -6 results+=("$CSIZE") run_benchmark "${cmd[@]}" -Y -6 @@ -90,7 +91,7 @@ single_file() echo -n "$level" args=("$file" -s "$usize" "-$level") - run_benchmark "$topdir/benchmark" "${args[@]}" + run_benchmark "$topdir/build/programs/benchmark" "${args[@]}" echo -n " | $CSIZE / $CTIME" if $include_old; then @@ -101,7 +102,8 @@ single_file() if (( level > 9 )); then echo -n " | N/A" else - run_benchmark "$topdir/benchmark" "${args[@]}" -Y + run_benchmark "$topdir/build/programs/benchmark" \ + "${args[@]}" -Y echo -n " | $CSIZE / $CTIME" fi echo diff --git a/scripts/detect.sh b/scripts/detect.sh deleted file mode 100755 index 93064b8b..00000000 --- a/scripts/detect.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/sh - -set -eu - -# Use CC if specified in environment, else default to "cc". -: "${CC:=cc}" - -# Use CFLAGS if specified in environment. -: "${CFLAGS:=}" - -echo "/* THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT. */" -echo "#ifndef CONFIG_H" -echo "#define CONFIG_H" - -program_compiles() { - echo "$1" | $CC $CFLAGS -Wno-error -x c - -o /dev/null > /dev/null 2>&1 -} - -check_function() { - funcname=$1 - macro="HAVE_$(echo "$funcname" | tr '[:lower:]' '[:upper:]')" - - echo - echo "/* Is the $funcname() function available? */" - if program_compiles "int main() { $funcname(); }"; then - echo "#define $macro 1" - else - echo "/* $macro is not set */" - fi -} - -have_stat_field() { - program_compiles "#include - #include - int main() { struct stat st; st.$1; }" -} - -check_stat_nanosecond_precision() { - echo - echo "/* Does stat() provide nanosecond-precision timestamps? */" - if have_stat_field st_atim; then - echo "#define HAVE_STAT_NANOSECOND_PRECISION 1" - elif have_stat_field st_atimespec; then - # Nonstandard field names used by OS X and older BSDs - echo "#define HAVE_STAT_NANOSECOND_PRECISION 1" - echo "#define st_atim st_atimespec" - echo "#define st_mtim st_mtimespec" - echo "#define st_ctim st_ctimespec" - else - echo "/* HAVE_STAT_NANOSECOND_PRECISION is not set */" - fi -} - -check_function clock_gettime -check_function futimens -check_function futimes -check_function posix_fadvise -check_function posix_madvise - -check_stat_nanosecond_precision - -echo -echo "#endif /* CONFIG_H */" diff --git a/scripts/exec_tests.sh b/scripts/exec_tests.sh index c748e423..b4ad2d5b 100644 --- a/scripts/exec_tests.sh +++ b/scripts/exec_tests.sh @@ -6,6 +6,10 @@ set -eu +DIR=${1:-.} + +cd "$DIR" + run_cmd() { echo "$WRAPPER $*" $WRAPPER "$@" > /dev/null diff --git a/scripts/make-windows-releases.sh b/scripts/make-windows-releases.sh index cc463ed2..332f42c5 100755 --- a/scripts/make-windows-releases.sh +++ b/scripts/make-windows-releases.sh @@ -3,14 +3,17 @@ set -eu -o pipefail for arch in 'i686' 'x86_64'; do - make clean - make -j CC=${arch}-w64-mingw32-gcc CFLAGS="-Werror" all \ - benchmark.exe checksum.exe dir=libdeflate-$(git describe --tags | tr -d v)-windows-${arch}-bin - rm -rf "$dir" "$dir.zip" + rm -rf build "$dir" "$dir.zip" + CFLAGS="-Werror" ${arch}-w64-mingw32-cmake -B build -G Ninja \ + -DLIBDEFLATE_BUILD_TESTS=1 + cmake --build build mkdir "$dir" - cp libdeflate.{dll,dll.a,def,a} libdeflate.h ./*.exe "$dir" - ${arch}-w64-mingw32-strip "$dir/libdeflate.dll" "$dir"/*.exe + cp libdeflate.h build/libdeflate.{dll,dll.a,a} \ + build/programs/{benchmark,checksum}.exe "$dir" + cp build/programs/libdeflate-gzip.exe "$dir"/gzip.exe + cp build/programs/libdeflate-gzip.exe "$dir"/gunzip.exe + ${arch}-w64-mingw32-strip "$dir"/libdeflate.dll "$dir"/*.exe for file in COPYING NEWS.md README.md; do sed < $file > "$dir/${file}.txt" -e 's/$/\r/g' done diff --git a/scripts/pgo_build.sh b/scripts/pgo_build.sh deleted file mode 100755 index 2eb2b231..00000000 --- a/scripts/pgo_build.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Try gcc profile-guided optimizations - -set -eu - -MAKE="make -j$(grep -c processor /proc/cpuinfo)" -DATAFILE="$HOME/data/silesia" - -$MAKE benchmark > /dev/null -echo "=====================" -echo "Original performance:" -echo "---------------------" -./benchmark "$@" "$DATAFILE" - -$MAKE CFLAGS=-fprofile-generate LDFLAGS=-fprofile-generate benchmark > /dev/null -./benchmark "$@" "$DATAFILE" > /dev/null -$MAKE CFLAGS=-fprofile-use benchmark > /dev/null -rm -f {lib,programs}/*.gcda -echo "==========================" -echo "PGO-optimized performance:" -echo "--------------------------" -./benchmark "$@" "$DATAFILE" diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index d3e8f222..963e34d5 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -44,9 +44,9 @@ CC_VERSION=$($CC --version | head -1) UNAME=$(uname) ARCH=$(uname -m) -SHLIB=libdeflate.so +SHLIB=build/libdeflate.so if [ "$UNAME" = Darwin ]; then - SHLIB=libdeflate.dylib + SHLIB=build/libdeflate.dylib fi for skip in SKIP_FREESTANDING SKIP_VALGRIND SKIP_UBSAN SKIP_ASAN SKIP_CFI \ @@ -115,6 +115,14 @@ valgrind_version_at_least() { [ "$want_vers" = "$(echo -e "$vers\n$want_vers" | sort -V | head -1)" ] } +# Build libdeflate, including the test programs. Set the special test support +# flag to get support for LIBDEFLATE_DISABLE_CPU_FEATURES. +build() { + CFLAGS="$CFLAGS -DTEST_SUPPORT__DO_NOT_USE=1" scripts/cmake-helper.sh \ + -DLIBDEFLATE_BUILD_TESTS=1 "$@" > /dev/null + $MAKE -C build > /dev/null +} + build_and_run_tests() { local quick=false if [ "${1:-}" = "--quick" ]; then @@ -124,9 +132,7 @@ build_and_run_tests() { begin "CC=$CC CFLAGS=\"$CFLAGS\" WRAPPER=\"$WRAPPER\" $*" - # Build libdeflate, including the test programs. Set the special test - # support flag to get support for LIBDEFLATE_DISABLE_CPU_FEATURES. - $MAKE "$@" TEST_SUPPORT__DO_NOT_USE=1 all test_programs > /dev/null + build "$@" # When not using -march=native, run the tests multiple times with # different combinations of CPU features disabled. This is needed to @@ -158,7 +164,7 @@ build_and_run_tests() { fi log "Using LIBDEFLATE_DISABLE_CPU_FEATURES=$disable_str" LIBDEFLATE_DISABLE_CPU_FEATURES="$disable_str" \ - sh ./scripts/exec_tests.sh > /dev/null + sh ./scripts/exec_tests.sh build/programs/ > /dev/null done end } @@ -169,15 +175,15 @@ verify_freestanding_build() { return 0 fi log "Verifying that freestanding build is really freestanding" - if nm libdeflate.so | grep -v '\<__stack_chk_fail\>' | grep -q ' U ' - then + build -DLIBDEFLATE_FREESTANDING=1 + if nm $SHLIB | grep -v '\<__stack_chk_fail\>' | grep -q ' U '; then echo 1>&2 "Freestanding lib links to external functions!:" - nm libdeflate.so | grep ' U ' + nm $SHLIB | grep ' U ' return 1 fi - if ldd libdeflate.so | grep -q -v '\'; then + if ldd $SHLIB | grep -q -v '\'; then echo 1>&2 "Freestanding lib links to external libraries!:" - ldd libdeflate.so + ldd $SHLIB return 1 fi } @@ -202,8 +208,8 @@ is_compatible_system_gzip() { } gzip_tests() { - local gzips=("$PWD/gzip") - local gunzips=("$PWD/gunzip") + local gzips=("$PWD/build/programs/libdeflate-gzip") + local gunzips=("$PWD/build/programs/libdeflate-gzip -d") if [ "${1:-}" != "--quick" ]; then if is_compatible_system_gzip /bin/gzip; then gzips+=(/bin/gzip) @@ -218,7 +224,7 @@ gzip_tests() { local gzip gunzip begin "Running gzip program tests with CC=\"$CC\" CFLAGS=\"$CFLAGS\"" - $MAKE gzip gunzip > /dev/null + build for gzip in "${gzips[@]}"; do for gunzip in "${gunzips[@]}"; do log "GZIP=$gzip, GUNZIP=$gunzip" @@ -237,7 +243,7 @@ do_run_tests() { elif [ "$UNAME" = Darwin ]; then log "Skipping freestanding build tests due to unsupported OS" else - build_and_run_tests FREESTANDING=1 + build_and_run_tests -DLIBDEFLATE_FREESTANDING=1 verify_freestanding_build fi fi @@ -245,13 +251,13 @@ do_run_tests() { } check_symbol_prefixes() { + build log "Checking that all global symbols are prefixed with \"libdeflate_\"" - $MAKE libdeflate.a > /dev/null - if nm libdeflate.a | grep ' T ' | grep -E -v " _?libdeflate_"; then + if nm build/libdeflate.a | grep ' T ' | grep -E -v " _?libdeflate_" + then fail "Some global symbols aren't prefixed with \"libdeflate_\"" fi log "Checking that all exported symbols are prefixed with \"libdeflate\"" - $MAKE $SHLIB > /dev/null if nm $SHLIB | grep ' T ' \ | grep -E -v " _?(libdeflate_|_init\>|_fini\>)"; then fail "Some exported symbols aren't prefixed with \"libdeflate_\"" @@ -274,34 +280,20 @@ test_use_shared_lib() { return fi log "Testing USE_SHARED_LIB=1" - $MAKE gzip > /dev/null - if is_dynamically_linked gzip; then + build + if is_dynamically_linked build/programs/libdeflate-gzip; then fail "Binary should be statically linked by default" fi - $MAKE USE_SHARED_LIB=1 all check > /dev/null - if ! is_dynamically_linked gzip; then + build -DLIBDEFLATE_USE_SHARED_LIB=1 > /dev/null + if ! is_dynamically_linked build/programs/libdeflate-gzip; then fail "Binary isn't dynamically linked" fi } -install_uninstall_tests() { - local shell - - begin "Testing 'make install' and 'make uninstall'" - for shell in '/bin/bash' '/bin/dash'; do - log "Trying SHELL=$shell" - $MAKE SHELL=$shell clean > /dev/null - $MAKE SHELL=$shell DESTDIR="$TMPDIR/inst" install > /dev/null - if (( "$(file_count "$TMPDIR/inst")" == 0 )); then - fail "'make install' didn't install any files" - fi - make SHELL=$shell DESTDIR="$TMPDIR/inst" uninstall > /dev/null - if (( "$(file_count "$TMPDIR/inst")" != 0 )); then - fail "'make uninstall' didn't uninstall all files" - fi - rm -r "$TMPDIR/inst" - done - end +install_tests() { + begin "Testing install" + build + $MAKE -C build install DESTDIR=inst } run_tests() { @@ -365,7 +357,7 @@ run_tests() { log "Skipping CFI tests because compiler ($CC_VERSION) doesn't support CFI" fi - install_uninstall_tests + install_tests check_symbol_prefixes test_use_shared_lib } diff --git a/scripts/toolchain-i686-w64-mingw32.cmake b/scripts/toolchain-i686-w64-mingw32.cmake new file mode 100644 index 00000000..0b806364 --- /dev/null +++ b/scripts/toolchain-i686-w64-mingw32.cmake @@ -0,0 +1,8 @@ +set(CMAKE_SYSTEM_NAME Windows) +set(CMAKE_SYSTEM_PROCESSOR i686) +set(CMAKE_C_COMPILER i686-w64-mingw32-gcc) +set(CMAKE_FIND_ROOT_PATH /usr/i686-w64-mingw32) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) diff --git a/scripts/toolchain-x86_64-w64-mingw32.cmake b/scripts/toolchain-x86_64-w64-mingw32.cmake new file mode 100644 index 00000000..f9d6e37f --- /dev/null +++ b/scripts/toolchain-x86_64-w64-mingw32.cmake @@ -0,0 +1,8 @@ +set(CMAKE_SYSTEM_NAME Windows) +set(CMAKE_SYSTEM_PROCESSOR x86_64) +set(CMAKE_C_COMPILER x86_64-w64-mingw32-gcc) +set(CMAKE_FIND_ROOT_PATH /usr/x86_64-w64-mingw32) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)