diff --git a/.github/workflows/prebuilt-tdlib.yml b/.github/workflows/prebuilt-tdlib.yml index 0f11454..d74c800 100644 --- a/.github/workflows/prebuilt-tdlib.yml +++ b/.github/workflows/prebuilt-tdlib.yml @@ -20,142 +20,77 @@ on: # NOTE: The ZLIB_USE_STATIC_LIBS option requires CMake >= 3.24 jobs: build-linux-x64: - name: Build TDLib (GNU/Linux x86_64) - runs-on: ubuntu-latest + name: 'Build TDLib / Linux x86_64 glibc' + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v27 with: - repository: 'tdlib/td' - ref: ${{ inputs.tdlib }} - - name: Create a build script - run: | - cat > prebuilt-tdlib-docker.sh <> info.txt - ldd --version | grep ldd >> info.txt - openssl version >> info.txt - sed -n 's/#define ZLIB_VERSION "\([^"]*\)"/zlib version: \1/p' /hbb_shlib/include/zlib.h >> info.txt - EOF - - name: Build in docker - run: | - mkdir to-upload - docker run -v $(pwd):/td ghcr.io/phusion/holy-build-box/hbb-64 \ - bash /td/prebuilt-tdlib-docker.sh - - name: Info - working-directory: to-upload + nix_path: nixpkgs=channel:nixos-unstable + - name: Build TDLib run: | - cat info.txt - echo "-----" - du -hs libtdjson.so - ldd libtdjson.so + cd packages/prebuilt-tdlib/ci + ./build-linux.sh ${{ inputs.tdlib }} x86_64-linux-gnu.2.22 - uses: actions/upload-artifact@v4 with: name: tdlib-linux-x64 - path: to-upload - build-macos: - name: Build TDLib (macOS universal) - runs-on: macos-12 - env: - openssl_dir: /usr/local/opt/custom_openssl - openssl_tag: '1.1.2100' # OpenSSL 1.1.1u - zlib_ver: '1.2.13' - zlib_dir: /usr/local/opt/custom_zlib + path: ${{ env.TO_UPLOAD }} + build-macos-x64: + name: 'Build TDLib / macOS x86_64' + runs-on: macos-13 steps: - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v27 with: - repository: 'tdlib/td' - ref: ${{ inputs.tdlib }} - - name: Install gperf via homebrew - run: HOMEBREW_NO_INSTALL_CLEANUP=1 brew install gperf - - name: CMake version - run: cmake --version - # https://github.com/up9cloud/ios-libtdjson/blob/d4426dca16933b6178acb030955de92f040b7574/build.sh - - name: Download pre-built OpenSSL - run: | - # around 80 mb - curl -SL https://github.com/krzyzanowskim/OpenSSL/archive/refs/tags/${openssl_tag}.tar.gz -o OpenSSL.tar.gz - tar xzf OpenSSL.tar.gz - # As for now, MACOSX_DEPLOYMENT_TARGET in these pre-built openssl binaries should be 10.13 - mv OpenSSL-${openssl_tag}/macosx ${openssl_dir} - - name: Download and build zlib + nix_path: nixpkgs=channel:nixpkgs-unstable + - name: Build TDLib run: | - set -x - curl -SL https://github.com/madler/zlib/releases/download/v${zlib_ver}/zlib-${zlib_ver}.tar.gz -o zlib.tar.gz - tar xzf zlib.tar.gz - cd zlib-${zlib_ver} - MACOSX_DEPLOYMENT_TARGET=10.14 CFLAGS="-O2 -mmacosx-version-min=10.14 -arch x86_64" \ - ./configure --static --prefix=${zlib_dir} - MACOSX_DEPLOYMENT_TARGET=10.14 make - make install && rm ${zlib_dir}/lib/libz.a - cp libz.a libz-x86_64.a - make clean - MACOSX_DEPLOYMENT_TARGET=11.0 CFLAGS="-O2 -mmacosx-version-min=11.0 -arch arm64" \ - ./configure --static - MACOSX_DEPLOYMENT_TARGET=11.0 make - mv libz.a libz-arm64.a - lipo -create libz-x86_64.a libz-arm64.a -output libz.a - cp libz.a ${zlib_dir}/lib/libz.a + cd packages/prebuilt-tdlib/ci && ./build-macos.sh ${{ inputs.tdlib }} + - uses: actions/upload-artifact@v4 + with: + name: temp-macos-x64 + path: ${{ env.TO_UPLOAD }} + build-macos-arm64: + name: 'Build TDLib / macOS arm64' + runs-on: macos-14 + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v27 + with: + nix_path: nixpkgs=channel:nixpkgs-unstable - name: Build TDLib run: | - mkdir to-upload - mkdir -p build && cd build - MACOSX_DEPLOYMENT_TARGET=10.14 cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_OSX_ARCHITECTURES='x86_64;arm64' \ - -DOPENSSL_USE_STATIC_LIBS=TRUE -DZLIB_USE_STATIC_LIBS=TRUE \ - -DOPENSSL_FOUND=1 \ - -DOPENSSL_CRYPTO_LIBRARY=${openssl_dir}/lib/libcrypto.a \ - -DOPENSSL_SSL_LIBRARY=${openssl_dir}/lib/libssl.a \ - -DOPENSSL_LIBRARIES="${openssl_dir}/lib/libcrypto.a;${openssl_dir}/lib/libssl.a" \ - -DOPENSSL_INCLUDE_DIR=${openssl_dir}/include \ - -DZLIB_INCLUDE_DIR=${zlib_dir}/include \ - -DZLIB_LIBRARY=${zlib_dir}/lib/libz.a \ - .. - cmake --build . --target tdjson -- -j 3 - cd .. - cp -L build/libtdjson.dylib to-upload/libtdjson.dylib - install_name_tool -id @rpath/libtdjson.dylib to-upload/libtdjson.dylib - - name: Info - working-directory: to-upload + cd packages/prebuilt-tdlib/ci && ./build-macos.sh ${{ inputs.tdlib }} + - uses: actions/upload-artifact@v4 + with: + name: temp-macos-arm64 + path: ${{ env.TO_UPLOAD }} + build-macos: + name: Create universal macOS shared library + needs: [build-macos-x64, build-macos-arm64] + runs-on: macos-13 + steps: + - uses: actions/download-artifact@v4 + with: + pattern: temp-macos-* + - name: Combine shared libraries run: | - touch info.txt - grep OPENSSL_VERSION_TEXT ${openssl_dir}/include/openssl/opensslv.h | sed 's/.*"\([^"]*\)"/\1/' >> info.txt - echo "zlib version: ${zlib_ver}" >> info.txt - otool -L libtdjson.dylib >> info.txt - cat info.txt - echo "-----" - uname -a + mkdir -p to-upload + lipo temp-macos-x64/libtdjson.dylib temp-macos-arm64/libtdjson.dylib \ + -output to-upload/libtdjson.dylib -create + cd to-upload file libtdjson.dylib - du -hs libtdjson.dylib - otool -arch x86_64 -l libtdjson.dylib - echo "---" - otool -arch arm64 -l libtdjson.dylib + otool -L libtdjson.dylib + # echo "--x86_64--" >> info.txt + # cat ../temp-macos-x64/libtdjson.dylib >> info.txt + # echo "--arm64--" >> info.txt + # cat ../temp-macos-arm64/libtdjson.dylib >> info.txt - uses: actions/upload-artifact@v4 with: name: tdlib-macos path: to-upload build-windows-x64: - name: Build TDLib (Windows x86_64) + name: 'Build TDLib / Windows x86_64' runs-on: windows-2019 steps: - uses: actions/checkout@v4 @@ -182,12 +117,10 @@ jobs: -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake ` -DVCPKG_TARGET_TRIPLET=x64-windows-static ` -DOPENSSL_USE_STATIC_LIBS=TRUE -DZLIB_USE_STATIC_LIBS=TRUE .. - cmake --build . --target tdjson --config Release --parallel 2 + cmake --build . --target tdjson --config Release --parallel 4 cd .. cp build\Release\tdjson.dll to-upload\tdjson.dll - - name: Info - # It should be possible to print the dependencies using dumpbin.exe - run: | + # It should be possible to print the dependencies using dumpbin.exe vcpkg list | Select-String openssl,zlib > to-upload\info.txt cat to-upload\info.txt - uses: actions/upload-artifact@v4 @@ -196,16 +129,21 @@ jobs: path: to-upload test: - name: 'Test the ${{ matrix.os[1] }} pre-built libraries' + name: 'Test / ${{ matrix.v.bin }} / ${{ matrix.v.os }}' needs: [build-linux-x64, build-macos, build-windows-x64] - runs-on: ${{ matrix.os[0] }} + runs-on: ${{ matrix.v.os }} strategy: fail-fast: false matrix: - os: - - [ubuntu-latest, tdlib-linux-x64] - - [macos-latest, tdlib-macos] - - [windows-latest, tdlib-windows-x64] + v: + - os: ubuntu-latest + bin: tdlib-linux-x64 + - os: macos-13 + bin: tdlib-macos + - os: macos-14 + bin: tdlib-macos + - os: windows-latest + bin: tdlib-windows-x64 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -217,7 +155,7 @@ jobs: run: npm uninstall -D prebuilt-tdlib # just in case - uses: actions/download-artifact@v4 with: - name: ${{ matrix.os[1] }} + name: ${{ matrix.v.bin }} merge-multiple: true - run: npm run test:integration env: @@ -245,6 +183,7 @@ jobs: pattern: tdlib-* path: packages/prebuilt-tdlib/prebuilds - run: tree packages/prebuilt-tdlib + - run: du -hsc packages/prebuilt-tdlib/prebuilds/* - run: npm run test:integration env: PREBUILT_PATH: packages/prebuilt-tdlib diff --git a/.github/workflows/publish-tdl.yml b/.github/workflows/publish-tdl.yml index c84dd5d..0838702 100644 --- a/.github/workflows/publish-tdl.yml +++ b/.github/workflows/publish-tdl.yml @@ -24,6 +24,8 @@ jobs: node-version: lts/* cache: npm - uses: goto-bus-stop/setup-zig@v2 + with: + version: '0.13.0' if: runner.os == 'Linux' - name: Install dependencies run: npm install diff --git a/packages/prebuilt-tdlib/.gitignore b/packages/prebuilt-tdlib/.gitignore index 9fba098..50d12e5 100644 --- a/packages/prebuilt-tdlib/.gitignore +++ b/packages/prebuilt-tdlib/.gitignore @@ -1,2 +1,3 @@ prebuilds/tdlib-* prebuilds/*.json +to-upload/ diff --git a/packages/prebuilt-tdlib/README.md b/packages/prebuilt-tdlib/README.md index c9da615..82285dc 100644 --- a/packages/prebuilt-tdlib/README.md +++ b/packages/prebuilt-tdlib/README.md @@ -10,8 +10,8 @@ The shared libraries are statically linked against OpenSSL and zlib to prevent compatibility issues in Node.js. Supported systems: -- GNU/Linux x86_64 (requires glibc >= 2.17) -- macOS x86_64, arm64 (universal, requires macOS >= 10.14) +- Linux x86_64 (requires glibc >= 2.22) +- macOS x86_64, arm64 (universal, requires macOS >= 10.12) - Windows x86_64 To install `prebuilt-tdlib` for the latest TDLib version that `prebuilt-tdlib` @@ -121,6 +121,15 @@ An incomplete list is available below (mostly exceptions or "notable" versions): Changes to the building process of `prebuilt-tdlib` are noted below. +### (unreleased) + +The building process of Linux and macOS builds is significantly changed. + +- on macOS, TDLib is built using macOS SDK from nixpkgs, and the minimal + supported macOS version is now 10.12 instead of 10.14. +- On Linux, TDLib is now built using zig. The minimal glibc version is 2.22 + instead of 2.17. + ### 2024-05-08 First published as `prebuilt-tdlib@td-1.8.29`. diff --git a/packages/prebuilt-tdlib/ci/build-linux.sh b/packages/prebuilt-tdlib/ci/build-linux.sh new file mode 100755 index 0000000..90ba87d --- /dev/null +++ b/packages/prebuilt-tdlib/ci/build-linux.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +_tdlib=$1 +_target="${2:-x86_64-linux-gnu.2.22}" + +if [ -z "$_tdlib" ]; then + echo "Not enough arguments: expected TDLib rev" + exit 1 +fi + +set -ex + +nix-build tdlib-linux.nix -v --argstr rev "$_tdlib" --argstr target "$_target" + +mkdir to-upload +cp -L ./result/lib/libtdjson.so to-upload/libtdjson.so +cd to-upload + +# Info +ldd libtdjson.so + +if [ -z "$GITHUB_ENV" ]; then + echo "Note: GITHUB_ENV not found" + exit +fi + +# Resulting directory +echo "TO_UPLOAD=$(pwd)" >> $GITHUB_ENV diff --git a/packages/prebuilt-tdlib/ci/build-macos.sh b/packages/prebuilt-tdlib/ci/build-macos.sh new file mode 100755 index 0000000..62a94e1 --- /dev/null +++ b/packages/prebuilt-tdlib/ci/build-macos.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +_tdlib=$1 + +if [ -z "$_tdlib" ]; then + echo "Not enough arguments: expected TDLib rev" + exit 1 +fi + +set -ex + +nix-build tdlib-macos.nix -v --argstr rev "$_tdlib" + +mkdir to-upload +cp -L ./result/lib/libtdjson.dylib to-upload/libtdjson.dylib +cd to-upload + +install_name_tool -id @rpath/libtdjson.dylib libtdjson.dylib +install_name_tool -change \ + "$(otool -L libtdjson.dylib | grep -o '/nix/store.*libc++.*.dylib')" \ + '/usr/lib/libc++.1.dylib' \ + libtdjson.dylib + +# Info +otool -L libtdjson.dylib +otool -l libtdjson.dylib + +if [ -z "$GITHUB_ENV" ]; then + echo "Note: GITHUB_ENV not found" + exit +fi + +# Resulting directory +echo "TO_UPLOAD=$(pwd)" >> $GITHUB_ENV diff --git a/packages/prebuilt-tdlib/ci/openssl-zig.nix b/packages/prebuilt-tdlib/ci/openssl-zig.nix new file mode 100644 index 0000000..e09fe6f --- /dev/null +++ b/packages/prebuilt-tdlib/ci/openssl-zig.nix @@ -0,0 +1,5 @@ +{ openssl, zig-toolchain }: +(openssl.override { static = true; }).overrideAttrs (final: prev: { + preConfigure = zig-toolchain.env; + doCheck = false; +}) diff --git a/packages/prebuilt-tdlib/ci/tdlib-linux.nix b/packages/prebuilt-tdlib/ci/tdlib-linux.nix new file mode 100644 index 0000000..42da1ed --- /dev/null +++ b/packages/prebuilt-tdlib/ci/tdlib-linux.nix @@ -0,0 +1,57 @@ +{ pkgs ? import {}, target, rev }: +let + inherit (pkgs) lib stdenv; + zig-toolchain = import ./zig-toolchain.nix { inherit pkgs target; }; + zlib = pkgs.callPackage ./zlib-zig.nix { inherit zig-toolchain; }; + openssl = pkgs.callPackage ./openssl-zig.nix { inherit zig-toolchain; }; +in +# Based on https://github.com/NixOS/nixpkgs/blob/af51e23ce535b1bfa8484021ff3913d876e09082/pkgs/development/libraries/tdlib/default.nix +stdenv.mkDerivation { + pname = "tdlib"; + version = "0.0"; + + src = builtins.fetchTarball "https://github.com/tdlib/td/archive/${rev}.tar.gz"; + + buildInputs = [ openssl zlib ]; + nativeBuildInputs = with pkgs; [ cmake gperf ]; + + preConfigure = '' + mkdir native-build && cd native-build + cmake .. + cmake --build . --target prepare_cross_compiling + cd .. + '' + zig-toolchain.env; + + patches = [ ./tdlib-zig.patch ]; + + cmakeFlags = [ + "-DCMAKE_BUILD_TYPE=Release" + "-DOPENSSL_USE_STATIC_LIBS=TRUE" + "-DZLIB_USE_STATIC_LIBS=TRUE" + "-DCMAKE_SYSTEM_NAME=Linux" + "-DCMAKE_CROSSCOMPILING=TRUE" + ]; + + buildPhase = '' + cmake --build . --target tdjson -j $NIX_BUILD_CORES + ''; + + installPhase = '' + runHook preInstall + mkdir -p "$out"/lib + cp -L ./libtdjson.so "$out"/lib/ + runHook postInstall + ''; + + # https://github.com/tdlib/td/issues/1974 + postPatch = '' + substituteInPlace CMake/GeneratePkgConfig.cmake \ + --replace 'function(generate_pkgconfig' \ + 'include(GNUInstallDirs) + function(generate_pkgconfig' \ + --replace '\$'{prefix}/'$'{CMAKE_INSTALL_LIBDIR} '$'{CMAKE_INSTALL_FULL_LIBDIR} \ + --replace '\$'{prefix}/'$'{CMAKE_INSTALL_INCLUDEDIR} '$'{CMAKE_INSTALL_FULL_INCLUDEDIR} + '' + lib.optionalString (stdenv.isDarwin && stdenv.isAarch64) '' + sed -i "/vptr/d" test/CMakeLists.txt + ''; +} diff --git a/packages/prebuilt-tdlib/ci/tdlib-macos.nix b/packages/prebuilt-tdlib/ci/tdlib-macos.nix new file mode 100644 index 0000000..4b927d6 --- /dev/null +++ b/packages/prebuilt-tdlib/ci/tdlib-macos.nix @@ -0,0 +1,46 @@ +{ rev }: +let + pkgs = import {}; + inherit (pkgs) lib stdenv; + zlib = pkgs.zlib.override { static = true; shared = false; }; + openssl = pkgs.openssl.override { static = true; }; +in +# Based on https://github.com/NixOS/nixpkgs/blob/af51e23ce535b1bfa8484021ff3913d876e09082/pkgs/development/libraries/tdlib/default.nix +stdenv.mkDerivation { + pname = "tdlib"; + version = "0.0"; + + src = builtins.fetchTarball "https://github.com/tdlib/td/archive/${rev}.tar.gz"; + + buildInputs = [ openssl zlib ]; + nativeBuildInputs = with pkgs; [ cmake gperf ]; + + cmakeFlags = [ + "-DCMAKE_BUILD_TYPE=Release" + "-DOPENSSL_USE_STATIC_LIBS=TRUE" + "-DZLIB_USE_STATIC_LIBS=TRUE" + ]; + + buildPhase = '' + cmake --build . --target tdjson -j $NIX_BUILD_CORES + ''; + + installPhase = '' + runHook preInstall + mkdir -p "$out"/lib + cp -L ./libtdjson.dylib "$out"/lib/ + runHook postInstall + ''; + + # https://github.com/tdlib/td/issues/1974 + postPatch = '' + substituteInPlace CMake/GeneratePkgConfig.cmake \ + --replace 'function(generate_pkgconfig' \ + 'include(GNUInstallDirs) + function(generate_pkgconfig' \ + --replace '\$'{prefix}/'$'{CMAKE_INSTALL_LIBDIR} '$'{CMAKE_INSTALL_FULL_LIBDIR} \ + --replace '\$'{prefix}/'$'{CMAKE_INSTALL_INCLUDEDIR} '$'{CMAKE_INSTALL_FULL_INCLUDEDIR} + '' + lib.optionalString (stdenv.isDarwin && stdenv.isAarch64) '' + sed -i "/vptr/d" test/CMakeLists.txt + ''; +} diff --git a/packages/prebuilt-tdlib/ci/tdlib-zig.patch b/packages/prebuilt-tdlib/ci/tdlib-zig.patch new file mode 100644 index 0000000..cd0f32b --- /dev/null +++ b/packages/prebuilt-tdlib/ci/tdlib-zig.patch @@ -0,0 +1,16 @@ +diff --git a/CMake/TdSetUpCompiler.cmake b/CMake/TdSetUpCompiler.cmake +index ff5991cfb..e1f7ea28b 100644 +--- a/CMake/TdSetUpCompiler.cmake ++++ b/CMake/TdSetUpCompiler.cmake +@@ -63,9 +63,9 @@ function(td_set_up_compiler) + elseif (EMSCRIPTEN) + set(TD_LINKER_FLAGS "-Wl,--gc-sections") + elseif (ANDROID) +- set(TD_LINKER_FLAGS "-Wl,--gc-sections -Wl,--exclude-libs,ALL -Wl,--icf=safe") ++ set(TD_LINKER_FLAGS "-Wl,--gc-sections -Wl,--icf=safe") + else() +- set(TD_LINKER_FLAGS "-Wl,--gc-sections -Wl,--exclude-libs,ALL") ++ set(TD_LINKER_FLAGS "-Wl,--gc-sections") + endif() + endif() + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${TD_LINKER_FLAGS}") diff --git a/packages/prebuilt-tdlib/ci/zig-toolchain.nix b/packages/prebuilt-tdlib/ci/zig-toolchain.nix new file mode 100644 index 0000000..ea6d7e2 --- /dev/null +++ b/packages/prebuilt-tdlib/ci/zig-toolchain.nix @@ -0,0 +1,28 @@ +{ pkgs ? import {}, target }: +# See also https://gist.github.com/Cloudef/acb74ff9e36ab41709479240596ab501 +let + write-zig-compiler-wrapper = cmd: + pkgs.writeShellScript "zig-${cmd}" '' + ZIG_LOCAL_CACHE_DIR="$TMPDIR/zig-cache-${target}" \ + ZIG_GLOBAL_CACHE_DIR="$ZIG_LOCAL_CACHE_DIR" \ + ${pkgs.zig}/bin/zig ${cmd} -target ${target} -s "$@" + ''; + write-zig-wrapper = cmd: + pkgs.writeShellScript "zig-${cmd}" '' + ZIG_LOCAL_CACHE_DIR="$TMPDIR/zig-cache-${target}" \ + ZIG_GLOBAL_CACHE_DIR="$ZIG_LOCAL_CACHE_DIR" \ + ${pkgs.zig}/bin/zig ${cmd} "$@" + ''; +in +rec { + zig-cc = write-zig-compiler-wrapper "cc"; + zig-cxx = write-zig-compiler-wrapper "c++"; + zig-ar = write-zig-wrapper "ar"; + zig-ranlib = write-zig-wrapper "ranlib"; + env = '' + export CC="${zig-cc}" + export CXX="${zig-cxx}" + export AR="${zig-ar}" + export RANLIB="${zig-ranlib}" + ''; +} diff --git a/packages/prebuilt-tdlib/ci/zlib-zig.nix b/packages/prebuilt-tdlib/ci/zlib-zig.nix new file mode 100644 index 0000000..948b5cb --- /dev/null +++ b/packages/prebuilt-tdlib/ci/zlib-zig.nix @@ -0,0 +1,14 @@ +{ zlib, zig-toolchain }: +(zlib.override { static = true; shared = false; }).overrideAttrs (final: prev: { + preConfigure = prev.preConfigure + zig-toolchain.env; + doCheck = false; + # preCheck = '' + # ls + # echo --- + # ldd ./minigzip + # ./minigzip + # echo START + # make check + # echo FINISH + # ''; +})