From 2cea75baa9258dfdd90865d4d403cefdb5810ec8 Mon Sep 17 00:00:00 2001 From: Oleg Grenrus Date: Wed, 17 Feb 2021 00:31:27 +0200 Subject: [PATCH] Move benchmarks into separate project and package This allows benchmarking without forcing e.g. aeson to depend on local uuid-types. Also, conveniently, we can generate own GHA workflow for second, benchmark, project. So we can still run CI that benchmarks are building fine. --- .git-blame-ignore-revs | 2 +- .github/workflows/haskell-ci-bench.yml | 177 ++++++++++++++++++ .github/workflows/haskell-ci.yml | 2 +- cabal.bench.project | 4 + cabal.haskell-ci | 1 - cabal.project | 2 - haskell-ci.sh | 5 + uuid-bench/LICENSE | 28 +++ .../bench/uuid-benchmark.hs | 0 .../bench/uuid-types-benchmark.hs | 0 uuid-bench/src-uuid | 1 + uuid-bench/src-uuid-types | 1 + uuid-bench/uuid-bench.cabal | 119 ++++++++++++ uuid-types/cabal.project | 1 + uuid-types/tests/summarize-bench-results.pl | 14 -- uuid-types/uuid-types.cabal | 25 +-- uuid/tests/summarize-bench-results.pl | 14 -- uuid/uuid.cabal | 26 +-- 18 files changed, 344 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/haskell-ci-bench.yml create mode 100644 cabal.bench.project create mode 100755 haskell-ci.sh create mode 100644 uuid-bench/LICENSE rename uuid/tests/BenchUUID.hs => uuid-bench/bench/uuid-benchmark.hs (100%) rename uuid-types/tests/BenchUUID.hs => uuid-bench/bench/uuid-types-benchmark.hs (100%) create mode 120000 uuid-bench/src-uuid create mode 120000 uuid-bench/src-uuid-types create mode 100644 uuid-bench/uuid-bench.cabal create mode 100644 uuid-types/cabal.project delete mode 100644 uuid-types/tests/summarize-bench-results.pl delete mode 100644 uuid/tests/summarize-bench-results.pl diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 5e26c20..1d08ce5 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,2 +1,2 @@ # Move code into subdirectories -37ae11973e287135e02cfcbde17b7830f79f5e69 +bf1ac1bad848bb2c081f0cae1631bf53811ec05a diff --git a/.github/workflows/haskell-ci-bench.yml b/.github/workflows/haskell-ci-bench.yml new file mode 100644 index 0000000..6aec236 --- /dev/null +++ b/.github/workflows/haskell-ci-bench.yml @@ -0,0 +1,177 @@ +# This GitHub workflow config has been generated by a script via +# +# haskell-ci 'github' '--project' 'cabal.bench.project' '-o' '.github/workflows/haskell-ci-bench.yml' '--github-action-name' 'Benchmarks' +# +# To regenerate the script (for example after adjusting tested-with) run +# +# haskell-ci regenerate +# +# For more information, see https://github.com/haskell-CI/haskell-ci +# +# version: 0.11.20210214 +# +# REGENDATA ("0.11.20210214",["github","--project","cabal.bench.project","-o",".github/workflows/haskell-ci-bench.yml","--github-action-name","Benchmarks"]) +# +name: Benchmarks +on: + push: + branches: + - master + pull_request: + branches: + - master +jobs: + linux: + name: Benchmarks - Linux - GHC ${{ matrix.ghc }} + runs-on: ubuntu-18.04 + container: + image: buildpack-deps:bionic + continue-on-error: ${{ matrix.allow-failure }} + strategy: + matrix: + include: + - ghc: 8.10.3 + allow-failure: false + - ghc: 8.8.4 + allow-failure: false + - ghc: 8.6.4 + allow-failure: false + - ghc: 8.4.4 + allow-failure: false + - ghc: 8.2.2 + allow-failure: false + - ghc: 8.0.2 + allow-failure: false + fail-fast: false + steps: + - name: apt + run: | + apt-get update + apt-get install -y --no-install-recommends gnupg ca-certificates dirmngr curl git software-properties-common + apt-add-repository -y 'ppa:hvr/ghc' + apt-get update + apt-get install -y ghc-$GHC_VERSION cabal-install-3.2 + env: + GHC_VERSION: ${{ matrix.ghc }} + - name: Set PATH and environment variables + run: | + echo "$HOME/.cabal/bin" >> $GITHUB_PATH + echo "LANG=C.UTF-8" >> $GITHUB_ENV + echo "CABAL_DIR=$HOME/.cabal" >> $GITHUB_ENV + echo "CABAL_CONFIG=$HOME/.cabal/config" >> $GITHUB_ENV + HC=/opt/ghc/$GHC_VERSION/bin/ghc + echo "HC=$HC" >> $GITHUB_ENV + echo "HCPKG=/opt/ghc/$GHC_VERSION/bin/ghc-pkg" >> $GITHUB_ENV + echo "HADDOCK=/opt/ghc/$GHC_VERSION/bin/haddock" >> $GITHUB_ENV + echo "CABAL=/opt/cabal/3.2/bin/cabal -vnormal+nowrap" >> $GITHUB_ENV + HCNUMVER=$(${HC} --numeric-version|perl -ne '/^(\d+)\.(\d+)\.(\d+)(\.(\d+))?$/; print(10000 * $1 + 100 * $2 + ($3 == 0 ? $5 != 1 : $3))') + echo "HCNUMVER=$HCNUMVER" >> $GITHUB_ENV + echo "ARG_TESTS=--enable-tests" >> $GITHUB_ENV + echo "ARG_BENCH=--enable-benchmarks" >> $GITHUB_ENV + echo "HEADHACKAGE=false" >> $GITHUB_ENV + echo "ARG_COMPILER=--ghc --with-compiler=/opt/ghc/$GHC_VERSION/bin/ghc" >> $GITHUB_ENV + echo "GHCJSARITH=0" >> $GITHUB_ENV + env: + GHC_VERSION: ${{ matrix.ghc }} + - name: env + run: | + env + - name: write cabal config + run: | + mkdir -p $CABAL_DIR + cat >> $CABAL_CONFIG < cabal-plan.xz + echo 'de73600b1836d3f55e32d80385acc055fd97f60eaa0ab68a755302685f5d81bc cabal-plan.xz' | sha256sum -c - + xz -d < cabal-plan.xz > $HOME/.cabal/bin/cabal-plan + rm -f cabal-plan.xz + chmod a+x $HOME/.cabal/bin/cabal-plan + cabal-plan --version + - name: checkout + uses: actions/checkout@v2 + with: + path: source + - name: initial cabal.project for sdist + run: | + touch cabal.project + echo "packages: $GITHUB_WORKSPACE/source/uuid-bench" >> cabal.project + cat cabal.project + - name: sdist + run: | + mkdir -p sdist + $CABAL sdist all --output-dir $GITHUB_WORKSPACE/sdist + - name: unpack + run: | + mkdir -p unpacked + find sdist -maxdepth 1 -type f -name '*.tar.gz' -exec tar -C $GITHUB_WORKSPACE/unpacked -xzvf {} \; + - name: generate cabal.project + run: | + PKGDIR_uuid_bench="$(find "$GITHUB_WORKSPACE/unpacked" -maxdepth 1 -type d -regex '.*/uuid-bench-[0-9.]*')" + echo "PKGDIR_uuid_bench=${PKGDIR_uuid_bench}" >> $GITHUB_ENV + touch cabal.project + touch cabal.project.local + echo "packages: ${PKGDIR_uuid_bench}" >> cabal.project + if [ $((HCNUMVER >= 80200)) -ne 0 ] ; then echo "package uuid-bench" >> cabal.project ; fi + if [ $((HCNUMVER >= 80200)) -ne 0 ] ; then echo " ghc-options: -Werror=missing-methods" >> cabal.project ; fi + cat >> cabal.project <> cabal.project.local + cat cabal.project + cat cabal.project.local + - name: dump install plan + run: | + $CABAL v2-build $ARG_COMPILER $ARG_TESTS $ARG_BENCH --dry-run all + cabal-plan + - name: cache + uses: actions/cache@v2 + with: + key: ${{ runner.os }}-${{ matrix.ghc }}-${{ github.sha }} + path: ~/.cabal/store + restore-keys: ${{ runner.os }}-${{ matrix.ghc }}- + - name: install dependencies + run: | + $CABAL v2-build $ARG_COMPILER --disable-tests --disable-benchmarks --dependencies-only -j2 all + $CABAL v2-build $ARG_COMPILER $ARG_TESTS $ARG_BENCH --dependencies-only -j2 all + - name: build w/o tests + run: | + $CABAL v2-build $ARG_COMPILER --disable-tests --disable-benchmarks all + - name: build + run: | + $CABAL v2-build $ARG_COMPILER $ARG_TESTS $ARG_BENCH all --write-ghc-environment-files=always + - name: cabal check + run: | + cd ${PKGDIR_uuid_bench} || false + ${CABAL} -vnormal check + - name: haddock + run: | + $CABAL v2-haddock $ARG_COMPILER --with-haddock $HADDOCK $ARG_TESTS $ARG_BENCH all + - name: unconstrained build + run: | + rm -f cabal.project.local + $CABAL v2-build $ARG_COMPILER --disable-tests --disable-benchmarks all diff --git a/.github/workflows/haskell-ci.yml b/.github/workflows/haskell-ci.yml index ec52696..0930f6f 100644 --- a/.github/workflows/haskell-ci.yml +++ b/.github/workflows/haskell-ci.yml @@ -75,7 +75,7 @@ jobs: HCNUMVER=$(${HC} --numeric-version|perl -ne '/^(\d+)\.(\d+)\.(\d+)(\.(\d+))?$/; print(10000 * $1 + 100 * $2 + ($3 == 0 ? $5 != 1 : $3))') echo "HCNUMVER=$HCNUMVER" >> $GITHUB_ENV echo "ARG_TESTS=--enable-tests" >> $GITHUB_ENV - echo "ARG_BENCH=--disable-benchmarks" >> $GITHUB_ENV + echo "ARG_BENCH=--enable-benchmarks" >> $GITHUB_ENV echo "HEADHACKAGE=false" >> $GITHUB_ENV echo "ARG_COMPILER=--ghc --with-compiler=/opt/ghc/$GHC_VERSION/bin/ghc" >> $GITHUB_ENV echo "GHCJSARITH=0" >> $GITHUB_ENV diff --git a/cabal.bench.project b/cabal.bench.project new file mode 100644 index 0000000..7fbd86b --- /dev/null +++ b/cabal.bench.project @@ -0,0 +1,4 @@ +-- separate package for benchmarks +-- this way we can build criterion +packages: uuid-bench +benchmarks: True diff --git a/cabal.haskell-ci b/cabal.haskell-ci index c344e59..e1f1b77 100644 --- a/cabal.haskell-ci +++ b/cabal.haskell-ci @@ -1,2 +1 @@ branches: master -benchmarks: False diff --git a/cabal.project b/cabal.project index 62c2fdd..20759c7 100644 --- a/cabal.project +++ b/cabal.project @@ -6,12 +6,10 @@ packages: uuid-types/ package uuid-types tests: True - benchmarks: False -- profiling: True -- profiling-detail: toplevel-functions package uuid tests: True - benchmarks: False -- profiling: True -- profiling-detail: toplevel-functions diff --git a/haskell-ci.sh b/haskell-ci.sh new file mode 100755 index 0000000..c5f5e04 --- /dev/null +++ b/haskell-ci.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +# haskell-ci doesn't know how to regenerate multiple GHA workflows +haskell-ci github cabal.project +haskell-ci github --project cabal.bench.project -o .github/workflows/haskell-ci-bench.yml --github-action-name Benchmarks diff --git a/uuid-bench/LICENSE b/uuid-bench/LICENSE new file mode 100644 index 0000000..dc46bca --- /dev/null +++ b/uuid-bench/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2008, Antoine Latter + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + * The names of the authors may not be used to endorse or promote +products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/uuid/tests/BenchUUID.hs b/uuid-bench/bench/uuid-benchmark.hs similarity index 100% rename from uuid/tests/BenchUUID.hs rename to uuid-bench/bench/uuid-benchmark.hs diff --git a/uuid-types/tests/BenchUUID.hs b/uuid-bench/bench/uuid-types-benchmark.hs similarity index 100% rename from uuid-types/tests/BenchUUID.hs rename to uuid-bench/bench/uuid-types-benchmark.hs diff --git a/uuid-bench/src-uuid b/uuid-bench/src-uuid new file mode 120000 index 0000000..f9a0df4 --- /dev/null +++ b/uuid-bench/src-uuid @@ -0,0 +1 @@ +../uuid/src \ No newline at end of file diff --git a/uuid-bench/src-uuid-types b/uuid-bench/src-uuid-types new file mode 120000 index 0000000..7d4c72e --- /dev/null +++ b/uuid-bench/src-uuid-types @@ -0,0 +1 @@ +../uuid-types/src \ No newline at end of file diff --git a/uuid-bench/uuid-bench.cabal b/uuid-bench/uuid-bench.cabal new file mode 100644 index 0000000..130345b --- /dev/null +++ b/uuid-bench/uuid-bench.cabal @@ -0,0 +1,119 @@ +cabal-version: 1.12 +name: uuid-bench +version: 0 +copyright: + (c) 2017-2018 Herbert Valerio Riedel + (c) 2008-2014 Antoine Latter + +author: Antoine Latter +maintainer: Oleg Grenrus +license: BSD3 +license-file: LICENSE +category: Data +build-type: Simple + +-- Benchmarking with older GHCs is not meaningful +tested-with: + GHC ==8.0.2 || ==8.2.2 || ==8.4.4 || ==8.6.4 || ==8.8.4 || ==8.10.3 + +synopsis: UUID benchmarks +description: + UUID benchmarks. + This package is combined uuid and uuid-types, renamed so we do not + cause benchmark tools to be build against local package. + +homepage: https://github.com/haskell-hvr/uuid +bug-reports: https://github.com/haskell-hvr/uuid/issues + +source-repository head + type: git + location: https://github.com/haskell-hvr/uuid.git + subdir: uuid-bench + +library + build-depends: + base >=4.9 && <5 + , binary >=0.4 && <0.9 + , bytestring >=0.9 && <0.11 + , cryptohash-md5 >=0.11.100 && <0.12 + , cryptohash-sha1 >=0.11.100 && <0.12 + , deepseq >=1.3 && <1.5 + , entropy >=0.3.7 && <0.5 + , hashable >=1.2.7.0 && <1.4 + , network-info >=0.2 && <0.3 + , random >=1.0.1 && <1.2 + , text >=1.2.3 && <1.3 + , time >=1.1 && <1.12 + , uuid-types >=1.0.2 && <2 + + -- uuid modules + exposed-modules: + Data.UUID + Data.UUID.Util + Data.UUID.V1 + Data.UUID.V3 + Data.UUID.V4 + Data.UUID.V5 + + other-modules: + Data.UUID.Named + Data.Word.Util + + -- uuid-types modules + exposed-modules: Data.UUID.Types + + -- Exposed for companion projects; *NOT* part of the official API: + exposed-modules: + Data.UUID.Types.Internal + Data.UUID.Types.Internal.Builder + + default-language: Haskell2010 + other-extensions: + DeriveDataTypeable + TypeFamilies + + ghc-options: -Wall + hs-source-dirs: src-uuid-types src-uuid + +benchmark uuid-types-benchmark + type: exitcode-stdio-1.0 + main-is: uuid-types-benchmark.hs + hs-source-dirs: bench + default-language: Haskell2010 + other-extensions: CPP + ghc-options: -Wall + + -- inherited constraints + build-depends: + base + , bytestring + , random + , uuid-bench + + -- deps w/o inherited constraints + build-depends: + containers >=0.5 && <0.7 + , criterion >=1.5 && <1.6 + , unordered-containers >=0.2.7 && <0.3 + +benchmark uuid-benchmark + type: exitcode-stdio-1.0 + main-is: uuid-benchmark.hs + hs-source-dirs: bench + default-language: Haskell2010 + default-extensions: + CPP + DeriveDataTypeable + + ghc-options: -Wall -fno-warn-orphans + + -- inherited constraints + build-depends: + base + , random + , uuid + + -- deps w/o inherited constraints + build-depends: + criterion >=1.5 && <1.6 + , mersenne-random-pure64 >=0.2 && <0.3 diff --git a/uuid-types/cabal.project b/uuid-types/cabal.project new file mode 100644 index 0000000..e6fdbad --- /dev/null +++ b/uuid-types/cabal.project @@ -0,0 +1 @@ +packages: . diff --git a/uuid-types/tests/summarize-bench-results.pl b/uuid-types/tests/summarize-bench-results.pl deleted file mode 100644 index df1451c..0000000 --- a/uuid-types/tests/summarize-bench-results.pl +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env perl - - -while(<>) { - if (/^benchmarking (.*)$/) { - $name = $1; - } - if (/^mean:\s*([\d.]*) (\w*),/) { - $mean = 1 * $1; - $units = $2; - - printf "%-30s %10.2f %2s\n", $name, $mean, $units; - } -} diff --git a/uuid-types/uuid-types.cabal b/uuid-types/uuid-types.cabal index 4071863..3772ed2 100644 --- a/uuid-types/uuid-types.cabal +++ b/uuid-types/uuid-types.cabal @@ -50,7 +50,7 @@ library , bytestring >=0.9 && <0.11 , deepseq >=1.3 && <1.5 , hashable >=1.2.7.0 && <1.4 - , random >=1.0.1 && <1.2 + , random >=1.1 && <1.2 , text >=1.2.3 && <1.3 exposed-modules: Data.UUID.Types @@ -66,7 +66,7 @@ library TypeFamilies ghc-options: -Wall - hs-source-dirs: src + hs-source-dirs: src test-suite testuuid type: exitcode-stdio-1.0 @@ -86,26 +86,7 @@ test-suite testuuid -- deps w/o inherited constraints build-depends: ghc-byteorder >=4.11 && <4.12 - , QuickCheck >=2.14 && <2.15 + , QuickCheck >=2.14.2 && <2.15 , tasty >=1.4.0.1 && <1.5 , tasty-hunit >=0.10 && <0.11 , tasty-quickcheck >=0.10 && <0.11 - -benchmark benchmark - type: exitcode-stdio-1.0 - main-is: BenchUUID.hs - hs-source-dirs: tests - default-language: Haskell2010 - other-extensions: CPP - ghc-options: -Wall - build-depends: - base - , bytestring - , containers >=0.4 && <0.7 - , criterion >=1.5 && <1.6 - , random - , unordered-containers >=0.2.7 && <0.3 - , uuid-types - --- inherited constraints --- deps w/o inherited constraints diff --git a/uuid/tests/summarize-bench-results.pl b/uuid/tests/summarize-bench-results.pl deleted file mode 100644 index df1451c..0000000 --- a/uuid/tests/summarize-bench-results.pl +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env perl - - -while(<>) { - if (/^benchmarking (.*)$/) { - $name = $1; - } - if (/^mean:\s*([\d.]*) (\w*),/) { - $mean = 1 * $1; - $units = $2; - - printf "%-30s %10.2f %2s\n", $name, $mean, $units; - } -} diff --git a/uuid/uuid.cabal b/uuid/uuid.cabal index 606bbd2..72a4585 100644 --- a/uuid/uuid.cabal +++ b/uuid/uuid.cabal @@ -47,7 +47,7 @@ library , cryptohash-sha1 >=0.11.100 && <0.12 , entropy >=0.3.7 && <0.5 , network-info >=0.2 && <0.3 - , random >=1.0.1 && <1.2 + , random >=1.1 && <1.2 , text >=1.2.3 && <1.3 , time >=1.1 && <1.12 , uuid-types >=1.0.2 && <2 @@ -86,29 +86,9 @@ test-suite testuuid , random , uuid + -- deps w/o inherited constraints build-depends: - QuickCheck >=2.14 && <2.15 + QuickCheck >=2.14.2 && <2.15 , tasty >=1.4.0.1 && <1.5 , tasty-hunit >=0.10 && <0.11 , tasty-quickcheck >=0.10 && <0.11 - --- deps w/o inherited constraints -benchmark benchmark - type: exitcode-stdio-1.0 - main-is: BenchUUID.hs - hs-source-dirs: tests - default-language: Haskell2010 - default-extensions: - CPP - DeriveDataTypeable - - ghc-options: -Wall -fno-warn-orphans - build-depends: - base - , criterion >=1.5 && <1.6 - , mersenne-random-pure64 >=0.2 && <0.3 - , random - , uuid - --- inherited constraints --- deps w/o inherited constraints