Fix dirio tests #4568
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PERF | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| # References: | |
| # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow | |
| # https://docs.github.com/en/actions/using-workflows/reusing-workflows#reusable-workflows-and-starter-workflows | |
| # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onworkflow_call | |
| # https://docs.github.com/en/actions/learn-github-actions/contexts#about-contexts-and-expressions | |
| # You can override the default DIFF_CUTOFF_PERCENT by specifying a cutoff along | |
| # with the benchmark target. | |
| # Eg, "Data.Async:12" where "Data.Async" is the benchmark target, ":" is the | |
| # seperator, and "12" is the new cutoff percent | |
| #---------------------------------------------------------------------- | |
| #-- Benchmarks listed in alphabetical order | |
| #---------------------------------------------------------------------- | |
| # Removed Data.Fold.Prelude - was failing for some reason (memory issue?) | |
| jobs: | |
| build: | |
| name: "Comparison: master vs PR (ubuntu-latest ghc-9.14.1)" | |
| env: | |
| GHCVER: 9.14.1 | |
| GHCUP_VERSION: 0.1.50.2 | |
| CI_BENCHMARKS_WITH_CUTOFF: >- | |
| Data.Array | |
| Data.Array.Generic | |
| Data.Array.Stream | |
| Data.Fold | |
| Data.Fold.Window | |
| Data.MutArray | |
| Data.Parser | |
| Data.ParserK | |
| Data.ParserK.Chunked | |
| Data.ParserK.Chunked.Generic | |
| Data.RingArray | |
| Data.Scanl | |
| Data.Scanl.Concurrent | |
| Data.Scanl.Window | |
| Data.Serialize | |
| Data.Stream | |
| Data.Stream.Concurrent | |
| Data.Stream.ConcurrentEager | |
| Data.Stream.ConcurrentInterleaved | |
| Data.Stream.ConcurrentOrdered | |
| Data.Stream.Prelude | |
| Data.StreamK:6 | |
| Data.Unbox | |
| Data.Unbox.Derive.TH | |
| Data.Unfold | |
| Data.Unfold.Prelude | |
| FileSystem.DirIO | |
| FileSystem.Handle | |
| Unicode.Parser | |
| Unicode.Stream | |
| CI_FIELDS: allocated | |
| CI_DIFF_CUTOFF_PERCENT: 3 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout the PR branch | |
| uses: actions/checkout@v4 | |
| # This should happen before cache restore. | |
| - name: Remove ~/.ghcup symlink | |
| run: | | |
| rm -f ~/.ghcup | |
| - name: Cache ghcup and ghc (non-Windows) | |
| uses: actions/cache@v4 | |
| if: runner.os != 'Windows' | |
| with: | |
| path: | | |
| ~/.ghcup | |
| key: ${{ runner.os }}-ghcup-${{ env.GHCUP_VERSION }}-${{ env.GHCVER }} | |
| - name: Cache hackage package index (non-Windows) | |
| uses: actions/cache@v4 | |
| if: runner.os != 'Windows' | |
| with: | |
| path: | | |
| ~/.cache/cabal/packages | |
| # Bump the key version to clear the cache | |
| key: cache-cabal-packages | |
| - name: Cache cabal build dependencies (non-Windows) | |
| uses: actions/cache@v4 | |
| if: runner.os != 'Windows' | |
| with: | |
| path: | | |
| ~/.local/state/cabal | |
| # Bump the key version to clear the cache | |
| key: performance-comparison-deps | |
| - name: Update CI_BENCHMARKS environment variable | |
| run: | | |
| CI_BENCHMARKS="" | |
| for i in $CI_BENCHMARKS_WITH_CUTOFF | |
| do | |
| bname=$(echo "$i" | cut -d: -f1) | |
| CI_BENCHMARKS="$CI_BENCHMARKS $bname" | |
| done | |
| echo "CI_BENCHMARKS=$CI_BENCHMARKS" >> $GITHUB_ENV | |
| # Use a sane PATH, especially to pick the right GHC | |
| - name: Set PATH to add .local and .ghcup | |
| run: echo "$HOME/.local/bin:$HOME/.ghcup/bin:/bin:/usr/bin" > $GITHUB_PATH | |
| - name: Download ghc | |
| run: | | |
| if ! ghc --version 2>/dev/null | grep -q "$GHCVER" | |
| then | |
| curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VERSION/x86_64-linux-ghcup-$GHCUP_VERSION | |
| chmod +x ./ghcup | |
| export GHCUP_INSTALL_BASE_PREFIX=$HOME | |
| ./ghcup install ghc $GHCVER | |
| ./ghcup set ghc $GHCVER | |
| ./ghcup install cabal | |
| fi | |
| # XXX update only if not present | |
| cabal update | |
| - name: Install bench-runner from PR branch | |
| run: | | |
| cabal install bench-runner --project-file=cabal.project.report --installdir=./ --allow-newer | |
| # ----------------------------------------------------------------- | |
| # -- Generate reports for the base branch and upload | |
| # ----------------------------------------------------------------- | |
| - name: Checkout the "master" branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: master | |
| clean: false | |
| - name: Run benchmarks for "master" branch | |
| run: | | |
| # --cabal-build-options "-j1" | |
| ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw | |
| # XXX Print dependencies of streamly-core and streamly for comparison | |
| # in case there is a regression due to version changes. | |
| # ghc-pkg --package-db=./dist-newstyle/packagedb/ghc-9.10.3/ field streamly depends | |
| # ----------------------------------------------------------------- | |
| # -- Download, generate reports for the current branch and append | |
| # ----------------------------------------------------------------- | |
| - name: Checkout the PR branch | |
| uses: actions/checkout@v4 | |
| with: | |
| # preserve bench-runner and the "charts" directory created by bench-runner | |
| clean: false | |
| - name: Run benchmarks and append | |
| run: | | |
| # --cabal-build-options "-j1" | |
| ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw --append | |
| # ----------------------------------------------------------------- | |
| # -- Compare | |
| # ----------------------------------------------------------------- | |
| - name: List all benchmarks | |
| run: | | |
| ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --no-measure | |
| - name: Compare benchmarks | |
| run: | | |
| EXIT_STATUS=0 | |
| for i in $CI_BENCHMARKS_WITH_CUTOFF | |
| do | |
| arrI=(${i//:/ }) | |
| bname=${arrI[0]} | |
| cutoff=${arrI[1]} | |
| test -z "$cutoff" && cutoff=$CI_DIFF_CUTOFF_PERCENT | |
| echo | |
| echo "Checking $bname for regressions greater than $cutoff percent" | |
| ! ./bench-runner \ | |
| --package-name streamly-benchmarks \ | |
| --package-version 0.0.0 \ | |
| --targets "$bname" \ | |
| --fields "$CI_FIELDS" \ | |
| --no-measure --silent \ | |
| --diff-cutoff-percent $cutoff \ | |
| | grep -v "^$" | |
| test $? -eq 1 && EXIT_STATUS=1 | |
| done | |
| # Always exit with success so that we are able to use the cache | |
| echo | |
| echo "------------------------------------------------------------------------" | |
| if test $EXIT_STATUS -ne 0 | |
| then | |
| echo "ERROR: Performance regressions are more than ${CI_DIFF_CUTOFF_PERCENT}%." | |
| else | |
| echo "SUCCESS: Performance regressions are within ${CI_DIFF_CUTOFF_PERCENT}%." | |
| fi | |
| echo "------------------------------------------------------------------------" | |
| exit 0 |