regression-check.yml

name: PERF

on:
  workflow_dispatch:
  pull_request:

# References:
# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
# https://docs.github.com/en/actions/using-workflows/reusing-workflows#reusable-workflows-and-starter-workflows
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onworkflow_call
# https://docs.github.com/en/actions/learn-github-actions/contexts#about-contexts-and-expressions

# You can override the default DIFF_CUTOFF_PERCENT by specifying a cutoff along
# with the benchmark target.
# Eg, "Data.Async:12" where "Data.Async" is the benchmark target, ":" is the
# seperator, and "12" is the new cutoff percent
#----------------------------------------------------------------------
#-- Benchmarks listed in alphabetical order
#----------------------------------------------------------------------
# Removed Data.Fold.Prelude - was failing for some reason (memory issue?)
jobs:
  build:
    name: "Comparison: master vs PR (ubuntu-latest ghc-9.14.1)"
    env:
      GHCVER: 9.14.1
      GHCUP_VERSION: 0.1.50.2
      CI_BENCHMARKS_WITH_CUTOFF: >-
        Data.Array
        Data.Array.Generic
        Data.Array.Stream
        Data.Fold
        Data.Fold.Window
        Data.MutArray
        Data.Parser
        Data.ParserK
        Data.ParserK.Chunked
        Data.ParserK.Chunked.Generic
        Data.RingArray
        Data.Scanl
        Data.Scanl.Concurrent
        Data.Scanl.Window
        Data.Serialize
        Data.Stream
        Data.Stream.Concurrent
        Data.Stream.ConcurrentEager
        Data.Stream.ConcurrentInterleaved
        Data.Stream.ConcurrentOrdered
        Data.Stream.Prelude
        Data.StreamK:6
        Data.Unbox
        Data.Unbox.Derive.TH
        Data.Unfold
        Data.Unfold.Prelude
        FileSystem.DirIO
        FileSystem.Handle
        Unicode.Parser
        Unicode.Stream
      CI_FIELDS: allocated
      CI_DIFF_CUTOFF_PERCENT: 3

    runs-on: ubuntu-latest

    steps:

    - name: Checkout the PR branch
      uses: actions/checkout@v4

    # This should happen before cache restore.
    - name: Remove ~/.ghcup symlink
      run: |
        rm -f ~/.ghcup

    - name: Cache ghcup and ghc (non-Windows)
      uses: actions/cache@v4
      if: runner.os != 'Windows'
      with:
        path: |
          ~/.ghcup
        key: ${{ runner.os }}-ghcup-${{ env.GHCUP_VERSION }}-${{ env.GHCVER }}

    - name: Cache hackage package index (non-Windows)
      uses: actions/cache@v4
      if: runner.os != 'Windows'
      with:
        path: |
            ~/.cache/cabal/packages
        # Bump the key version to clear the cache
        key: cache-cabal-packages

    - name: Cache cabal build dependencies (non-Windows)
      uses: actions/cache@v4
      if: runner.os != 'Windows'
      with:
        path: |
            ~/.local/state/cabal
        # Bump the key version to clear the cache
        key: performance-comparison-deps

    - name: Update CI_BENCHMARKS environment variable
      run: |
        CI_BENCHMARKS=""
        for i in $CI_BENCHMARKS_WITH_CUTOFF
        do
          bname=$(echo "$i" | cut -d: -f1)
          CI_BENCHMARKS="$CI_BENCHMARKS $bname"
        done
        echo "CI_BENCHMARKS=$CI_BENCHMARKS" >> $GITHUB_ENV

    # Use a sane PATH, especially to pick the right GHC
    - name: Set PATH to add .local and .ghcup
      run: echo "$HOME/.local/bin:$HOME/.ghcup/bin:/bin:/usr/bin" > $GITHUB_PATH

    - name: Download ghc
      run: |
        if ! ghc --version 2>/dev/null | grep -q "$GHCVER"
        then
          curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VERSION/x86_64-linux-ghcup-$GHCUP_VERSION
          chmod +x ./ghcup
          export GHCUP_INSTALL_BASE_PREFIX=$HOME
          ./ghcup install ghc $GHCVER
          ./ghcup set ghc $GHCVER
          ./ghcup install cabal
        fi
        # XXX update only if not present
        cabal update

    - name: Install bench-runner from PR branch
      run: |
        cabal install bench-runner --project-file=cabal.project.report --installdir=./ --allow-newer

    # -----------------------------------------------------------------
    # -- Generate reports for the base branch and upload
    # -----------------------------------------------------------------

    - name: Checkout the "master" branch
      uses: actions/checkout@v4
      with:
        ref: master
        clean: false

    - name: Run benchmarks for "master" branch
      run: |
        # --cabal-build-options "-j1"
        ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw
        # XXX Print dependencies of streamly-core and streamly for comparison
        # in case there is a regression due to version changes.
        # ghc-pkg --package-db=./dist-newstyle/packagedb/ghc-9.10.3/ field streamly depends

    # -----------------------------------------------------------------
    # -- Download, generate reports for the current branch and append
    # -----------------------------------------------------------------

    - name: Checkout the PR branch
      uses: actions/checkout@v4
      with:
        # preserve bench-runner and the "charts" directory created by bench-runner
        clean: false

    - name: Run benchmarks and append
      run: |
        # --cabal-build-options "-j1"
        ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw --append

    # -----------------------------------------------------------------
    # -- Compare
    # -----------------------------------------------------------------

    - name: List all benchmarks
      run: |
        ./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --no-measure

    - name: Compare benchmarks
      run: |

        EXIT_STATUS=0

        for i in $CI_BENCHMARKS_WITH_CUTOFF
        do
          arrI=(${i//:/ })

          bname=${arrI[0]}
          cutoff=${arrI[1]}

          test -z "$cutoff" && cutoff=$CI_DIFF_CUTOFF_PERCENT

          echo
          echo "Checking $bname for regressions greater than $cutoff percent"
          ! ./bench-runner  \
                    --package-name streamly-benchmarks \
                    --package-version 0.0.0 \
                    --targets "$bname" \
                    --fields "$CI_FIELDS" \
                    --no-measure --silent \
                    --diff-cutoff-percent $cutoff \
                | grep -v "^$"
          test $? -eq 1 && EXIT_STATUS=1
        done

        # Always exit with success so that we are able to use the cache
        echo
        echo "------------------------------------------------------------------------"
        if test $EXIT_STATUS -ne 0
        then
          echo "ERROR: Performance regressions are more than ${CI_DIFF_CUTOFF_PERCENT}%."
        else
          echo "SUCCESS: Performance regressions are within ${CI_DIFF_CUTOFF_PERCENT}%."
        fi
        echo "------------------------------------------------------------------------"
        exit 0