resource: support checkpoint and eventlog truncation #13000
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: [pull_request, push, merge_group] | |
| name: ci | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| check-pr: | |
| name: validate commits | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'pull_request' | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - run: git fetch origin master | |
| - uses: flux-framework/pr-validator@master | |
| spelling: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| - name: Check Spelling | |
| uses: crate-ci/typos@v1.46.0 | |
| coccinelle: | |
| name: coccinelle | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| - name: install coccinelle | |
| run: sudo apt update; sudo apt install -y coccinelle | |
| - name: run coccinelle checks | |
| run: | | |
| rc=0 | |
| for f in t/cocci/*.cocci; do | |
| echo "checking $f ..." | |
| spatch --sp-file $f --dir src/ --very-quiet >spatch.out 2>&1 | |
| cat spatch.out | |
| test ! -s spatch.out || rc=1 | |
| done | |
| exit $rc | |
| python-lint: | |
| name: python linting | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/setup-python@v6.2.0 | |
| - uses: actions/checkout@v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - name: install linting and formatting deps | |
| run: pip install -r scripts/requirements-dev.txt | |
| - name: format and linting checks | |
| run: pre-commit run --all-files | |
| check-sched: | |
| needs: [python-lint] | |
| name: flux-sched check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - run: git fetch --tags || true | |
| - run: > | |
| src/test/docker/docker-run-checks.sh --install-only \ | |
| --tag=fluxrm/flux-core:bookworm | |
| - run: > | |
| cd .. && git clone https://github.com/flux-framework/flux-sched && | |
| cd flux-sched && | |
| src/test/docker/docker-run-checks.sh -j 4 -i bookworm -- CXXFLAGS=-Wno-error=maybe-uninitialized | |
| check-accounting: | |
| needs: [python-lint] | |
| name: flux-accounting check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - run: > | |
| src/test/docker/docker-run-checks.sh --image=el8 --install-only \ | |
| --tag=fluxrm/flux-core:el8 | |
| - run: > | |
| cd .. && git clone https://github.com/flux-framework/flux-accounting && | |
| cd flux-accounting && src/test/docker/docker-run-checks.sh -j 4 | |
| check-pmix: | |
| needs: [python-lint] | |
| name: flux-pmix check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - run: > | |
| src/test/docker/docker-run-checks.sh | |
| --image=el8 | |
| --install-only | |
| --tag=fluxrm/flux-core:el8 | |
| - run: > | |
| cd .. && | |
| git clone https://github.com/flux-framework/flux-pmix && | |
| cd flux-pmix && | |
| src/test/docker/docker-run-checks.sh -j 4 -i el8 | |
| --build-arg OMPI_BRANCH=v5.0.0rc12 | |
| --build-arg OPENPMIX_BRANCH=v4.2.3 | |
| check-pam: | |
| needs: [python-lint] | |
| name: flux-pam check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - run: > | |
| src/test/docker/docker-run-checks.sh | |
| --image=el8 | |
| --install-only | |
| --tag=fluxrm/flux-core:el8 | |
| - run: > | |
| cd .. && | |
| git clone https://github.com/flux-framework/flux-pam && | |
| cd flux-pam && | |
| src/test/docker/docker-run-checks.sh -j 4 -i el8 | |
| build-macos: | |
| needs: [python-lint] | |
| name: macos build only | |
| runs-on: macos-14 | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - name: Install dependencies | |
| run: scripts/install-deps-macos.sh | |
| - name: autogen, configure | |
| run: scripts/configure-macos.sh | |
| - name: make, including test programs | |
| run: make check -j4 TESTS= | |
| - name: check what works so far | |
| run: scripts/check-macos.sh | |
| - name: tmate debug | |
| if: ${{ !cancelled() && runner.debug }} | |
| uses: mxschmitt/action-tmate@v3.23 | |
| generate-matrix: | |
| # https://stackoverflow.com/questions/59977364 | |
| name: Generate build matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - id: set-matrix | |
| run: echo "matrix=$(src/test/generate-matrix.py)" >> $GITHUB_OUTPUT | |
| - run: src/test/generate-matrix.py | jq -S . | |
| - run: echo "GITHUB_BRANCH=${GITHUB_REF#refs/heads}" >> $GITHUB_OUTPUT | |
| - run: echo "GITHUB_TAG=${GITHUB_REF#refs/tags}" >> $GITHUB_OUTPUT | |
| - run: echo "EVENT_NAME=${{ github.event_name }}" >> $GITHUB_OUTPUT | |
| ci-checks: | |
| needs: [generate-matrix] | |
| runs-on: ${{matrix.runner}} | |
| env: | |
| TAP_DRIVER_QUIET: 1 | |
| FLUX_TEST_TIMEOUT: 300 | |
| DOCKER_REPO: fluxrm/flux-core | |
| DOCKER_USERNAME: travisflux | |
| DOCKER_PASSWORD: ${{ secrets.DOCKER_HUB_TRAVISFLUX_TOKEN }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| strategy: | |
| matrix: ${{fromJson(needs.generate-matrix.outputs.matrix)}} | |
| fail-fast: false | |
| name: ${{matrix.name}} | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - name: fetch annotated tag | |
| if: > | |
| (matrix.create_release || matrix.docker_tag) && | |
| github.ref != 'refs/heads/master' | |
| run: | | |
| # Ensure git-describe works on a tag. | |
| # (checkout@v4 action may have left current tag as | |
| # lightweight instead of annotated. See | |
| # https://github.com/actions/checkout/issues/290) | |
| # | |
| echo github.ref == ${{ github.ref }} ; | |
| git fetch -f origin ${{ github.ref }}:${{ github.ref }} ; | |
| echo git describe now reports $(git describe --always) | |
| - name: setup python | |
| env: ${{matrix.env}} | |
| if: matrix.coverage | |
| uses: actions/setup-python@v6.2.0 | |
| with: | |
| # Use python3.12 to ensure compatbility with coverage tool: | |
| python-version: '3.12' | |
| - name: coverage setup | |
| env: ${{matrix.env}} | |
| if: matrix.coverage | |
| run: | | |
| # Use python3 coverage to match version in flux docker image | |
| sudo apt update ; \ | |
| sudo apt install -yy python3-pip ; \ | |
| pip3 install --upgrade pip ; | |
| pip3 install --upgrade --force-reinstall coverage ; | |
| - name: generate dumpfile from most recent flux-core tag | |
| if: (matrix.create_release != true) | |
| run: | | |
| src/test/create-kvs-dumpfile.sh -d /tmp/dumpfile && | |
| if test -f /tmp/dumpfile/*.bz2; then | |
| cp /tmp/dumpfile/*.tar.bz2 $(pwd)/t/job-manager/dumps/valid | |
| fi | |
| - name: docker buildx | |
| uses: docker/setup-buildx-action@v4.0.0 | |
| if: matrix.needs_buildx | |
| - name: docker-run-checks | |
| timeout-minutes: ${{matrix.timeout_minutes}} | |
| env: ${{matrix.env}} | |
| run: ${{matrix.command}} | |
| - name: tmate debug | |
| if: ${{ !cancelled() && runner.debug }} | |
| uses: mxschmitt/action-tmate@v3.23 | |
| - name: annotate errors | |
| if: failure() || cancelled() | |
| env: ${{matrix.env}} | |
| run: src/test/checks-annotate.sh | |
| - name: coverage report | |
| if: success() && matrix.coverage | |
| env: | |
| DOCKER_REPO: | |
| uses: codecov/codecov-action@v6.0.0 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| flags: ${{matrix.coverage_flags}} | |
| - name: docker deploy | |
| if: success() && matrix.docker_tag | |
| env: ${{matrix.env}} | |
| run: src/test/docker-deploy.sh | |
| - name: create release | |
| id: create_release | |
| if: | | |
| success() | |
| && matrix.create_release | |
| && github.repository == 'flux-framework/flux-core' | |
| env: ${{matrix.env}} | |
| uses: softprops/action-gh-release@v3.0.0 | |
| with: | |
| tag_name: ${{ matrix.tag }} | |
| name: flux-core ${{ matrix.tag }} | |
| prerelease: true | |
| files: flux-core*.tar.gz | |
| body: | | |
| View [Release Notes](https://github.com/${{ github.repository }}/blob/${{ matrix.tag }}/NEWS.md) for flux-core ${{ matrix.tag }} | |
| generate-manifest: | |
| name: Generate docker manifest | |
| runs-on: ubuntu-latest | |
| needs: [ci-checks] | |
| if: (startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/master') | |
| env: | |
| DOCKER_REPO: fluxrm/flux-core | |
| DOCKER_USERNAME: travisflux | |
| DOCKER_PASSWORD: ${{ secrets.DOCKER_HUB_TRAVISFLUX_TOKEN }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| - name: make and push manifest as fluxrm/flux-core | |
| run: | | |
| echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin | |
| src/test/docker-manifest.py | |