|
| 1 | +name: benchmark-run |
| 2 | + |
| 3 | +# Runs the committed benchmark (`just bench`) and records the numbers. |
| 4 | +# |
| 5 | +# This is a measurement, not a test: nothing here fails the build. Unlike |
| 6 | +# benchmark-ab, which compares two refs inside one job, this just reports the |
| 7 | +# absolute throughput of the current ref alongside Crow for reference. |
| 8 | +# |
| 9 | +# Absolute req/s is only meaningful against other runs on the same runner type, |
| 10 | +# so compare like with like when reading the history. |
| 11 | +# |
| 12 | +# Non-SSL only. Windows is excluded: benchmark/Makefile depends on `nc`, `&` |
| 13 | +# and `kill`, so it would need a PowerShell rewrite first. |
| 14 | + |
| 15 | +on: |
| 16 | + workflow_dispatch: |
| 17 | + inputs: |
| 18 | + duration: |
| 19 | + description: "Load duration per server" |
| 20 | + required: false |
| 21 | + default: "5s" |
| 22 | + connections: |
| 23 | + description: "Concurrent connections" |
| 24 | + required: false |
| 25 | + default: "10" |
| 26 | + crow: |
| 27 | + description: "Also benchmark Crow v1.3.1 for reference" |
| 28 | + type: boolean |
| 29 | + required: false |
| 30 | + default: true |
| 31 | + |
| 32 | +permissions: |
| 33 | + contents: read |
| 34 | + |
| 35 | +jobs: |
| 36 | + bench: |
| 37 | + strategy: |
| 38 | + fail-fast: false |
| 39 | + matrix: |
| 40 | + os: [ubuntu-latest, macos-latest] |
| 41 | + runs-on: ${{ matrix.os }} |
| 42 | + steps: |
| 43 | + - name: checkout |
| 44 | + uses: actions/checkout@v4 |
| 45 | + |
| 46 | + # macos runners ship without Go. |
| 47 | + - name: setup Go |
| 48 | + uses: actions/setup-go@v5 |
| 49 | + with: |
| 50 | + go-version: stable |
| 51 | + |
| 52 | + - name: install bombardier |
| 53 | + run: go install github.com/codesenberg/bombardier@latest |
| 54 | + |
| 55 | + # crow_all.h includes <asio.hpp>, which no runner has out of the box. |
| 56 | + - name: install asio |
| 57 | + if: ${{ inputs.crow }} |
| 58 | + run: | |
| 59 | + if [ "$RUNNER_OS" = "Linux" ]; then |
| 60 | + sudo apt-get update && sudo apt-get install -y libasio-dev |
| 61 | + else |
| 62 | + brew install asio |
| 63 | + fi |
| 64 | +
|
| 65 | + - name: run benchmark |
| 66 | + run: | |
| 67 | + # Without pipefail the `tee` below swallows a build failure and the |
| 68 | + # job reports success having measured nothing. |
| 69 | + set -o pipefail |
| 70 | + export PATH="$(go env GOPATH)/bin:$PATH" |
| 71 | + CROW_FLAGS="-std=c++17" |
| 72 | + if [ "$RUNNER_OS" = "macOS" ]; then |
| 73 | + CROW_FLAGS="$CROW_FLAGS -I$(brew --prefix asio)/include" |
| 74 | + fi |
| 75 | + if [ "${{ inputs.crow }}" = "true" ]; then TARGET=bench-all; else TARGET=bench; fi |
| 76 | + make -C benchmark "$TARGET" \ |
| 77 | + CROW_CXXFLAGS="$CROW_FLAGS" \ |
| 78 | + BENCH="bombardier -c ${{ inputs.connections }} -d ${{ inputs.duration }} localhost:8080" \ |
| 79 | + 2>&1 | tee /tmp/bench.txt |
| 80 | +
|
| 81 | + # pipefail only catches a failed build. Each Makefile recipe ends in |
| 82 | + # `kill`, so a bombardier that never ran still leaves make happy — check |
| 83 | + # that the measurements are actually there. |
| 84 | + - name: check results were produced |
| 85 | + run: | |
| 86 | + expected=1 |
| 87 | + if [ "${{ inputs.crow }}" = "true" ]; then expected=2; fi |
| 88 | + got=$(grep -c "Reqs/sec" /tmp/bench.txt || true) |
| 89 | + if [ "$got" -lt "$expected" ]; then |
| 90 | + echo "::error::expected $expected benchmark result(s), found $got" |
| 91 | + exit 1 |
| 92 | + fi |
| 93 | +
|
| 94 | + - name: record results |
| 95 | + if: always() |
| 96 | + run: | |
| 97 | + { |
| 98 | + echo "## Benchmark (${{ matrix.os }})" |
| 99 | + echo "" |
| 100 | + echo "- ref: \`${{ github.ref_name }}\` (${{ github.sha }})" |
| 101 | + echo "- connections=${{ inputs.connections }} duration=${{ inputs.duration }}" |
| 102 | + echo "" |
| 103 | + echo '```' |
| 104 | + cat /tmp/bench.txt |
| 105 | + echo '```' |
| 106 | + } >> "$GITHUB_STEP_SUMMARY" |
0 commit comments