Skip to content

Commit 19989a2

Browse files
aduh95aduh95
committed
tools: use lighter shell to aggregate benchmark results
Signed-off-by: Antoine du Hamel <duhamelantoine1995@gmail.com>
1 parent bf5c221 commit 19989a2

4 files changed

Lines changed: 185 additions & 215 deletions

File tree

‎.github/workflows/benchmark.yml‎

Lines changed: 31 additions & 210 deletions
Original file line numberDiff line numberDiff line change
@@ -1,196 +1,17 @@
11
name: Benchmark
22

33
on:
4-
workflow_dispatch:
5-
inputs:
6-
repo:
7-
type: string
8-
description: GitHub repository to fetch from (default to the current repo)
9-
pr_id:
10-
type: number
11-
required: true
12-
description: The PR to test
13-
commit:
14-
required: true
15-
type: string
16-
description: The expect HEAD of the PR
17-
category:
18-
required: true
19-
type: string
20-
description: The category (or categories) of tests to run, for example buffers, cluster etc. Maps to a folders in node/benchmark
21-
filter:
22-
type: string
23-
description: A substring to restrict the benchmarks to run in a category. e.g. `net-c2c`
24-
runs:
25-
type: number
26-
default: 30
27-
description: How many times to repeat each benchmark
28-
post-comment:
29-
type: boolean
30-
description: Post a comment linking to the run, with the aggregated result once known.
31-
token:
32-
type: string
33-
description: A GitHub token to post comments cross repository (not recommended)
4+
push:
345

356
permissions:
367
contents: read
378

389
jobs:
39-
post-comment:
40-
if: inputs.post-comment
41-
outputs:
42-
body: ${{ steps.comment.outputs.COMMENT_BODY }}
43-
url: ${{ steps.comment.outputs.COMMENT_URL }}
44-
runs-on: ubuntu-slim
45-
permissions:
46-
pull-requests: write
47-
steps:
48-
- name: Mark token input as sensitive
49-
if: inputs.token != ''
50-
run: echo "::add-mask::${{ inputs.token }}"
51-
- name: Add link to the current run
52-
id: comment
53-
run: |
54-
FILTER_IF_SET=
55-
[ -z "$FILTER" ] || FILTER_IF_SET=" / $FILTER"
56-
COMMENT_BODY="Benchmark GHA (${CATEGORIES}${FILTER_IF_SET}): ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
57-
echo "COMMENT_BODY=$COMMENT_BODY" >> "$GITHUB_OUTPUT"
58-
echo "COMMENT_URL=$(gh pr comment -R "${REPO:-$GITHUB_REPOSITORY}" "$PR_ID" --body "$COMMENT_BODY")" >> "$GITHUB_OUTPUT"
59-
env:
60-
CATEGORIES: ${{ inputs.category }}
61-
FILTER: ${{ inputs.filter }}
62-
GH_TOKEN: ${{ inputs.token || github.token }}
63-
REPO: ${{ inputs.repo }}
64-
PR_ID: ${{ inputs.pr_id }}
65-
build:
66-
strategy:
67-
fail-fast: true
68-
matrix:
69-
include:
70-
- runner: ubuntu-24.04
71-
system: x86_64-linux
72-
- runner: ubuntu-24.04-arm
73-
system: aarch64-linux
74-
- runner: macos-15-intel
75-
system: x86_64-darwin
76-
- runner: macos-latest
77-
system: aarch64-darwin
78-
name: '${{ matrix.system }}: with shared libraries'
79-
runs-on: ${{ matrix.runner }}
80-
steps:
81-
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
82-
with:
83-
repository: ${{ inputs.repo || github.repository }}
84-
ref: refs/pull/${{ inputs.pr_id }}/merge
85-
persist-credentials: false
86-
fetch-depth: 2
87-
88-
- name: Validate PR head and roll back to base commit
89-
run: |
90-
[ "$(git rev-parse HEAD^2)" = "$EXPECTED_SHA" ]
91-
git reset HEAD^ --hard
92-
env:
93-
EXPECTED_SHA: ${{ inputs.commit }}
94-
95-
- uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6
96-
with:
97-
extra_nix_config: sandbox = true
98-
99-
- uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17
100-
with:
101-
# We do not pass any `authToken` to avoid polluting the cache with potentially untrusted code.
102-
name: nodejs
103-
104-
- name: Configure sccache
105-
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
106-
with:
107-
script: |
108-
core.exportVariable('SCCACHE_GHA_VERSION', 'on');
109-
core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on');
110-
core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
111-
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
112-
113-
- name: Build Node.js on the base commit
114-
run: |
115-
nix-shell \
116-
-I nixpkgs=./tools/nix/pkgs.nix \
117-
--pure --keep TAR_DIR --keep FLAKY_TESTS \
118-
--keep SCCACHE_GHA_ENABLED --keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \
119-
--arg useSeparateDerivationForV8 true \
120-
--arg loadJSBuiltinsDynamically false \
121-
--arg ccache "${NIX_SCCACHE:-null}" \
122-
--arg devTools '[]' \
123-
--arg benchmarkTools '[]' \
124-
--run '
125-
make build-ci -j4 V=1
126-
'
127-
mv out/Release/node base_node
128-
129-
- name: Checkout the merge commit
130-
run: git reset FETCH_HEAD --hard
131-
132-
- name: Re-build Node.js on the merge commit
133-
# ccache is disabled here to avoid polluting the cache. Local build outputs should make this build relatively quick anyway.
134-
run: |
135-
nix-shell \
136-
-I nixpkgs=./tools/nix/pkgs.nix \
137-
--pure \
138-
--arg useSeparateDerivationForV8 true \
139-
--arg loadJSBuiltinsDynamically false \
140-
--arg ccache 'null' \
141-
--arg devTools '[]' \
142-
--arg benchmarkTools '[]' \
143-
--run '
144-
make -j4 V=1
145-
'
146-
147-
- name: Run benchmark
148-
run: |
149-
nix-shell \
150-
-I nixpkgs=./tools/nix/pkgs.nix \
151-
--pure --keep FILTER --keep LC_ALL --keep LANG \
152-
--arg loadJSBuiltinsDynamically false \
153-
--arg ccache 'null' \
154-
--arg icu 'null' \
155-
--arg sharedLibDeps '{}' \
156-
--arg devTools '[]' \
157-
--run '
158-
set -o pipefail
159-
./base_node benchmark/compare.js \
160-
--filter "$FILTER" \
161-
--runs ${{ inputs.runs }} \
162-
--old ./base_node --new ./node \
163-
-- ${{ inputs.category }} \
164-
| tee /dev/stderr \
165-
> ${{ matrix.system }}.csv
166-
echo "> [!WARNING] "
167-
echo "> Do not take GHA benchmark results as face value, always confirm them"
168-
echo "> using a dedicated machine, e.g. Jenkins CI."
169-
echo
170-
echo "Benchmark results:"
171-
echo
172-
echo '"'"'```'"'"'
173-
Rscript benchmark/compare.R < ${{ matrix.system }}.csv
174-
echo '"'"'```'"'"'
175-
echo
176-
echo "> [!WARNING] "
177-
echo "> Do not take GHA benchmark results as face value, always confirm them"
178-
echo "> using a dedicated machine, e.g. Jenkins CI."
179-
' | tee /dev/stderr >> "$GITHUB_STEP_SUMMARY"
180-
env:
181-
FILTER: ${{ inputs.filter }}
182-
183-
- name: Upload raw benchmark results
184-
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
185-
with:
186-
name: csv-${{ matrix.system }}
187-
path: ${{ matrix.system }}.csv
188-
18910
aggregate-results:
190-
needs: [build, post-comment]
191-
if: ${{ always() && needs.build.result == 'success' && needs.post-comment.result == (inputs.post-comment && 'success' || 'skipped') }}
11+
# needs: [build, post-comment]
12+
# if: ${{ always() && needs.build.result == 'success' && needs.post-comment.result == (inputs.post-comment && 'success' || 'skipped') }}
19213
name: Aggregate benchmark results
193-
runs-on: ubuntu-slim
14+
runs-on: ubuntu-latest
19415
permissions:
19516
contents: read
19617
pull-requests: write
@@ -199,28 +20,54 @@ jobs:
19920
with:
20021
persist-credentials: false
20122
sparse-checkout: |
23+
why.sh
20224
benchmark/*.R
20325
tools/nix/*.nix
20426
*.nix
20527
sparse-checkout-cone-mode: false
20628

20729
- name: Download benchmark raw results
30+
if: false
20831
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
20932
with:
21033
pattern: csv-*
21134
merge-multiple: true
21235
path: raw-results
36+
repository: nodejs/node
37+
github-token: ${{ secrets.GH_USER_TOKEN }}
38+
run-id: 29243127788
21339

21440
- uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6
21541
with:
21642
extra_nix_config: sandbox = true
21743

44+
- uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17
45+
with:
46+
name: nodejs-test
47+
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
48+
49+
- run: |
50+
set -x
51+
SHELL_DRV=$(nix-instantiate \
52+
-I nixpkgs=./tools/nix/pkgs.nix \
53+
-E '(import <nixpkgs> {}).mkShellNoCC {
54+
buildInputs = import ./tools/nix/benchmarkTools.nix {
55+
withHttpBenchmarkDeps = false;
56+
};
57+
}')
58+
SHELL_STORE_PATH=$(nix-build --no-out-link $SHELL_DRV)
59+
nix --extra-experimental-features nix-command why-depends "$SHELL_STORE_PATH" /nix/store/0axpqqmj1rcanmm2i8vzpd8xxwkmq2jf-gcc-15.2.0
60+
21861
- name: Benchmark results
62+
if: false
21963
run: |
22064
nix-shell \
22165
-I nixpkgs=./tools/nix/pkgs.nix \
22266
--pure \
223-
-E '(import <nixpkgs> {}).mkShell { buildInputs = import ./tools/nix/benchmarkTools.nix { withHttpBenchmarkDeps = false; }; }' \
67+
-E '(import <nixpkgs> {}).mkShellNoCC {
68+
buildInputs = import ./tools/nix/benchmarkTools.nix {
69+
withHttpBenchmarkDeps = false;
70+
}; }' \
22471
--run '
22572
export LC_ALL=C.UTF-8
22673
echo "> [!WARNING] "
@@ -237,29 +84,3 @@ jobs:
23784
echo "> Do not take GHA benchmark results as face value, always confirm them"
23885
echo "> using a dedicated machine, e.g. Jenkins CI."
23986
' | tee /dev/stderr ${{ inputs.post-comment && 'body.txt' || '' }} >> "$GITHUB_STEP_SUMMARY"
240-
- name: Mark token input as sensitive
241-
if: inputs.token != ''
242-
run: echo "::add-mask::${{ inputs.token }}"
243-
- name: Edit comment
244-
if: inputs.post-comment
245-
run: |
246-
{
247-
echo "$COMMENT_BODY"
248-
echo
249-
echo '<details><summary>Results</summary>'
250-
echo
251-
cat body.txt
252-
echo
253-
echo '</details>'
254-
} | jq -Rrcs '{ body: . }' | gh api \
255-
--method PATCH \
256-
-H "Accept: application/vnd.github+json" \
257-
-H "X-GitHub-Api-Version: 2026-03-10" \
258-
"/repos/${REPO:-$GITHUB_REPOSITORY}/issues/comments/${COMMENT_URL##*-}" \
259-
--input -
260-
env:
261-
GH_TOKEN: ${{ inputs.token || github.token }}
262-
REPO: ${{ inputs.repo }}
263-
COMMENT_BODY: ${{ needs.post-comment.outputs.body }}
264-
COMMENT_URL: ${{ needs.post-comment.outputs.url }}
265-
PR_ID: ${{ inputs.pr_id }}

0 commit comments

Comments
 (0)