Skip to content

Commit 908d381

Browse files
AriouzAriouz
committed
wip: ci-unittest-retagger workflow file
1 parent 68cff18 commit 908d381

File tree

2 files changed

+300
-31
lines changed

2 files changed

+300
-31
lines changed

‎.github/workflows/ci-unittest-retagger.yml‎

Lines changed: 300 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,300 @@
1+
name: Run weekly unittest retagger
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
- cron: '0 0 * * 1'
7+
8+
jobs:
9+
generate-tier1:
10+
runs-on: ubuntu-latest
11+
outputs:
12+
matrix: ${{ steps.set-matrix.outputs.matrix }}
13+
env:
14+
TARGET: tier1
15+
JOBS: python-svm-build|python-unittest-retagger
16+
steps: &generate_matrix
17+
- uses: actions/checkout@v4
18+
- name: Download sjsonnet
19+
run: |
20+
curl -L -o sjsonnet https://github.com/databricks/sjsonnet/releases/download/0.5.7/sjsonnet-0.5.7-linux-x86_64
21+
chmod +x sjsonnet
22+
- name: Extract job matrix
23+
id: set-matrix
24+
run: |
25+
python3 .github/scripts/extract_matrix.py ./sjsonnet ./ci.jsonnet ${TARGET} ${JOBS} > matrix.json
26+
cat matrix.json
27+
echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
28+
29+
generate-tier2:
30+
runs-on: ubuntu-latest
31+
outputs:
32+
matrix: ${{ steps.set-matrix.outputs.matrix }}
33+
env:
34+
TARGET: tier2
35+
JOBS: ${{ inputs.jobs_to_run }}
36+
steps: *generate_matrix
37+
38+
generate-tier3:
39+
runs-on: ubuntu-latest
40+
outputs:
41+
matrix: ${{ steps.set-matrix.outputs.matrix }}
42+
env:
43+
TARGET: tier3
44+
JOBS: ${{ inputs.jobs_to_run }}
45+
steps: *generate_matrix
46+
47+
tier1:
48+
needs: generate-tier1
49+
runs-on: ${{ matrix.os }}
50+
env: ${{ matrix.env }}
51+
strategy:
52+
fail-fast: false
53+
matrix:
54+
include: ${{ fromJson(needs.generate-tier1.outputs.matrix) }}
55+
steps: &buildsteps
56+
57+
- name: Actions/Checkout
58+
uses: actions/checkout@main
59+
with:
60+
path: main
61+
62+
- name: Setup Unix paths like on buildbot CI
63+
if: runner.os != 'Windows'
64+
run: |
65+
echo "PARENT_DIRECTORY=$(pwd)" >> $GITHUB_ENV
66+
- name: Setup Windows paths like on buildbot CI
67+
if: runner.os == 'Windows'
68+
run: |
69+
"PARENT_DIRECTORY=$PWD" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
70+
71+
# Set up Python and packages
72+
- uses: actions/setup-python@v6
73+
if: ${{ matrix.python_version }}
74+
with:
75+
python-version: ${{ matrix.python_version }}
76+
- name: Install python packages on Unix
77+
if: ${{ runner.os != 'Windows' && matrix.python_packages }}
78+
run: |
79+
for pkg in ${{ matrix.python_packages }}; do
80+
pip install "$pkg" || true
81+
done
82+
- name: Install python packages on Windows
83+
if: ${{ runner.os == 'Windows' && matrix.python_packages }}
84+
run: |
85+
$packages = '${{ matrix.python_packages }}' -split ' '
86+
foreach ($pkg in $packages) {
87+
pip install "$pkg"
88+
}
89+
90+
- name: Install system packages on Linux
91+
if: ${{ runner.os == 'Linux' && matrix.system_packages }}
92+
run: |
93+
sudo apt-get -qq update
94+
for pkg in ${{ matrix.system_packages }}; do
95+
sudo apt-get -qq install "$pkg" || true
96+
done
97+
- name: Install system packages on macOS
98+
if: ${{ runner.os == 'macOS' && matrix.system_packages }}
99+
run: |
100+
echo $PATH
101+
brew update
102+
for pkg in ${{ matrix.system_packages }}; do
103+
brew install "$pkg" || true
104+
done
105+
- name: Install system packages on Windows
106+
if: ${{ runner.os == 'Windows' && matrix.system_packages }}
107+
run: |
108+
$packages = '${{ matrix.system_packages }}' -split ' '
109+
foreach ($pkg in $packages) {
110+
try {
111+
choco install $pkg -y
112+
} catch {
113+
Write-Host "Failed to install $pkg, skipping."
114+
}
115+
}
116+
117+
# Setup mx, buildtools and labsjdk
118+
- name: Get mx, buildtools, and labsjdk
119+
if: ${{ runner.os != 'Windows' && matrix.mx_version }}
120+
shell: bash
121+
run: |
122+
git clone https://github.com/graalvm/mx
123+
if [[ "${RUNNER_OS}" == "Windows" ]]; then
124+
./mx/mx.cmd fetch-jdk -A --jdk-id labsjdk-ce-latest
125+
else
126+
./mx/mx fetch-jdk -A --jdk-id labsjdk-ce-latest
127+
fi
128+
git -C mx checkout ${{ matrix.mx_version }}
129+
130+
- name: Setup mx and JAVA_HOME on Unix
131+
if: ${{ runner.os != 'Windows' && matrix.mx_version }}
132+
run: |
133+
echo "$(pwd)/mx/" >> "$GITHUB_PATH"
134+
echo "$(pwd)/.github/scripts/" >> "$GITHUB_PATH"
135+
echo "JAVA_HOME=$HOME/.mx/jdks/labsjdk-ce-latest" >> "$GITHUB_ENV"
136+
echo "JVMCI_VERSION_CHECK=ignore" >> "$GITHUB_ENV"
137+
- name: Setup JAVA_HOME on MacOS
138+
if: ${{ runner.os == 'macOS' && matrix.mx_version }}
139+
run: |
140+
echo "JAVA_HOME=$HOME/.mx/jdks/labsjdk-ce-latest/Contents/Home" >> "$GITHUB_ENV"
141+
- name: Setup mx and JAVA_HOME on Windows
142+
if: ${{ runner.os == 'Windows' && matrix.mx_version }}
143+
run: |
144+
"$PWD/mx" | Out-File -FilePath "$env:GITHUB_PATH" -Append
145+
"$PWD/.github/scripts" | Out-File -FilePath "$env:GITHUB_PATH" -Append
146+
"JAVA_HOME=$HOME/.mx/jdks/labsjdk-ce-latest" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
147+
"JVMCI_VERSION_CHECK=ignore" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
148+
149+
- name: Download artifacts
150+
uses: actions/download-artifact@v5
151+
if: ${{ matrix.require_artifact }}
152+
with:
153+
pattern: ${{ matrix.require_artifact[0] }}
154+
merge-multiple: true
155+
continue-on-error: true
156+
157+
- name: Export artifact paths Linux
158+
if: ${{ matrix.require_artifact }}
159+
run: |
160+
echo "ARTIFACT_PATHS=${{ matrix.require_artifact[1] }}/${{ matrix.require_artifact[0] }}.tar" >> "$GITHUB_ENV"
161+
echo "ARTIFACT_PATH_PREFIX=${{ matrix.require_artifact[1] }}" >> "$GITHUB_ENV"
162+
- name: Export artifact paths Windows
163+
if: ${{ matrix.require_artifact && runner.os == 'Windows'}}
164+
run: |
165+
"ARTIFACT_PATHS=${{ matrix.require_artifact[1] }}/${{ matrix.require_artifact[0] }}.tar" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
166+
"ARTIFACT_PATH_PREFIX=${{ matrix.require_artifact[1] }}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
167+
168+
- name: Install MSBuild
169+
if: ${{ runner.os == 'Windows' }}
170+
uses: microsoft/setup-msbuild@v2
171+
172+
- name: Process matrix downloads
173+
if: ${{ matrix.downloads_steps }}
174+
run: |
175+
${{ matrix.downloads_steps }}
176+
177+
- name: Setup
178+
working-directory: main
179+
if: ${{ matrix.setup_steps }}
180+
run: |
181+
${{ matrix.setup_steps }}
182+
183+
- name: Check disk space
184+
run: df -h
185+
186+
- name: Run on Unix
187+
working-directory: main
188+
if: ${{ matrix.run_steps && runner.os != 'Windows' }}
189+
run: |
190+
env
191+
${{ matrix.run_steps }}
192+
- name: Run on Windows
193+
if: ${{ matrix.run_steps && runner.os == 'Windows' }}
194+
shell: cmd
195+
run: |
196+
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
197+
${{ matrix.run_steps }}
198+
199+
- name: Tar artifacts
200+
if: ${{ matrix.provide_artifact }}
201+
shell: bash
202+
working-directory: main
203+
run: |
204+
ls && pwd
205+
tar cf ${{ matrix.provide_artifact[0] }}.tar ${{ matrix.provide_artifact[1] }}
206+
- name: Upload artifacts
207+
if: ${{ matrix.provide_artifact }}
208+
uses: actions/upload-artifact@v5
209+
with:
210+
name: ${{ matrix.provide_artifact[0] }}
211+
path: main/${{ matrix.provide_artifact[0] }}.tar
212+
retention-days: 1
213+
214+
- name: Upload logs
215+
if: ${{ matrix.logs }}
216+
uses: actions/upload-artifact@v5
217+
with:
218+
name: ${{ format('{0}_logs', matrix.name) }}
219+
path: |
220+
${{ matrix.logs }}
221+
retention-days: 15
222+
223+
tier2:
224+
if: ${{ success() || inputs.jobs_to_run }}
225+
needs: [generate-tier2, tier1]
226+
runs-on: ${{ matrix.os }}
227+
env: ${{ matrix.env }}
228+
strategy:
229+
fail-fast: false
230+
matrix:
231+
include: ${{ fromJson(needs.generate-tier2.outputs.matrix) }}
232+
steps: *buildsteps
233+
234+
tier3:
235+
if: ${{ success() || inputs.jobs_to_run }}
236+
needs: [generate-tier3, tier2]
237+
runs-on: ${{ matrix.os }}
238+
env: ${{ matrix.env }}
239+
strategy:
240+
fail-fast: false
241+
matrix:
242+
include: ${{ fromJson(needs.generate-tier3.outputs.matrix) }}
243+
steps: *buildsteps
244+
245+
merge_all_reports:
246+
if: ${{ success() || inputs.jobs_to_run }}
247+
needs: [tier3]
248+
runs-on: ubuntu-latest
249+
strategy:
250+
fail-fast: false
251+
matrix:
252+
include: ${{ fromJson(needs.generate-tier3.outputs.matrix) }}
253+
steps:
254+
- *buildsteps
255+
- name: Download reports
256+
uses: actions/download-artifact@v5
257+
with:
258+
pattern: python-unittest-retagger*
259+
merge-multiple: true
260+
continue-on-error: true
261+
- name: Merge retagger reports
262+
workdir: main
263+
run: |
264+
mkdir -p ../retagger-reports
265+
sh -c mv retagger-report*.json ../retagger-reports
266+
cd ../retagger-reports
267+
python3 ../main/.github/scripts/merge_retagger_results.py
268+
cd ../main
269+
python3 ./graalpython/com.oracle.graal.python.test/src/runner.py merge-tags-from-report ../retagger-reports/reports-merged.json
270+
sh -c git diff >> reports_diff
271+
272+
- name: Export reports diff file
273+
uses: actions/upload-artifact@v5
274+
with:
275+
name: retagger.diff
276+
path: main/diff_reports
277+
retention-days: 1
278+
279+
# {
280+
# packages: {
281+
# mx: "7.34.1",
282+
# python3: "==3.12.8",
283+
# },
284+
# publishArtifacts: [
285+
# {
286+
# name: "retagger.diff",
287+
# dir: ".",
288+
# patterns: ["diff_reports"]
289+
# }
290+
# ],
291+
# run: [
292+
# ["mkdir", "-p", "../retagger-reports"],
293+
# ["sh", "-c", "mv retagger-report*.json ../retagger-reports"],
294+
# ["cd", "../retagger-reports"],
295+
# ["python3", "../main/.github/scripts/merge_retagger_results.py"],
296+
# ["cd", "../main"],
297+
# ["python3", "./graalpython/com.oracle.graal.python.test/src/runner.py", "merge-tags-from-report", "../retagger-reports/reports-merged.json"],
298+
# ["sh", "-c", "git diff >> diff_reports"],
299+
# ]
300+
# },

‎ci.jsonnet‎

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -548,37 +548,6 @@
548548
["git", "push", "--force", "origin", "published"],
549549
]
550550
},
551-
{
552-
name: "python-unittest-retagger-merge",
553-
targets: ["tier3"],
554-
capabilities: ["linux", "amd64"],
555-
packages: {
556-
mx: "7.34.1",
557-
python3: "==3.12.8",
558-
},
559-
requireArtifacts: [
560-
{
561-
name: "python-unittest-retagger*",
562-
dir: ".",
563-
}
564-
],
565-
publishArtifacts: [
566-
{
567-
name: "retagger.diff",
568-
dir: ".",
569-
patterns: ["diff_reports"]
570-
}
571-
],
572-
run: [
573-
["mkdir", "-p", "../retagger-reports"],
574-
["sh", "-c", "mv retagger-report*.json ../retagger-reports"],
575-
["cd", "../retagger-reports"],
576-
["python3", "../main/.github/scripts/merge_retagger_results.py"],
577-
["cd", "../main"],
578-
["python3", "./graalpython/com.oracle.graal.python.test/src/runner.py", "merge-tags-from-report", "../retagger-reports/reports-merged.json"],
579-
["sh", "-c", "git diff >> diff_reports"],
580-
]
581-
},
582551
],
583552
}
584553

0 commit comments

Comments
 (0)