Skip to content

Commit d8af9b2

Browse files
wudidapaopaowudidapaopao
authored
Use self hosted test instance (#425)
* test: use new instances * chore: update workflow * chore: update workflow * chore: update workflow * chore: update workflow * chore: update workflow * chore: update workflow * chore: update workflow * chore: update workflow * chore: update workflow * fix: fix docker * fix: fix docker * fix: fix docker * fix: fix docker * fix: fix docker * fix: fix docker * fix: fix docker * chore: update workflow * fix: fix docker * fix: fix docker * test: update workflow * test: update workflow * test: update workflow * test: update workflow * test: update workflow * chore: update workflow * chore: update workflow * test: use new instances for linux x86_64 * fix: fix gh install * chore: optimize static build * chore: empty commit * chore: modify static build dir * test: skip on linux x86_64 due to S3 access permissions * test: use new x86_64 and aarch64 test instances * test: use new x86_64 and aarch64 test instances for cross-compile * test: disable delta lake tests on macOS * test: update test_delta_lake.py * test: install libsqlite3-dev
1 parent c9103f8 commit d8af9b2

12 files changed

+431
-327
lines changed

‎.github/scripts/build-musllinux-arm64.sh‎

Lines changed: 282 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,282 @@
1+
#!/bin/sh
2+
set -e
3+
4+
cd /workspace
5+
6+
# Configure git safe directory in container
7+
apk update
8+
apk add --no-cache git python3 py3-pip py3-setuptools
9+
echo "=== Configure git safe directory ==="
10+
git config --global --add safe.directory /workspace
11+
git describe --tags
12+
python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('version:', get_latest_git_tag())"
13+
14+
# 1. Check system info
15+
echo "=== Container System Info ==="
16+
echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)"
17+
if [ -f /lib/ld-musl-aarch64.so.1 ]; then
18+
echo "musl libc aarch64"
19+
elif [ -f /lib/libc.musl-aarch64.so.1 ]; then
20+
echo "musl libc aarch64"
21+
else
22+
echo "Not musl libc"
23+
fi
24+
echo "Workspace mounted at: /workspace"
25+
ls -la /workspace
26+
27+
# 2. Install build dependencies
28+
echo "=== Installing build dependencies ==="
29+
apk add --no-cache make build-base openssl-dev zlib-dev \
30+
bzip2-dev readline-dev sqlite-dev wget curl llvm \
31+
ncurses-dev xz-dev tk-dev libxml2-dev \
32+
libffi-dev linux-headers
33+
apk add --no-cache make cmake ccache ninja yasm gawk
34+
apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20
35+
36+
# 3. Scan SQLite vulnerabilities
37+
echo "=== Scanning SQLite vulnerabilities ==="
38+
# Install grype
39+
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
40+
grype db update
41+
42+
# Check SQLite vulnerabilities
43+
echo "Scanning SQLite packages for vulnerabilities..."
44+
GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true)
45+
echo "Raw grype output:"
46+
echo "$GRYPE_RAW_OUTPUT"
47+
48+
SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true)
49+
if [ -n "$SQLITE_SCAN_OUTPUT" ]; then
50+
echo "SQLite vulnerabilities found in packages! Build should be reviewed."
51+
echo "SQLite vulnerability details:"
52+
echo "$SQLITE_SCAN_OUTPUT"
53+
else
54+
echo "No SQLite vulnerabilities found"
55+
fi
56+
57+
# 4. Setup Python environments
58+
echo "=== Setting up Python environments ==="
59+
# Setup pyenv
60+
curl https://pyenv.run | bash
61+
export PATH="$HOME/.pyenv/bin:$PATH"
62+
eval "$(pyenv init -)"
63+
64+
# Install Python versions
65+
for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do
66+
echo "Installing Python $version"
67+
pyenv install $version:latest
68+
done
69+
pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14
70+
71+
# Verify installations
72+
echo "Installed versions:"
73+
pyenv versions
74+
for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do
75+
if ! pyenv versions --bare | grep -q "^$version"; then
76+
echo "ERROR: Python $version is not installed!"
77+
exit 1
78+
fi
79+
echo "Python $version is installed"
80+
done
81+
echo "All Python versions verified successfully!"
82+
83+
# Install Rust
84+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
85+
source $HOME/.cargo/env
86+
rustup toolchain install nightly-2025-07-07
87+
rustup component add --toolchain nightly-2025-07-07 rust-src
88+
89+
# Install Python dependencies
90+
for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do
91+
echo "Installing dependencies for Python $version"
92+
pyenv shell $version
93+
python -m pip install --upgrade pip
94+
if [ "$version" = "3.8" ]; then
95+
python -m pip install setuptools tox twine psutil wheel
96+
else
97+
python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel
98+
fi
99+
pyenv shell --unset
100+
done
101+
102+
# Update version for release (if triggered by tag)
103+
echo "GITHUB_REF: '$GITHUB_REF'"
104+
if [ "${GITHUB_REF#refs/tags/v}" != "$GITHUB_REF" ]; then
105+
pyenv shell 3.9
106+
107+
# Install bump-my-version
108+
python -m pip install bump-my-version
109+
TAG_NAME=${GITHUB_REF#refs/tags/v}
110+
bump-my-version replace --new-version $TAG_NAME
111+
echo "Version files updated to $TAG_NAME"
112+
pyenv shell --unset
113+
fi
114+
115+
# 5. Build chdb
116+
echo "=== Building chdb ==="
117+
echo "Timestamp: $(date)"
118+
echo "Current directory: $(pwd)"
119+
echo "Available disk space: $(df -h .)"
120+
121+
# Setup clang
122+
echo "Setting up clang compiler..."
123+
ln -sf /usr/bin/clang-20 /usr/bin/clang
124+
ln -sf /usr/bin/clang++-20 /usr/bin/clang++
125+
export CC=/usr/bin/clang
126+
export CXX=/usr/bin/clang++
127+
echo "Compiler versions:"
128+
$CC --version
129+
$CXX --version
130+
131+
# Build
132+
echo "Starting chdb build with Python 3.8..."
133+
pyenv shell 3.8
134+
python --version
135+
echo "Build start time: $(date)"
136+
bash ./chdb/build-musl.sh
137+
echo "Build end time: $(date)"
138+
139+
# Test
140+
echo "Running smoke test with Python 3.9..."
141+
pyenv shell 3.9
142+
python --version
143+
echo "Test start time: $(date)"
144+
bash -x ./chdb/test_smoke.sh
145+
echo "Test end time: $(date)"
146+
147+
# Check build results
148+
echo "Build results summary:"
149+
ccache -s
150+
echo "chdb directory contents:"
151+
ls -lh chdb
152+
echo "Build artifacts size:"
153+
du -sh chdb
154+
155+
# 6. Create and audit wheels
156+
echo "=== Creating and auditing wheels ==="
157+
echo "Wheel creation start time: $(date)"
158+
echo "Available disk space before wheel build: $(df -h .)"
159+
160+
# Build wheels
161+
echo "Building wheels with Python 3.8..."
162+
pyenv shell 3.8
163+
python --version
164+
echo "Running make wheel..."
165+
make wheel
166+
echo "Wheel build completed at: $(date)"
167+
echo "Initial wheel files:"
168+
ls -lh dist/ || echo "No dist directory yet"
169+
170+
# Install patchelf
171+
echo "Installing patchelf for wheel auditing..."
172+
wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz
173+
tar -xvf patchelf.tar.gz
174+
cp bin/patchelf /usr/bin/
175+
chmod +x /usr/bin/patchelf
176+
echo "patchelf version: $(patchelf --version)"
177+
178+
# Audit wheels
179+
echo "Auditing wheels with Python 3.13..."
180+
pyenv shell 3.13
181+
python --version
182+
python -m pip install auditwheel
183+
echo "auditwheel version: $(auditwheel --version)"
184+
echo "Starting wheel audit at: $(date)"
185+
auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl
186+
echo "Wheel audit completed at: $(date)"
187+
188+
# Clean up non-musllinux wheels
189+
echo "Cleaning up non-musllinux wheels..."
190+
echo "Before cleanup:"
191+
ls -lh dist/
192+
rm -f dist/*-linux_aarch64.whl
193+
echo "After cleanup:"
194+
ls -lh dist/
195+
echo "Final wheel sizes:"
196+
du -sh dist/*
197+
198+
# 7. Test wheels
199+
echo "=== Testing wheels ==="
200+
echo "Wheel testing start time: $(date)"
201+
echo "Available wheels for testing:"
202+
ls -lh dist/*.whl
203+
echo "Wheel file details:"
204+
file dist/*.whl
205+
206+
TOTAL_TESTS=6
207+
CURRENT_TEST=0
208+
TEST_FAILED=false
209+
210+
for version in 3.9 3.10 3.11 3.12 3.13 3.14; do
211+
CURRENT_TEST=$((CURRENT_TEST + 1))
212+
echo "=== Test $CURRENT_TEST/$TOTAL_TESTS: Python $version ==="
213+
echo "Test start time: $(date)"
214+
215+
echo "Switching to Python $version..."
216+
pyenv shell $version
217+
python --version
218+
echo "pip version: $(python -m pip --version)"
219+
220+
echo "Installing chdb wheel..."
221+
python -m pip install dist/*.whl --force-reinstall
222+
echo "Installation completed at: $(date)"
223+
224+
echo "Running basic query test..."
225+
python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')"
226+
227+
echo "Running full test suite..."
228+
if make test; then
229+
echo "Test suite PASSED for Python $version at: $(date)"
230+
else
231+
echo "Test suite FAILED for Python $version at: $(date)"
232+
TEST_FAILED=true
233+
break
234+
fi
235+
236+
pyenv shell --unset
237+
echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully"
238+
echo ""
239+
done
240+
241+
echo "All wheel tests completed at: $(date)"
242+
243+
# Check if any tests failed
244+
if [ "$TEST_FAILED" = true ]; then
245+
echo "ERROR: One or more test suites failed!"
246+
echo "Test failure detected - aborting build process"
247+
exit 1
248+
fi
249+
250+
# Create test success marker file only if all tests passed
251+
echo "All tests passed successfully!"
252+
echo "Creating test success marker..."
253+
touch /workspace/.test_success_marker
254+
echo "Test success marker created at: $(date)"
255+
256+
# 8. Scan chdb libraries
257+
echo "=== Scanning chdb libraries ==="
258+
FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)"
259+
SQLITE_VULNERABILITIES_FOUND=false
260+
261+
for file in $FILES_TO_SCAN; do
262+
if [ -f "$file" ]; then
263+
echo "=== Scanning $file ==="
264+
SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
265+
echo "$SCAN_OUTPUT"
266+
267+
if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
268+
echo "SQLite vulnerability found in $file"
269+
SQLITE_VULNERABILITIES_FOUND=true
270+
fi
271+
fi
272+
done
273+
274+
if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then
275+
echo "SQLite vulnerabilities detected in chdb libraries!"
276+
else
277+
echo "No SQLite vulnerabilities found in chdb libraries"
278+
fi
279+
280+
# Show final results
281+
echo "=== Final wheel files ==="
282+
ls -la ./dist/

‎.github/workflows/build_linux_arm64_wheels-gh.yml‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ on:
2424
jobs:
2525
build_universal_wheel:
2626
name: Build Universal Wheel (Linux arm64)
27-
runs-on: GH-Linux-ARM64
27+
runs-on: [self-hosted, linux, arm64, ubuntu-latest]
2828
if: ${{ !github.event.pull_request.draft }}
2929
steps:
3030
- name: Check CPU capabilities and requirements
@@ -47,11 +47,24 @@ jobs:
4747
sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \
4848
libbz2-dev libreadline-dev wget curl llvm \
4949
libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \
50-
libffi-dev liblzma-dev
50+
libffi-dev liblzma-dev libsqlite3-dev golang-go
51+
- name: Install GitHub CLI
52+
run: |
53+
wget https://github.com/cli/cli/releases/download/v2.82.1/gh_2.82.1_linux_arm64.tar.gz -O gh.tar.gz
54+
tar -xf gh.tar.gz
55+
sudo cp gh_*/bin/gh /usr/local/bin/
56+
sudo chmod +x /usr/local/bin/gh
57+
if ! gh --version; then
58+
echo "ERROR: GitHub CLI installation failed!"
59+
exit 1
60+
fi
61+
echo "GitHub CLI installed successfully"
5162
- name: Scan SQLite vulnerabilities with grype
5263
run: |
5364
# Install grype and required tools
54-
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
65+
mkdir -p $HOME/.local/bin
66+
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b $HOME/.local/bin
67+
echo "$HOME/.local/bin" >> $GITHUB_PATH
5568
sudo apt-get update && sudo apt-get install -y jq lsb-release
5669
5770
# Detect OS distribution info
@@ -60,11 +73,11 @@ jobs:
6073
echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION"
6174
6275
# Update grype vulnerability database
63-
grype db update
76+
$HOME/.local/bin/grype db update
6477
6578
# Check SQLite vulnerabilities in installed packages
6679
echo "Scanning SQLite packages for vulnerabilities..."
67-
GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true)
80+
GRYPE_RAW_OUTPUT=$($HOME/.local/bin/grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true)
6881
echo "Raw grype output:"
6982
echo "$GRYPE_RAW_OUTPUT"
7083
@@ -81,6 +94,8 @@ jobs:
8194
continue-on-error: true
8295
- name: Setup pyenv
8396
run: |
97+
# Remove existing pyenv installation if present
98+
rm -rf $HOME/.pyenv
8499
curl https://pyenv.run | bash
85100
export PATH="$HOME/.pyenv/bin:$PATH"
86101
eval "$(pyenv init -)"
@@ -227,7 +242,7 @@ jobs:
227242
for file in $FILES_TO_SCAN; do
228243
if [ -f "$file" ]; then
229244
echo "=== Scanning $file ==="
230-
SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
245+
SCAN_OUTPUT=$($HOME/.local/bin/grype "$file" 2>/dev/null || true)
231246
echo "$SCAN_OUTPUT"
232247
233248
if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
@@ -270,7 +285,10 @@ jobs:
270285
patchelf --version
271286
- name: Audit wheels
272287
run: |
273-
python3 -m pip install auditwheel
288+
export PATH="$HOME/.pyenv/bin:$PATH"
289+
eval "$(pyenv init -)"
290+
pyenv shell 3.9
291+
python -m pip install auditwheel
274292
auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl
275293
continue-on-error: false
276294
- name: Show files
@@ -301,7 +319,7 @@ jobs:
301319
run: |
302320
export PATH="$HOME/.pyenv/bin:$PATH"
303321
eval "$(pyenv init -)"
304-
pyenv shell 3.8
322+
pyenv shell 3.9
305323
python -m pip install dist/*.whl --force-reinstall
306324
jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb
307325
pyenv shell --unset

0 commit comments

Comments
 (0)