Skip to content

Commit dded227

Browse files
pmp-ppmp-p
committed
fix ci oom
1 parent 434611a commit dded227

File tree

11 files changed

+25914
-165
lines changed

11 files changed

+25914
-165
lines changed

‎config‎

Lines changed: 10 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -31,24 +31,12 @@ do
3131
done
3232

3333
export HPY=${HOST_PREFIX}/bin/python${PYBUILD}
34-
35-
if [ -f $HPY ]
36-
then
37-
echo -n # host python has been built successfully
38-
else
39-
export HPY=$(ls ${HOST_PREFIX}/bin/python3.?? 2>/dev/null | sort | tail -n 1)
40-
fi
41-
42-
if [ -f $HPY ]
43-
then
44-
export PYMINOR=$(echo $HPY|cut -d. -f2)
45-
export PIP=${HOST_PREFIX}/bin/pip3.${PYMINOR}
46-
fi
34+
export PIP=${HOST_PREFIX}/bin/pip${PYBUILD}
4735

4836
# cpython build opts
4937
export CPOPTS="-Os -g0 -fPIC"
5038
export CPPFLAGS="-I$PREFIX/include"
51-
export LDFLAGS="-L$PREFIX/lib"
39+
export LDFLAGS="-msoft-float -L$PREFIX/lib"
5240

5341
# module build opts
5442
export CFLDPFX="$CPPFLAGS $LDFLAGS -Wno-unused-command-line-argument"
@@ -59,13 +47,19 @@ then
5947
export QUIET=""
6048
else
6149
export COPTS="-Os -g0 -fPIC"
62-
export QUIET="2>&1 > $PYTHONPYCACHEPREFIX/.log"
50+
if ${VERBOSE:-false}
51+
then
52+
export QUIET=""
53+
else
54+
export QUIET="2>&1 > $PYTHONPYCACHEPREFIX/.log"
55+
fi
6356
fi
6457

6558
#stable
6659
# EMFLAVOUR=latest
6760
#git
68-
EMFLAVOUR=tot
61+
EMFLAVOUR=${EMFLAVOUR:-latest}
62+
6963

7064
export PYDK_PYTHON_HOST_PLATFORM=wasm32-$EMFLAVOUR-emscripten
7165

‎python-wasm-build.sh‎

Lines changed: 0 additions & 64 deletions
This file was deleted.

‎python-wasm-sdk.sh‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ reset
44
DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}"
55

66
export SDKDIR=/opt/python-wasm-sdk
7+
export SDKROOT=/opt/python-wasm-sdk
8+
79
export CIVER=${CIVER:-$DISTRIB}
810
export CI=true
911

@@ -15,11 +17,11 @@ ORIGIN=$(pwd)
1517

1618
# 3.12 3.11 3.10
1719

18-
for PYBUILD in 3.12 3.11
20+
for PYBUILD in 3.11 3.10
1921
do
2022
cd "$ORIGIN"
2123

22-
rm -rf ${SDKDIR}/*
24+
[ -f ${SDKDIR}/dev ] || rm -rf ${SDKDIR}/*
2325

2426
cp -Rf * ${SDKDIR}/
2527

@@ -36,14 +38,14 @@ do
3638

3739
cd ${SDKDIR}
3840
. scripts/cpython-fetch.sh
39-
41+
4042
cd ${SDKDIR}
4143
. support/__EMSCRIPTEN__.sh
4244

43-
. scripts/cpython-build-host.sh
44-
# >/dev/null
45+
. scripts/cpython-build-host.sh 2>&1 >/dev/null
46+
4547
. scripts/cpython-build-host-deps.sh
46-
# >/dev/null
48+
# >/dev/null
4749

4850
# use ./ or emsdk will pollute env
4951
./scripts/emsdk-fetch.sh

‎scripts/cpython-build-emsdk-deps.sh‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@
77
# echo " ${PIMINOR}"|grep -q 11$
88
if true
99
then
10-
HPFX=./devices/x86_64/usr/lib/python3.${PYMINOR}
11-
rm ./devices/emsdk/usr/lib/python3.${PYMINOR}/ensurepip/_bundled/setuptools-*-py3-none-any.whl
12-
cp -Rf $HPFX/setuptool* ./devices/emsdk/usr/lib/python3.${PYMINOR}/
13-
cp -Rf $HPFX/_distutils* ./devices/emsdk/usr/lib/python3.${PYMINOR}/
14-
cp -Rf $HPFX/pkg_resources ./devices/emsdk/usr/lib/python3.${PYMINOR}/
10+
HPFX=./devices/x86_64/usr/lib/python${PYBUILD}
11+
rm ./devices/emsdk/usr/lib/python${PYBUILD}/ensurepip/_bundled/setuptools-*-py3-none-any.whl
12+
cp -Rf $HPFX/setuptool* ./devices/emsdk/usr/lib/python${PYBUILD}/
13+
cp -Rf $HPFX/_distutils* ./devices/emsdk/usr/lib/python${PYBUILD}/
14+
cp -Rf $HPFX/pkg_resources ./devices/emsdk/usr/lib/python${PYBUILD}/
1515
fi
1616

1717
# ../../devices/x86_64/usr/bin/python3-wasm -mpip install .
@@ -68,7 +68,7 @@ else
6868
wget -q -c https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-$WEBP_VER.tar.gz \
6969
&& tar xfz libwebp-$WEBP_VER.tar.gz
7070
pushd libwebp-$WEBP_VER
71-
EMCC_CFLAGS="$ALL" $CNF \
71+
EMCC_CFLAGS="$ALL" CC=emcc $CNF \
7272
--disable-threading --disable-neon --disable-sse2 --enable-libwebpdecoder 2>&1>/dev/null
7373
EMCC_CFLAGS="$ALL" emmake make 2>&1>/dev/null
7474
emmake make install 2>&1>/dev/null
@@ -99,7 +99,7 @@ else
9999
fi
100100

101101
pushd SDL2_image-2.5.1
102-
CFLAGS=$CPOPTS EMCC_CFLAGS="$ALL" $CNF \
102+
CFLAGS=$CPOPTS EMCC_CFLAGS="$ALL" CC=emcc $CNF \
103103
--disable-sdltest --disable-jpg-shared --disable-png-shared
104104
#--disable-tif-shared
105105
EMCC_CFLAGS="$ALL" emmake make
@@ -119,7 +119,7 @@ NCOPTS="--enable-ext-colors --enable-ext-mouse --prefix=$PREFIX --disable-echo -
119119
--with-pkg-config-libdir=$PREFIX/lib/pkgconfig \
120120
--with-termlib --enable-termcap --disable-database"
121121

122-
if false
122+
if true
123123
then
124124
wget -q -c https://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.1.tar.gz && tar xfz ncurses-6.1.tar.gz
125125

‎scripts/cpython-build-emsdk.sh‎

Lines changed: 38 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,10 @@
1111
"
1212

1313

14-
export PYTHON_FOR_BUILD=${PYTHON_FOR_BUILD:-${HOST_PREFIX}/bin/python3}
14+
export PYTHON_FOR_BUILD=${PYTHON_FOR_BUILD:-${HPY}}
1515

1616
# remove old compiler wrapper to avoid conflicts
17-
rm $HOST_PREFIX/bin/cc
18-
17+
[ -f $HOST_PREFIX/bin/cc ] && rm $HOST_PREFIX/bin/cc
1918

2019
. ./scripts/emsdk-fetch.sh
2120

@@ -25,7 +24,7 @@ REBUILD_WASM=${REBUILD_WASMPY:-false}
2524
if $REBUILD || $REBUILD_WASMPY
2625
then
2726
rm -rf build/cpython-wasm/ build/pycache/config.cache
28-
rm build/cpython-wasm/libpython3.${PYMINOR}.a 2>/dev/null
27+
rm build/cpython-wasm/libpython${PYBUILD}.a 2>/dev/null
2928
REBUILD=true
3029
fi
3130

@@ -60,12 +59,12 @@ else
6059

6160
export EMCC_CFLAGS="-O0 -g0 -fPIC"
6261

63-
CFLAGS="-O0 -g0 -fPIC" \
64-
emconfigure $ROOT/src/libffi/configure --host=wasm32-tot-linux\
62+
CFLAGS="-O0 -g0 -fPIC" \
63+
emconfigure $ROOT/src/libffi/configure --host=wasm32-tot-linux\
6564
--prefix=$PREFIX --enable-static --disable-shared --disable-dependency-tracking\
66-
--disable-builddir --disable-multi-os-directory --disable-raw-api --disable-docs >/dev/null
65+
--disable-builddir --disable-multi-os-directory --disable-raw-api --disable-docs
6766

68-
emmake make install >/dev/null
67+
emmake make install
6968

7069
unset EMCC_CFLAGS
7170
popd
@@ -95,8 +94,7 @@ TESTSUITE=$TESTSUITE
9594
"
9695

9796

98-
99-
if [ -f build/cpython-wasm/libpython3.${PYMINOR}.a ]
97+
if [ -f build/cpython-wasm/libpython${PYBUILD}.a ]
10098
then
10199
echo "
102100
* not rebuilding cpython-wasm for [$PYDK_PYTHON_HOST_PLATFORM]
@@ -108,6 +106,15 @@ else
108106
"
109107

110108

109+
if [ -f /lowend ]
110+
then
111+
# yes, i only have a amd200GE with 32G
112+
NPROC=1
113+
export EMSDK_NUM_CORES=1
114+
else
115+
NPROC=$(nproc)
116+
fi
117+
111118

112119
mkdir -p build/cpython-wasm $PREFIX
113120
pushd build/cpython-wasm
@@ -142,8 +149,9 @@ else
142149
_decimal
143150
END
144151

152+
145153
if EMCC_CFLAGS="-sUSE_ZLIB -sUSE_BZIP2" \
146-
emmake make -j$(nproc) WASM_ASSETS_DIR=$(realpath ${PYTHONPYCACHEPREFIX}/empty)@/
154+
emmake make -j$NPROC WASM_ASSETS_DIR=$(realpath ${PYTHONPYCACHEPREFIX}/empty)@/
147155
then
148156
EMCC_CFLAGS="-sUSE_ZLIB -sUSE_BZIP2" \
149157
emmake make WASM_ASSETS_DIR=$(realpath ${PYTHONPYCACHEPREFIX}/empty)@/ install
@@ -161,23 +169,27 @@ EMCC_CFLAGS="-sUSE_ZLIB -sUSE_BZIP2" emmake make WASM_ASSETS_DIR=$(realpath ${PY
161169

162170
popd
163171

164-
mkdir -p ${ROOT}/prebuilt/emsdk
172+
mkdir -p ${SDKROOT}/prebuilt/emsdk/${PYBUILD}/site-packages
173+
mkdir -p ${SDKROOT}/prebuilt/emsdk/${PYBUILD}/lib-dynload
165174

166-
# move them to MEMFS
167-
mv $PREFIX/lib/python3.${PYMINOR}/lib-dynload/* $ROOT/prebuilt/emsdk/site-packages
175+
if [ -d $PREFIX/lib/python${PYBUILD}/lib-dynload ]
176+
then
177+
# move them to MEMFS
178+
mv $PREFIX/lib/python${PYBUILD}/lib-dynload/* ${SDKROOT}/prebuilt/emsdk/${PYBUILD}/lib-dynload/
168179

169-
# specific platform support
170-
cp -Rfv $ROOT/support/__EMSCRIPTEN__.patches/3.${PYMINOR}/. $HOST_PREFIX/lib/python3.${PYMINOR}/
180+
# specific platform support
181+
cp -Rfv $ROOT/support/__EMSCRIPTEN__.patches/${PYBUILD}/. $HOST_PREFIX/lib/python${PYBUILD}/
171182

172-
cp -vf build/cpython-wasm/libpython3.*.a prebuilt/emsdk/
173-
rmdir $PREFIX/lib/python3.${PYMINOR}/lib-dynload
183+
cp -vf build/cpython-wasm/libpython3.*.a prebuilt/emsdk/
184+
rmdir $PREFIX/lib/python${PYBUILD}/lib-dynload
185+
fi
174186
fi
175187

176188
mkdir -p $PYTHONPYCACHEPREFIX/sysconfig
177189

178190

179191
# FIXME: seems CI cannot locate that one with python3-wasm
180-
cp $PREFIX/lib/python3.${PYMINOR}/_sysconfigdata__emscripten_wasm32-emscripten.py $PYTHONPYCACHEPREFIX/sysconfig/_sysconfigdata__emscripten_debug.py
192+
cp $PREFIX/lib/python${PYBUILD}/_sysconfigdata__emscripten_wasm32-emscripten.py $PYTHONPYCACHEPREFIX/sysconfig/_sysconfigdata__emscripten_debug.py
181193
sed -i 's|-Os|-O0|g' $PYTHONPYCACHEPREFIX/sysconfig/_sysconfigdata__emscripten_debug.py
182194
sed -i 's|-g0|-g3|g' $PYTHONPYCACHEPREFIX/sysconfig/_sysconfigdata__emscripten_debug.py
183195

@@ -273,6 +285,10 @@ END
273285

274286
cat > $HOST_PREFIX/bin/python3-wasm <<END
275287
#!/bin/bash
288+
export PYBUILD=\${PYBUILD:-$PYBUILD}
289+
export PYMAJOR=$(echo -n \$PYBUILD|cut -d. -f1)
290+
export PYMINOR=$(echo -n \$PYBUILD|cut -d. -f2)
291+
276292
. $ROOT/${PYDK_PYTHON_HOST_PLATFORM}-shell.sh
277293
278294
# most important
@@ -287,15 +303,15 @@ export PYTHONHOME=$PREFIX
287303
288304
# find sysconfig ( tweaked )
289305
# but still can load dynload and setuptools
290-
PYTHONPATH=$(echo -n ${HOST_PREFIX}/lib/python3.${PYMINOR}/site-packages):\$PYTHONPATH
291-
export PYTHONPATH=$PYTHONPYCACHEPREFIX/sysconfig:$(echo -n ${HOST_PREFIX}/lib/python3.${PYMINOR}/lib-dynload):\$PYTHONPATH
306+
PYTHONPATH=$(echo -n ${HOST_PREFIX}/lib/python/\${PYBUILD}/site-packages):\$PYTHONPATH
307+
export PYTHONPATH=$PYTHONPYCACHEPREFIX/sysconfig:$(echo -n ${HOST_PREFIX}/lib/python\${PYBUILD}/lib-dynload):\$PYTHONPATH
292308
293309
294310
#probably useless
295311
export _PYTHON_HOST_PLATFORM=${PYDK_PYTHON_HOST_PLATFORM}
296312
export PYTHON_FOR_BUILD=${PYTHON_FOR_BUILD}
297313
298-
$HPY -u -B \$@
314+
${HOST_PREFIX}/bin/python\${PYBUILD} -u -B \$@
299315
END
300316

301317
chmod +x $HOST_PREFIX/bin/python3-wasm

‎scripts/cpython-build-host-deps.sh‎

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,22 +14,30 @@ SPY="$HPY setup.py install --single-version-externally-managed --root=/"
1414
$PIP install pip --upgrade
1515

1616

17-
if [ -d $HPFX/site-packages/setuptools ]
18-
then
19-
echo "
20-
* setuptools/_distutils/pkg_resources already migrated
17+
# to remove ctypes deps
18+
$PIP install setuptools --upgrade
19+
20+
21+
HPFX=./devices/x86_64/usr/lib/python${PYBUILD}
22+
rm -rf $HPFX/ensurepip/_bundled/*-py3-none-any.whl
23+
24+
25+
for moveit in setuptools _distutils _distutils_hack pkg_resources
26+
do
27+
if [ -d $HPFX/site-packages/${moveit} ]
28+
then
29+
echo "
30+
* migrating ${moveit}
2131
"
22-
else
23-
# to remove ctypes deps
24-
$PIP install setuptools --upgrade
25-
26-
HPFX=./devices/x86_64/usr/lib/python3.${PYMINOR}
27-
rm $HPFX/ensurepip/_bundled/setuptools-*-py3-none-any.whl
28-
mv $HPFX/site-packages/setuptools* $HPFX/
29-
mv $HPFX/site-packages/_distutils* $HPFX/
30-
mv $HPFX/site-packages/pkg_resources* $HPFX/
31-
fi
32+
rm -rf $HPFX/${moveit} rm -rf $HPFX/${moveit}-*
33+
mv $HPFX/site-packages/${moveit} $HPFX/
34+
mv $HPFX/site-packages/${moveit}-* $HPFX/
35+
fi
36+
done
37+
3238

39+
# https://github.com/aroberge/ideas, for code transformation
40+
$PIP install token-utils
3341

3442
export CC=clang
3543

0 commit comments

Comments
 (0)