feat: add zstd compression module for Python 3.14+ (#1866)

Author: mayeut

docker/Dockerfile

@@ -135,18 +135,28 @@ RUN --mount=type=bind,from=static_clang,target=/tmp/cross-compiler,ro \
 FROM build_base AS build_mpdecimal
 COPY build_scripts/build-mpdecimal.sh /build_scripts/
 RUN --mount=type=bind,from=static_clang,target=/tmp/cross-compiler,ro \
-    export MPDECIMAL_ROOT=mpdecimal-4.0.0 && \
-    export MPDECIMAL_HASH=942445c3245b22730fd41a67a7c5c231d11cb1b9936b9c0f76334fb7d0b4468c && \
+    export MPDECIMAL_ROOT=mpdecimal-4.0.1 && \
+    export MPDECIMAL_HASH=96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8 && \
     export MPDECIMAL_DOWNLOAD_URL=https://www.bytereef.org/software/mpdecimal/releases && \
     /tmp/cross-compiler/entrypoint /build_scripts/build-mpdecimal.sh
 
 
+FROM build_base AS build_zstd
+COPY build_scripts/build-zstd.sh /build_scripts/
+RUN --mount=type=bind,from=static_clang,target=/tmp/cross-compiler,ro \
+    export ZSTD_VERSION=1.5.7 && \
+    export ZSTD_HASH=eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 && \
+    export ZSTD_DOWNLOAD_URL=https://github.com/facebook/zstd/releases/download && \
+    /tmp/cross-compiler/entrypoint /build_scripts/build-zstd.sh
+
+
 FROM --platform=${BUILDPLATFORM} ghcr.io/sigstore/cosign/cosign:v${MANYLINUX_COSIGN_VERSION} AS cosign-bin
 
 
 FROM build_base AS build_cpython
 COPY --from=build_tcl_tk /manylinux-buildfs /
 COPY --from=build_mpdecimal /manylinux-buildfs /
+COPY --from=build_zstd /manylinux-buildfs /
 COPY --from=build_sqlite3 /manylinux-buildfs /
 RUN if command -v apk >/dev/null 2>&1; then ldconfig /; else ldconfig; fi
 COPY build_scripts/build-openssl.sh /build_scripts/
@@ -208,6 +218,7 @@ RUN --mount=type=bind,from=static_clang,target=/tmp/cross-compiler,ro \
 FROM runtime_base
 COPY --from=build_tcl_tk /manylinux-rootfs /
 COPY --from=build_mpdecimal /manylinux-rootfs /
+COPY --from=build_zstd /manylinux-rootfs /
 COPY --from=build_sqlite3 /manylinux-rootfs /
 COPY --from=build_git /manylinux-rootfs /
 COPY build_scripts /opt/_internal/build_scripts/
@@ -221,7 +232,8 @@ RUN --mount=type=bind,target=/build_cpython38,from=build_cpython38 \
     --mount=type=bind,target=/build_cpython314,from=build_cpython314 \
     --mount=type=bind,target=/build_cpython314_nogil,from=build_cpython314_nogil \
     mkdir -p /opt/_internal && \
-    cp -rf /build_cpython*/opt/_internal/* /opt/_internal/ && \
+    cp -rf /build_cpython*/opt/_internal/cpython* /opt/_internal/ && \
+    if test -n "$(find /build_cpython314/opt/_internal -maxdepth 1 -name 'openssl*' -print -quit)"; then cp -rf /build_cpython314/opt/_internal/openssl* /opt/_internal/; fi && \
     manylinux-entrypoint /opt/_internal/build_scripts/finalize.sh \
       pp311-pypy311_pp73
 

docker/build_scripts/build-zstd.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Top-level build script called from Dockerfile
+
+# Stop at any error, show all commands
+set -exuo pipefail
+
+# Get script directory
+MY_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+# Get build utilities
+# shellcheck source-path=SCRIPTDIR
+source "${MY_DIR}/build_utils.sh"
+
+# Install a more recent mpdecimal
+check_var "${ZSTD_VERSION}"
+check_var "${ZSTD_HASH}"
+check_var "${ZSTD_DOWNLOAD_URL}"
+ZSTD_ROOT="zstd-${ZSTD_VERSION}"
+
+PREFIX=/opt/_internal/${ZSTD_ROOT%%.*}
+
+fetch_source "${ZSTD_ROOT}.tar.gz" "${ZSTD_DOWNLOAD_URL}/v${ZSTD_VERSION}"
+check_sha256sum "${ZSTD_ROOT}.tar.gz" "${ZSTD_HASH}"
+tar xfz "${ZSTD_ROOT}.tar.gz"
+pushd "${ZSTD_ROOT}/lib"
+# add rpath
+sed -i "s|^Libs:|Libs: -Wl,--enable-new-dtags,-rpath=\${libdir} |g" ./libzstd.pc.in
+DESTDIR=/manylinux-rootfs MT=1 prefix=${PREFIX} make install-includes install-pc install-shared V=1 CPPFLAGS="${MANYLINUX_CPPFLAGS} -DZSTD_MULTITHREAD" CFLAGS="${MANYLINUX_CFLAGS} -DZSTD_MULTITHREAD -pthread -fPIC -fvisibility=hidden" LDFLAGS="${MANYLINUX_LDFLAGS} -shared -pthread"
+popd
+rm -rf "${ZSTD_ROOT}" "${ZSTD_ROOT}.tar.gz"
+
+# Strip what we can
+strip_ /manylinux-rootfs
+
+# Install for build
+mkdir /manylinux-buildfs
+cp -rlf /manylinux-rootfs/* /manylinux-buildfs/
+# copy pkgconfig
+mkdir -p /manylinux-buildfs/usr/local/lib/pkgconfig/
+ln -s "${PREFIX}/lib/pkgconfig/libzstd.pc" /manylinux-buildfs/usr/local/lib/pkgconfig/libzstd.pc
+
+# Clean-up for runtime
+rm -rf "/manylinux-rootfs/${PREFIX}/lib/pkgconfig" "/manylinux-rootfs/${PREFIX}/include"

docker/tests/modules-check.py

@@ -62,6 +62,19 @@ def test_sysconfig(self):
         assert config_vars["LDSHARED"] == f"{cc} -shared", config_vars["LDSHARED"]
         assert config_vars["LDCXXSHARED"] == f"{cxx} -shared", config_vars["LDCXXSHARED"]
 
+    @unittest.skipIf(sys.version_info[:2] < (3, 14), reason="not supported in this version")
+    def test_zstd(self):
+        from compression import zstd
+
+        print(f"{zstd.zstd_version_info=}", end=" ", file=sys.stderr)
+        assert zstd.zstd_version_info[:3] >= (1, 5, 7)
+
+    def test_ssl(self):
+        import ssl
+
+        print(f"{ssl.OPENSSL_VERSION_INFO=}", end=" ", file=sys.stderr)
+        assert ssl.OPENSSL_VERSION_INFO[:3] >= (1, 1, 1)
+
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)

tools/update_native_dependencies.py

@@ -145,6 +145,11 @@ def _update_sqlite(dry_run):
 def _update_with_gh(tool, dry_run):
     repo = {
         "libxcrypt": "besser82/libxcrypt",
+        "zstd": "facebook/zstd",
+    }
+    extension = {
+        "libxcrypt": "tar.xz",
+        "zstd": "tar.gz",
     }
     lines = DOCKERFILE.read_text().splitlines()
     re_ = re.compile(f"^    export {tool.upper()}_VERSION=(?P<version>\\S+) && \\\\$")
@@ -153,13 +158,13 @@ def _update_with_gh(tool, dry_run):
         if match is None:
             continue
         current_version = Version(match["version"])
-        latest_tag = latest(repo[tool], output_format="tag")
+        latest_tag = latest(repo[tool], output_format="tag", having_asset=True)
         latest_version = Version(latest_tag)
         if latest_version > current_version:
             url = re.match(
                 f"^    export {tool.upper()}_DOWNLOAD_URL=(?P<url>\\S+) && \\\\$", lines[i + 2]
             )["url"]
-            sha256 = _sha256(f"{url}/{latest_tag}/libxcrypt-{latest_version}.tar.xz")
+            sha256 = _sha256(f"{url}/{latest_tag}/{tool}-{latest_version}.{extension[tool]}")
             lines[i + 0] = f"    export {tool.upper()}_VERSION={latest_version} && \\"
             lines[i + 1] = f"    export {tool.upper()}_HASH={sha256} && \\"
             message = f"Bump {tool} {current_version} → {latest_version}"
@@ -257,7 +262,7 @@ def main():
             _update_with_root(tool, args.dry_run)
         except Exception as e:
             print(f"::warning::update: {e}\n", file=sys.stderr)
-    for tool in ["libxcrypt"]:
+    for tool in ["libxcrypt", "zstd"]:
         try:
             _update_with_gh(tool, args.dry_run)
         except Exception as e: