Fix Windows build

* Non-fixed build cpu/mem allocation while in github ci

* add psutil python package to ci run

* Enable Windows jobs

* Merge tags from new retagger char

Author: Ariouz

.github/scripts/extract_matrix.py

@@ -128,10 +128,12 @@ def get_download_steps(self, key: str, version: str) -> str:
         filename = download_link.split('/')[-1]
 
         if self.runs_on == "windows-latest":
-            return (f"""Invoke-WebRequest -Uri {download_link} -OutFile {filename}
+            return (f"""
+            Invoke-WebRequest -Uri {download_link} -OutFile {filename}
             $dirname = (& tar -tzf {filename} | Select-Object -First 1).Split('/')[0]
             tar -xzf {filename}
-            Add-Content $env:GITHUB_ENV "{key}=$(Resolve-Path $dirname)""")
+            Add-Content $env:GITHUB_ENV "{key}=$(Resolve-Path $dirname)"
+            """)
 
         return (f"wget -q {download_link} && "
             f"dirname=$(tar -tzf {filename} | head -1 | cut -f1 -d '/') && "
@@ -293,6 +295,7 @@ def get_tagged_jobs(buildspec, target, filter=None):
 
 
 def main(jsonnet_bin, ci_jsonnet, target, filter=None, indent=False):
+
     result = subprocess.check_output([jsonnet_bin, ci_jsonnet], text=True)
     buildspec = json.loads(result)
     tagged_jobs = get_tagged_jobs(buildspec, target, filter=filter)

.github/workflows/ci-matrix-gen.yml

@@ -126,7 +126,7 @@ jobs:
 
       # Setup mx, buildtools and labsjdk
       - name: Get mx, buildtools, and labsjdk
-        if: ${{ runner.os != 'Windows' && matrix.mx_version }}
+        if: ${{ matrix.mx_version }}
         shell: bash
         run: |
           git clone https://github.com/graalvm/mx
@@ -190,8 +190,59 @@ jobs:
         run: |
           ${{ matrix.setup_steps }}
 
-      - name: Check disk space
-        run: df -h
+      - name: Check disk space and memory usage
+        if: ${{ runner.os == 'macOS' }}
+        run: |
+          echo "before purge"
+          df -h
+          pagesize=$(sysctl -n hw.pagesize)
+          vm_stat
+          vm_stat | awk -v ps=$pagesize '
+            /Pages free/      {free=$3}
+            /Pages active/    {active=$3}
+            /Pages inactive/  {inactive=$3}
+            /Pages speculative/ {speculative=$3}
+            /Pages wired/     {wired=$4}
+            END {
+              used=(active+inactive+wired+speculative)*ps/1024/1024;
+              free=free*ps/1024/1024;
+              printf "Used: %.2f MB\nFree: %.2f MB\n", used, free
+            }'
+          ps aux | sort -nrk 4 | head
+          sudo purge
+          echo "after purge"
+          pagesize=$(sysctl -n hw.pagesize)
+          vm_stat
+          vm_stat | awk -v ps=$pagesize '
+            /Pages free/      {free=$3}
+            /Pages active/    {active=$3}
+            /Pages inactive/  {inactive=$3}
+            /Pages speculative/ {speculative=$3}
+            /Pages wired/     {wired=$4}
+            END {
+              used=(active+inactive+wired+speculative)*ps/1024/1024;
+              free=free*ps/1024/1024;
+              printf "Used: %.2f MB\nFree: %.2f MB\n", used, free
+            }'
+
+
+      - name: Free resources on macOS
+        run: |
+          echo "Disabling Spotlight indexing..."
+          sudo mdutil -a -i off
+          sudo mdutil -a -d
+          sudo launchctl bootout system /System/Library/LaunchDaemons/com.apple.metadata.mds.plist
+          sudo pkill -f "mds|mds_stores|mdworker_shared|Spotlight|corespotlightd"
+
+          echo "Forcing disk cache to be purged..."
+          sudo purge
+
+          echo "Top processes sorted by memory usage:"
+          ps -emo pid,pmem,pcpu,comm | head 2>/dev/null
+          echo ""
+          memory_pressure
+          echo ""
+        if: ${{ runner.os == 'macOS' }}
 
       - name: Run on Unix
         working-directory: main
@@ -200,6 +251,7 @@ jobs:
           env
           ${{ matrix.run_steps }}
       - name: Run on Windows
+        working-directory: main
         if: ${{ matrix.run_steps && runner.os == 'Windows' }}
         shell: cmd
         run: |

ci.jsonnet

@@ -244,8 +244,8 @@
         "python-unittest-retagger": ut_retagger + platform_spec(no_jobs) + batches(RETAGGER_SPLIT) + platform_spec({
             "linux:amd64:jdk-latest"     : tier2        + require(GPY_NATIVE_STANDALONE),
             "linux:aarch64:jdk-latest"   : tier3        + require(GPY_NATIVE_STANDALONE),
-            "darwin:aarch64:jdk-latest"  : weekly    + t("20:00:00"),
-            "windows:amd64:jdk-latest"   : weekly    + t("20:00:00"),
+            "darwin:aarch64:jdk-latest"  : tier3        + require(GPY_NATIVE_STANDALONE),
+            "windows:amd64:jdk-latest"   : tier3        + require(GPY_NATIVE_STANDALONE),
         }),
         "python-coverage-jacoco-tagged": cov_jacoco_tagged + batches(COVERAGE_SPLIT) + platform_spec(no_jobs) + platform_spec({
             "linux:amd64:jdk21"          : weekly    + t("20:00:00"),

ci/graal/ci/common.jsonnet

@@ -150,6 +150,7 @@ local common_json = import "../common.json";
       packages+: {
         python3: "==3.8.10",
         "pip:ninja_syntax": common_json.pip.ninja_syntax,
+        "pip:psutil": "==7.1.3",
         mx: common_json.mx_version,
       },
       python_version: "3", # To use the correct virtualenv

ci/python-gate.libsonnet

@@ -160,7 +160,7 @@
         },
         windows: {
             common: {
-                PATH: "$MAVEN_HOME\\bin;$PATH",
+                PATH: "$MAVEN_HOME\\bin;$PATH;C:\\hostedtoolcache\\windows\\Python\\3.12.3\\x64\\Scripts;C:\\hostedtoolcache\\windows\\Python\\3.12.3\\x64;C:\\Program Files\\PowerShell\\7;C:\\windows\\system32;C:\\windows;C:\\windows\\System32\\Wbem;C:\\windows\\System32\\WindowsPowerShell\\v1.0\\;C:\\windows\\System32\\OpenSSH\\;C:\\Program Files\\dotnet\\;C:\\Program Files\\Microsoft SQL Server\\150\\Tools\\Binn\\;C:\\Program Files\\Git\\cmd;C:\\Program Files\\Git\\usr\\bin;C:\\hostedtoolcache\\windows\\Ruby\\3.2.3\\x64\\bin;C:\\hostedtoolcache\\windows\\Go\\1.22.2\\x64\\bin;C:\\hostedtoolcache\\windows\\Node\\20.11.1\\x64\\;C:\\npm\\prefix;C:\\Program Files\\Amazon\\AWSCLIV2\\;C:\\Program Files\\Amazon\\SessionManagerPlugin\\bin\\;C:\\Program Files\\LLVM\\bin;C:\\Program Files\\CMake\\bin;C:\\Program Files\\OpenSSL\\bin;C:\\msys64\\usr\\bin;C:\\hostedtoolcache\\windows\\php\\8.2.15\\x64;C:\\Program Files\\Azure DevOps CLI\\bin;C:\\hostedtoolcache\\windows\\R\\4.3.3\\x64\\bin;C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\Enterprise\\Common7\\IDE\\Extensions\\Microsoft\\NuGet;",
             },
             amd64: {},
             aarch64: {},

graalpython/com.oracle.graal.python.test/src/runner.py

@@ -1392,7 +1392,7 @@ def main_merge_tags(args):
             test_file,
             results,
             tag_platform=args.platform,
-            untag_failed= os.environ.get("GITHUB_CI") is not None,
+            untag_failed=os.environ.get("GITHUB_CI") is not None,
             untag_skipped=True,
             untag_missing=True,
         )

mx.graalpython/mx_graalpython.py

@@ -37,6 +37,7 @@
 import subprocess
 import sys
 import time
+import psutil
 from functools import wraps
 from pathlib import Path
 from textwrap import dedent
@@ -122,6 +123,7 @@ def get_boolean_env(name, default=False):
 _COLLECTING_COVERAGE = False
 
 CI = get_boolean_env("CI")
+GITHUB_CI = get_boolean_env("GITHUB_CI")
 WIN32 = sys.platform == "win32"
 BUILD_NATIVE_IMAGE_WITH_ASSERTIONS = get_boolean_env('BUILD_WITH_ASSERTIONS', CI)
 BYTECODE_DSL_INTERPRETER = get_boolean_env('BYTECODE_DSL_INTERPRETER', False)
@@ -250,6 +252,22 @@ def _is_overridden_native_image_arg(prefix):
     return any(arg.startswith(prefix) for arg in extras)
 
 
+def github_ci_build_args():
+    total_mem = psutil.virtual_memory().total / (1024 ** 3)
+    min_bound = 8
+    max_mem = 14*1024
+    min_mem = int(1024 * (total_mem if total_mem < min_bound else total_mem * .9))
+    os_cpu = os.cpu_count() or int(os.environ.get("NUMBER_OF_PROCESSORS", 1)) or 1
+       
+    build_mem = min(min_mem, max_mem)
+    parallelism = os_cpu if os_cpu >= 4 and build_mem >= min_bound*1024 else 1
+    
+    return ["-Ob",
+            # f"-J-Xms{build_mem}m",
+            f"-J-Xms7g",
+            f"--parallelism={parallelism}"
+        ]
+
 def libpythonvm_build_args():
     build_args = []
     build_args += bytecode_dsl_build_args()
@@ -832,7 +850,7 @@ def graalpy_standalone_home(standalone_type, enterprise=False, dev=False, build=
     mx_args = ['-p', SUITE.dir, *(['--env', env_file] if env_file else [])]
 
     print(f"[DEBUG] GITHUB_CI env: {os.environ.get('GITHUB_CI')}")
-    if os.environ.get("GITHUB_CI"):
+    if GITHUB_CI:
         print("[DEBUG] Running in GitHub Ci")
         mx_args.append("--extra-image-builder-argument=-Ob")
     else: