[GR-59373] Merge GraalPy public and internal wheelbuilder.

.github/workflows/build-windows-amd64-wheels.yml

@@ -2,141 +2,34 @@ name: build-windows-amd64-wheels
 'on':
   workflow_dispatch:
     inputs:
-      name:
+      packages:
         type: string
-        description: Pkg to build (empty for all)
+        description: Pkgs to build (comma-separated, empty for all)
         required: false
-      graalpy:
+      graalpy_url:
         type: string
-        description: GraalPy download url prefix (empty for default)
-        required: false
+        description: GraalPy download url
+        required: true
 jobs:
-  numpy-windows-amd64:
+  build_wheels:
     runs-on: windows-latest
-    if: ${{ !cancelled() && (inputs.name == '' || inputs.name == 'numpy') }}
+    env:
+      PACKAGES_TO_BUILD: ${{ inputs.packages }}
     steps:
       - uses: ilammy/msvc-dev-cmd@v1
-      - name: Install dependencies
-        run: |
-          Invoke-WebRequest https://kumisystems.dl.sourceforge.net/project/gnuwin32/patch/2.5.9-7/patch-2.5.9-7-bin.zip -OutFile patch.zip
-          Expand-Archive patch.zip -DestinationPath ../patch -Force
       - name: Checkout
         uses: actions/checkout@main
-      - name: Setup custom GraalPy
-        if: inputs.graalpy != ''
-        run: |-
-          $ProgressPreference = 'SilentlyContinue'
-          Invoke-WebRequest ${{ inputs.graalpy }}-windows-amd64.zip -OutFile graalpy-windows-amd64.zip
-          Expand-Archive graalpy-windows-amd64.zip
-          mv graalpy-windows-amd64/* graalpy
-          graalpy/bin/graalpy.exe -s -m ensurepip
-          graalpy/bin/graalpy.exe -m pip install wheel
-      - name: Setup GraalPy
+      - name: Setup Python
         uses: actions/setup-python@v5
-        if: inputs.graalpy == ''
         with:
-          python-version: graalpy24.1
-      - name: Setup local GraalPy venv
-        if: inputs.graalpy == ''
-        run: python -m venv graalpy
-      - name: Build wheel
-        run: |-
-          $env:PIP_FIND_LINKS=$PWD
-          $env:PATH+=";$PWD\graalpy\bin;$PWD\graalpy\Scripts;$PWD\..\patch\bin"
-          graalpy/bin/graalpy -m pip wheel --find-links $PWD numpy
-      - name: Store wheels
-        uses: actions/upload-artifact@main
-        with:
-          name: numpy-windows-amd64
-          path: numpy*.whl
-          if-no-files-found: error
-  PyYAML-windows-amd64:
-    runs-on: windows-latest
-    needs: numpy-windows-amd64
-    if: ${{ !cancelled() && (inputs.name == '' || inputs.name == 'PyYAML') }}
-    steps:
-      - uses: ilammy/msvc-dev-cmd@v1
-      - name: Install dependencies
+          python-version: 3.12
+      - name: Build wheels
         run: |
-          Invoke-WebRequest https://kumisystems.dl.sourceforge.net/project/gnuwin32/patch/2.5.9-7/patch-2.5.9-7-bin.zip -OutFile patch.zip
-          Expand-Archive patch.zip -DestinationPath ../patch -Force
-      - name: Checkout
-        uses: actions/checkout@main
-      - name: Setup custom GraalPy
-        if: inputs.graalpy != ''
-        run: |-
-          $ProgressPreference = 'SilentlyContinue'
-          Invoke-WebRequest ${{ inputs.graalpy }}-windows-amd64.zip -OutFile graalpy-windows-amd64.zip
-          Expand-Archive graalpy-windows-amd64.zip
-          mv graalpy-windows-amd64/* graalpy
-          graalpy/bin/graalpy.exe -s -m ensurepip
-          graalpy/bin/graalpy.exe -m pip install wheel
-      - name: Setup GraalPy
-        uses: actions/setup-python@v5
-        if: inputs.graalpy == ''
-        with:
-          python-version: graalpy24.1
-      - name: Setup local GraalPy venv
-        if: inputs.graalpy == ''
-        run: python -m venv graalpy
-      - name: Download artifacts from numpy
-        uses: actions/download-artifact@main
-        continue-on-error: true
-        with:
-          name: numpy-windows-amd64
-      - name: Build wheel
-        run: |-
-          $env:PIP_FIND_LINKS=$PWD
-          $env:PATH+=";$PWD\graalpy\bin;$PWD\graalpy\Scripts;$PWD\..\patch\bin"
-          graalpy/bin/graalpy -m pip wheel --find-links $PWD PyYAML
-      - name: Store wheels
-        uses: actions/upload-artifact@main
-        with:
-          name: PyYAML-windows-amd64
-          path: PyYAML*.whl
-          if-no-files-found: error
-  ujson-windows-amd64:
-    runs-on: windows-latest
-    needs: PyYAML-windows-amd64
-    if: ${{ !cancelled() && (inputs.name == '' || inputs.name == 'ujson') }}
-    steps:
-      - uses: ilammy/msvc-dev-cmd@v1
-      - name: Install dependencies
-        run: |
-          Invoke-WebRequest https://kumisystems.dl.sourceforge.net/project/gnuwin32/patch/2.5.9-7/patch-2.5.9-7-bin.zip -OutFile patch.zip
-          Expand-Archive patch.zip -DestinationPath ../patch -Force
-      - name: Checkout
-        uses: actions/checkout@main
-      - name: Setup custom GraalPy
-        if: inputs.graalpy != ''
-        run: |-
-          $ProgressPreference = 'SilentlyContinue'
-          Invoke-WebRequest ${{ inputs.graalpy }}-windows-amd64.zip -OutFile graalpy-windows-amd64.zip
-          Expand-Archive graalpy-windows-amd64.zip
-          mv graalpy-windows-amd64/* graalpy
-          graalpy/bin/graalpy.exe -s -m ensurepip
-          graalpy/bin/graalpy.exe -m pip install wheel
-      - name: Setup GraalPy
-        uses: actions/setup-python@v5
-        if: inputs.graalpy == ''
-        with:
-          python-version: graalpy24.1
-      - name: Setup local GraalPy venv
-        if: inputs.graalpy == ''
-        run: python -m venv graalpy
-      - name: Download artifacts from PyYAML
-        uses: actions/download-artifact@main
-        continue-on-error: true
-        with:
-          name: PyYAML-windows-amd64
-      - name: Build wheel
-        run: |-
-          $env:PIP_FIND_LINKS=$PWD
-          $env:PATH+=";$PWD\graalpy\bin;$PWD\graalpy\Scripts;$PWD\..\patch\bin"
-          graalpy/bin/graalpy -m pip wheel --find-links $PWD ujson
+          $env:PATH+=";C:\Program Files\Git\usr\bin"
+          python3 scripts/wheelbuilder/build_wheels.py ${{ inputs.graalpy_url }}
       - name: Store wheels
         uses: actions/upload-artifact@main
         with:
-          name: ujson-windows-amd64
-          path: ujson*.whl
+          name: wheels
+          path: wheelhouse/*.whl
           if-no-files-found: error

CHANGELOG.md

@@ -13,6 +13,7 @@ language runtime. The main focus is on user-observable behavior of the engine.
   * This means all Python methods of these types are available on the corresponding foreign objects, which behave as close as possible as if they were Python objects.
   * See [the documentation](https://github.com/oracle/graalpython/blob/master/docs/user/Interoperability.md#interacting-with-foreign-objects-from-python-scripts) for more information.
 * Remove support for running with Sulong managed both in embeddings as well as through the `graalpy-managed` launcher.
+* Rewrite wheelbuilder to be easier to use and contribute to. This version is now the same we run internally to build publishable wheels for some platforms we support, so the community can build the same wheels on their own hardware easily if desired.
 
 ## Version 24.1.0
 * GraalPy is now considered stable for pure Python workloads. While many workloads involving native extension modules work, we continue to consider them experimental. You can use the command-line option `--python.WarnExperimentalFeatures` to enable warnings for such modules at runtime. In Java embeddings the warnings are enabled by default and you can suppress them by setting the context option 'python.WarnExperimentalFeatures' to 'false'.

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "947e97f768b6e4a5868c3afa76617c2b2f72ff33" }
+{ "overlay": "a53f75105e24ff44381970de172a8462fa45b9a7" }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/common/CExtContext.java

@@ -59,6 +59,7 @@
 import org.graalvm.shadowed.com.ibm.icu.impl.Punycode;
 import org.graalvm.shadowed.com.ibm.icu.text.StringPrepParseException;
 
+import com.oracle.graal.python.builtins.PythonOS;
 import com.oracle.graal.python.builtins.objects.cext.capi.CApiContext;
 import com.oracle.graal.python.builtins.objects.cext.common.CExtCommonNodes.CheckFunctionResultNode;
 import com.oracle.graal.python.builtins.objects.cext.common.LoadCExtException.ApiInitException;
@@ -309,7 +310,10 @@ public static Object loadCExtModule(Node location, PythonContext context, Module
                 String message = "Loading C extension module %s from '%s'. Support for the Python C API is considered experimental.";
                 if (!runViaLauncher) {
                     message += " See https://www.graalvm.org/latest/reference-manual/python/Native-Extensions/#embedding-limitations for the limitations. " +
-                                    "You can suppress this warning by setting the context option 'python.WarnExperimentalFeatures' to 'false'";
+                                    "You can suppress this warning by setting the context option 'python.WarnExperimentalFeatures' to 'false'.";
+                }
+                if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {
+                    message += " If loading binary wheels fails, please make sure you installed the latest Microsoft Visual C++ Redistributable from https://aka.ms/vs/17/release/vc_redist.x64.exe.";
                 }
                 getLogger().warning(message.formatted(spec.name, spec.path));
             }

pyproject.toml

@@ -8,4 +8,4 @@ required-version = '23'
 # Disables the black formatter in this repository
 # If individual suites want to enable the formatter, they can create a
 # pyproject.toml with their own configuration in their suite folder
-force-exclude = '.*'
+force-exclude = '.*graalpy.*|.*scripts/[^w].*py'

scripts/wheelbuilder/README.md

@@ -1,10 +1,10 @@
 # Scripts to build wheels for GraalPy.
 
 [GraalPy](https://github.com/oracle/graalpython) is compatible with many Python libraries, including those that extend the Python runtime with native code.
-However, implemented in Java and thus binary incompatible with existing extensions, users of native Python extension libraries such as NumPy, SciPy, or PyTorch have to build their own binaries when installing these libraries.
+However, implemented in Java and thus binary incompatible with existing extensions, users of native Python extension libraries such as NumPy, SciPy, or PyTorch have to build their own binaries when installing these libraries if neither the project nor the GraalPy team provides prebuilt wheels.
 For many libraries, this means installing additional build dependencies and sitting through long and resource-intensive compilation processes.
 
-This project is meant to be a place for the community to collect build recipes for as many popular packages as possible that can then be built once with GitHub Actions for each major release of GraalPy.
+This project is meant to be a place for the community to collect build recipes for as many popular packages as possible that can then be built individually or in CI/CD systems like GitHub Actions.
 
 ## Quickstart
 
@@ -16,42 +16,20 @@ This project is meant to be a place for the community to collect build recipes f
 
 4. Click on "Run workflow".
    You can enter a package name or build all packages.
-   See [the spec list](../../../../blob/master/scripts/wheelbuilder/generate_workflow.py) for which packages are available.
+   See [the platform subfolders](../../../../blob/master/scripts/wheelbuilder/) for which packages have buildscripts.
 
    ![](guide02.png)
 
 ## How to contribute
 
-There should be only one relevant file, `generate_workflow.py`.
-In it we collect `BuildSpec` objects that define how to build a particular package.
-Many packages do not need special definitions, just a name and maybe which platforms to build it for.
-System package dependencies can be specified by platform where needed.
-Dependencies between specs are not strictly necessary, but can reduce the overall build times and thus resource usage of GitHub Action runners.
-
-Changes to `generate_workflow.py` are reflected in the build specs by running the file.
-It creates GitHub Action workflow files, one for each platform, and a giant one with all jobs.
+We collect simple build scripts per platform and package in the `linux`, `darwin`, and `win32` subdirectories.
+The format is simply the package name followed by `.sh` for macOS and Linux or `.bat` for Windows.
+An additional component can be added in between the name and the extension.
+This file is then only run if the process environment contains a variable matching the middle component.
+That can be useful to put things like package installations specific to GitHub Actions while keeping the main build script generic for other platforms.
 
 ## How to run this
 
-Many packages use a lot of resources to build, and even those that do not quickly add up.
-We have chosen GitHub Action workflows as the cross-platform specification for how to build packages.
-
-### Running actions locally with nektos/act
-
-[Act](https://github.com/nektos/act) allows running GitHub actions locally.
-We can use that to just build packages on a local machine:
-
-```
-./act --artifact-server-path /tmp/artifacts \
-      -W .github/workflows/build-linux-amd64-wheels.yml \
-      -P self-hosted=-self-hosted \
-      -P macOS=-self-hosted -P Linux=-self-hosted \
-      -P X64=-self-hosted -P ARM64=-self-hosted \
-      --input name=psutil
-```
-
-You can vary the `--input name=` argument to select which package to build or to build all.
-
-On Linux you will need Docker or Podman.
-If you're using Podman, make sure you are running the system service (e.g.
-`podman system service -t 0` to run it in a shell), have symlinked or aliased `docker` to `podman`, and prepared you env with `export DOCKER_HOST=unix://$XDG_RUNTIME_DIR/podman/podman.sock` to allow `act` to pick up where podman is listening.
+Just run the `build_wheels.py` script.
+It expects a URL to download the GraalPy release from.
+You can set the environment variable `PACKAGES_TO_BUILD` to a comma-separated list of package build scripts you want to consider.