build(release): pypi-ready wheels, metadata, and publish workflow (#6)

.github/workflows/release.yml

@@ -0,0 +1,196 @@
+# Tagged-release wheels: build the cp310-cp313 runtime matrix + the kernel-cache
+# wheel on the self-hosted node, then publish
+#   - runtime wheels (manylinux, bare version)      -> PyPI, via twine + an API token
+#   - kernel-cache wheel (<version>+cu130) + runtime -> a DRAFT GitHub release on this repo
+#
+# INERT until a v* tag is pushed. The tag must point at a commit whose version.py
+# matches (vX.Y.Z == __version__ X.Y.Z); scripts/build-release-wheels.sh enforces
+# this in FREETOKEN_BUILD_RELEASE mode and refuses otherwise.
+#
+# This lane is separate from nightly-wheels.yml on purpose:
+#   - nightly ships +g<sha>-stamped, linux_x86_64-tagged cp312 wheels to the
+#     FreeToken-Web rolling `beta` release, which shipped Desktops resolve by
+#     asset name -- that channel's naming must not change.
+#   - release ships bare-versioned, manylinux-retagged cp310-cp313 wheels, which
+#     is what PyPI accepts.
+#
+# Security shape mirrors nightly-wheels.yml: fork code never reaches the
+# self-hosted node (no pull_request trigger + repo guard), and credentials only
+# exist on hosted runners. The upload tokens are ENVIRONMENT secrets on `pypi` /
+# `testpypi`, so the environment's reviewers gate every use of them -- NOT the
+# existing `release` environment, which holds the FreeToken-Web PAT and must
+# stay off this lane.
+#
+# One-time owner setup:
+#   - GitHub environments `pypi` and `testpypi`, `pypi` WITH required reviewers:
+#     once this workflow is on main, any v* tag reaches publish-pypi, and that
+#     reviewer gate is the only thing in front of it.
+#   - Environment secret `PYPI_TOKEN` on `pypi` and `TEST_PYPI_TOKEN` on
+#     `testpypi`. Scope each token to the single project `freetoken`, never
+#     account-wide, and store them per-environment rather than repo-wide.
+
+name: Release wheels
+
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+    inputs:
+      testpypi_rehearsal:
+        description: "Build .dev<run> wheels and publish them to TestPyPI (never touches PyPI)"
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+# Per-ref group, own namespace: sharing nightly-wheels' group would queue a release
+# behind a 40-minute nightly build, and a SHARED release group would let a newer tag
+# replace an earlier tag's still-PENDING run (GitHub keeps at most one pending run
+# per group). Per-ref, back-to-back tags each keep their run; the single build
+# runner serializes them anyway. No cancel-in-progress -- never kill a release mid-way.
+concurrency:
+  group: release-wheels-${{ github.ref }}
+
+jobs:
+  build:
+    # Guard against runs from forks of this repo; update on an org transfer.
+    # Mode is keyed on the EVENT, never on ref_type: a workflow_dispatch can be
+    # pointed at a tag ref, and that must stay a rehearsal, not become a release.
+    if: >-
+      github.repository == 'FlashML-org/FreeToken' &&
+      (github.event_name == 'push' || inputs.testpypi_rehearsal)
+    runs-on: [self-hosted, linux, engine-build]
+    timeout-minutes: 60
+    steps:
+      # The build container runs as root; an interrupted build can leave root-owned
+      # files that a plain checkout cannot delete. Wipe via a root container first.
+      - name: Clean workspace
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
+            sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          # Full history + tags: release mode verifies HEAD == tag == version.py
+          # via `git describe --exact-match`.
+          fetch-depth: 0
+      - name: Build wheels (manylinux container, cp310-cp313)
+        run: |
+          if [ "${{ github.event_name }}" = "push" ]; then
+            export FREETOKEN_BUILD_RELEASE=1
+          else
+            # Rehearsal: .dev<run_id> versions -- PEP 440-clean (no local
+            # segment) AND unique per run, so repeated rehearsals never collide
+            # with TestPyPI's no-reupload-ever rule. run_id, not run_number:
+            # run_number is stable across "Re-run all jobs", which would rebuild
+            # the identical filenames and let --skip-existing turn a re-run into
+            # a green no-op that leaves the previous attempt's bytes published.
+            export FREETOKEN_BUILD_DEV_STAMP="${{ github.run_id }}"
+          fi
+          export FT_PYTHON_MATRIX="cp310 cp311 cp312 cp313"
+          export FT_MANYLINUX_RETAG=1
+          scripts/ci/manylinux-build.sh
+      - name: Check the wheel set
+        run: |
+          ls -l dist/
+          rt="$(ls dist/freetoken-*manylinux*.whl 2>/dev/null | wc -l)"
+          kc="$(ls dist/freetoken_kernel_cache-*.whl 2>/dev/null | wc -l)"
+          [ "$rt" -eq 4 ] || { echo "expected 4 manylinux runtime wheels, got $rt" >&2; exit 1; }
+          [ "$kc" -eq 1 ] || { echo "expected 1 kernel-cache wheel, got $kc" >&2; exit 1; }
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: release-wheels
+          path: dist/*.whl
+          if-no-files-found: error
+          # Must outlast the `pypi` environment's review gate: GitHub lets a
+          # pending deployment sit for up to 30 days, and an expired artifact
+          # means redoing the whole 4-interpreter self-hosted build.
+          retention-days: 30
+
+  publish-testpypi:
+    needs: build
+    if: github.event_name == 'workflow_dispatch' && inputs.testpypi_rehearsal
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    environment: testpypi
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: release-wheels
+          path: dist
+      - name: Select the PyPI wheel set (runtime only, no local versions)
+        run: |
+          mkdir pypi-dist
+          cp dist/freetoken-*manylinux*.whl pypi-dist/
+          # The kernel-cache wheel stays on GitHub releases; a +local version
+          # here would be rejected by (Test)PyPI anyway -- fail early and loudly.
+          if ls pypi-dist/ | grep -qF '+'; then
+            echo "local-versioned wheel in the PyPI upload set:" >&2; ls pypi-dist/ >&2; exit 1
+          fi
+          pipx run twine check --strict pypi-dist/*
+      # --skip-existing keeps a partial rerun safe: (Test)PyPI never accepts the
+      # same filename twice, so without it a rerun after a mid-upload failure
+      # fails on the wheels that already landed.
+      - name: Upload to TestPyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }}
+          TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/
+        run: pipx run twine upload --verbose --skip-existing pypi-dist/*
+
+  publish-pypi:
+    needs: build
+    # Tag PUSH only -- a workflow_dispatch pointed at a tag ref must never reach PyPI.
+    if: github.event_name == 'push' && github.ref_type == 'tag'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    environment: pypi
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: release-wheels
+          path: dist
+      - name: Select the PyPI wheel set (runtime only, no local versions)
+        run: |
+          mkdir pypi-dist
+          cp dist/freetoken-*manylinux*.whl pypi-dist/
+          if ls pypi-dist/ | grep -qF '+'; then
+            echo "local-versioned wheel in the PyPI upload set:" >&2; ls pypi-dist/ >&2; exit 1
+          fi
+          pipx run twine check --strict pypi-dist/*
+      # See the note on --skip-existing in publish-testpypi. It matters more here:
+      # a PyPI filename burned by a half-finished upload can never be reused.
+      - name: Upload to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: pipx run twine upload --verbose --skip-existing pypi-dist/*
+
+  # Upload-only draft release on THIS repo carrying every wheel, most importantly
+  # the kernel-cache one PyPI cannot host. Deliberately not scripts/publish-wheels.sh:
+  # its prune-then-upload semantics and stamp-pairing gate are for the rolling beta
+  # channel, and would be wrong against an immutable version tag.
+  github-release:
+    needs: build
+    if: github.event_name == 'push' && github.ref_type == 'tag'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: release-wheels
+          path: dist
+      - name: Create draft release with all wheels
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          # Idempotent for re-runs: create the draft, or refresh assets on the
+          # existing release if a previous run got that far.
+          gh release create "$GITHUB_REF_NAME" --draft --verify-tag \
+            --title "$GITHUB_REF_NAME" \
+            --notes "Draft -- edit notes, then publish." \
+            -R "$GITHUB_REPOSITORY" \
+            dist/*.whl \
+          || gh release upload "$GITHUB_REF_NAME" dist/*.whl --clobber -R "$GITHUB_REPOSITORY"

docs/cli.md

@@ -13,8 +13,9 @@ ft <command> [args]
 | `ft checkpoint` | Convert an HF checkpoint to the FTW fast-load format |
 | `ft bench bw` | Benchmark CPU vs PCIe bandwidth to calibrate the MoE backend |
 
-`ft --version` prints the installed version (torch-free; release wheels
-include the `+g<sha>` build stamp). Every command supports `--help`.
+`ft --version` prints the installed version (torch-free; nightly wheels carry a
+`+g<sha>` build stamp, tagged releases a bare version). Every command supports
+`--help`.
 
 ## ft serve
 

freetoken-kernel-cache/pyproject.toml

@@ -22,9 +22,10 @@ requires-python = ">=3.10"
 license = "Apache-2.0"
 dependencies = ["apache-tvm-ffi==0.1.13.post3"]
 
-# cu130 wheels are not published to PyPI, and uv applies these to build-system.requires
-# when this directory is the build root — without them an isolated build pulls PyPI's
-# default-CUDA torch and stamps the wheel +cu126/+cu128. Mirrors ../pyproject.toml.
+# Provenance pin: PyPI's torch 2.11.0 is itself the cu130 build, but this index serves
+# ONLY cu130 wheels, so an isolated build can never resolve a different-CUDA torch and
+# stamp the wheel +cu126/+cu128. uv applies these to build-system.requires when this
+# directory is the build root. Mirrors ../pyproject.toml.
 [tool.uv.sources]
 torch = { index = "pytorch-cu130" }
 

install.sh

@@ -201,8 +201,9 @@ mkdir -p "$FT_HOME"
 # re-install can't inherit a stale/mismatched torch (e.g. an old cu128 venv after a cu130 bump).
 "$UV" venv "$VENV" --python "$PY_VERSION" --clear
 
-# torch and sglang-kernel cu130 wheels are not on PyPI. `unsafe-best-match` is needed
-# because the pytorch index also mirrors stale copies of common deps (e.g.
+# PyPI's torch 2.11.0 and sglang-kernel 0.4.5 are the same cu130 builds these indexes
+# serve; the explicit indexes pin provenance to the cu130 channels. `unsafe-best-match`
+# is needed because the pytorch index also mirrors stale copies of common deps (e.g.
 # packaging<=24.1) that would shadow PyPI under uv's first-index strategy; all indexes
 # here are trusted. [tool.uv.sources] does not survive into a built wheel, so the
 # indexes it names must be repeated below.

pyproject.toml

@@ -10,11 +10,26 @@ build-backend = "setuptools.build_meta"
 name = "freetoken"
 # Single source of truth: python/freetoken/version.py (read via the attr directive below).
 dynamic = ["version"]
-description = "FreeToken inference runtime"
+description = "Local MoE-offload LLM inference runtime with OpenAI- and Anthropic-compatible APIs"
 readme = "README.md"
 requires-python = ">=3.10"
 license = "Apache-2.0"
 license-files = ["LICENSE"]
+authors = [{ name = "FlashML Team" }]
+# No License:: classifier: setuptools>=77 rejects mixing one with the SPDX string above.
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Operating System :: POSIX :: Linux",
+    "Environment :: GPU :: NVIDIA CUDA",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
 # Version ranges are floor=last-verified, ceiling=next-major (kept loose enough to
 # resolve, tight enough to bar a silent breaking bump). These ranges are the contract;
 # there is no lockfile.
@@ -42,7 +57,7 @@ dependencies = [
     "torch>=2.11,<2.12",
     "tqdm>=4.66,<5",
     "transformers>=5.5,<6",
-    "triton==3.6.0",
+    "triton==3.6.0; platform_system == 'Linux'",
     "uvicorn>=0.30,<1",
 ]
 

python/freetoken/attention/fa.py

@@ -174,7 +174,7 @@ def _fa_sgl_impl(
         from sgl_kernel.flash_attn import flash_attn_with_kvcache
     except ImportError as e:
         raise ImportError(
-            "sgl_kernel.flash_attn is not found. Please install it with `pip install sgl-kernel`.\n"
+            "sgl_kernel.flash_attn is not found. Please install it with `pip install 'freetoken[sgl]'`.\n"
             "If you're sure it's correctly installed, try `apt update && apt install libnuma1`."
         ) from e
 

scripts/build-release-wheels.sh

@@ -13,6 +13,14 @@
 #   FREETOKEN_BUILD_KEEP_TEMP     keep setuptools/tvm build leftovers when true (default: 0)
 #   FREETOKEN_BUILD_STRIP         strip debug symbols from the runtime wheel's .so (default: 1)
 #   FREETOKEN_BUILD_NO_STAMP      skip the +g<sha> commit stamp -- dev builds only (default: 0)
+#   FREETOKEN_BUILD_RELEASE       tagged-release mode: no stamp, and HEAD must be at the
+#                                 tag v<version.py> exactly (default: 0)
+#   FREETOKEN_BUILD_DEV_STAMP     stamp .dev<N> instead of +g<sha> -- PEP 440-legal without
+#                                 a local segment, so TestPyPI rehearsals get a unique,
+#                                 uploadable version per run (default: unset)
+#   FREETOKEN_BUILD_SKIP_KERNEL_CACHE  build only the runtime wheel -- for the 2nd..Nth
+#                                 interpreter of a matrix build; the kernel cache is
+#                                 py3-none and only needs building once (default: 0)
 #   FREETOKEN_KERNEL_CACHE_SPECS  optional comma-separated subset of kernel spec names
 set -euo pipefail
 
@@ -26,6 +34,9 @@ CLEAN="${FREETOKEN_BUILD_CLEAN:-1}"
 KEEP_TEMP="${FREETOKEN_BUILD_KEEP_TEMP:-0}"
 STRIP="${FREETOKEN_BUILD_STRIP:-1}"
 NO_STAMP="${FREETOKEN_BUILD_NO_STAMP:-0}"
+RELEASE="${FREETOKEN_BUILD_RELEASE:-0}"
+DEV_STAMP="${FREETOKEN_BUILD_DEV_STAMP:-}"
+SKIP_KERNEL_CACHE="${FREETOKEN_BUILD_SKIP_KERNEL_CACHE:-0}"
 
 TRUE_VALUES=" 1 true yes on "
 
@@ -124,6 +135,30 @@ restore_version() {
   fi
 }
 stamp_version() {
+  if [[ -n "$DEV_STAMP" ]] && { enabled "$RELEASE" || enabled "$NO_STAMP"; }; then
+    die "FREETOKEN_BUILD_DEV_STAMP cannot be combined with RELEASE or NO_STAMP modes"
+  fi
+  # Release mode: the runtime wheel must carry a bare PEP 440 version (PyPI rejects
+  # any +local segment), so no stamp -- provenance comes from the tag instead, which
+  # is verified to point at exactly this commit and this version.py. The kernel-cache
+  # wheel then comes out as <version>+cu130 (no .g<sha>): build_backend.py appends
+  # .g<sha> only when version.py already carries one.
+  if enabled "$RELEASE"; then
+    enabled "$NO_STAMP" \
+      && die "FREETOKEN_BUILD_RELEASE and FREETOKEN_BUILD_NO_STAMP are mutually exclusive"
+    if [[ -n "$(git -C "$ROOT" status --porcelain)" ]]; then
+      die "working tree is not clean -- a release build must come from exactly the tagged commit."
+    fi
+    local version tag
+    version="$(sed -nE 's/^__version__ = "([^"+]+)".*$/\1/p' "$VERSION_FILE")"
+    [[ -n "$version" ]] || die "cannot read a version from $VERSION_FILE"
+    tag="$(git -C "$ROOT" describe --exact-match --tags HEAD 2>/dev/null)" \
+      || die "FREETOKEN_BUILD_RELEASE: HEAD is not at a tag (expected tag v$version)."
+    [[ "$tag" == "v$version" ]] \
+      || die "FREETOKEN_BUILD_RELEASE: HEAD tag is '$tag' but version.py says '$version' (expected tag v$version)."
+    say "release build: $version (tag $tag)"
+    return 0
+  fi
   if enabled "$NO_STAMP"; then
     warn "FREETOKEN_BUILD_NO_STAMP is set -- building UNSTAMPED dev wheels (do not release)"
     return 0
@@ -134,7 +169,7 @@ stamp_version() {
   # exactly that leftover (version.py is the only change, and it carries a stamp) and
   # restore it instead of dying "not clean" at the operator.
   if [[ "$(git -C "$ROOT" status --porcelain)" == " M python/freetoken/version.py" ]] \
-    && grep -qE '^__version__ = "[0-9][^"]*\+g[0-9a-f]{7,}"' "$VERSION_FILE"; then
+    && grep -qE '^__version__ = "[0-9][^"]*(\+g[0-9a-f]{7,}|\.dev[0-9]+)"' "$VERSION_FILE"; then
     git -C "$ROOT" checkout --quiet -- "python/freetoken/version.py"
     say "recovered a leftover version stamp from an interrupted build"
   fi
@@ -145,11 +180,18 @@ stamp_version() {
   if [[ -n "$(git -C "$ROOT" status --porcelain)" ]]; then
     die "working tree is not clean -- a stamped wheel would lie about its commit. Commit/stash (and clean untracked files) first, or set FREETOKEN_BUILD_NO_STAMP=1 for an unstamped dev build."
   fi
-  local sha
-  sha="$(git -C "$ROOT" rev-parse --short=9 HEAD)"
+  local suffix version
+  if [[ -n "$DEV_STAMP" ]]; then
+    [[ "$DEV_STAMP" =~ ^[0-9]+$ ]] || die "FREETOKEN_BUILD_DEV_STAMP must be a plain number (got '$DEV_STAMP')"
+    suffix=".dev${DEV_STAMP}"
+  else
+    suffix="+g$(git -C "$ROOT" rev-parse --short=9 HEAD)"
+  fi
+  version="$(sed -nE 's/^__version__ = "([^"+]+)".*$/\1/p' "$VERSION_FILE")"
+  [[ -n "$version" ]] || die "cannot read a version from $VERSION_FILE"
   # In-place edit, not overwrite: anything in version.py beyond the version line survives.
-  sed -i -E "s/^(__version__ = \"[0-9][^\"+]*)\"/\1+g${sha}\"/" "$VERSION_FILE"
-  grep -qE "^__version__ = \"[0-9][^\"+]*\+g${sha}\"" "$VERSION_FILE" \
+  sed -i -E "s/^(__version__ = \"[0-9][^\"+]*)\"/\1${suffix}\"/" "$VERSION_FILE"
+  grep -qF "__version__ = \"${version}${suffix}\"" "$VERSION_FILE" \
     || die "failed to stamp $VERSION_FILE"
   STAMPED=1
   say "stamped version: $(sed -nE 's/^__version__ = "([^"]+)".*$/\1/p' "$VERSION_FILE")"
@@ -186,16 +228,23 @@ warn_arch_override
 say "building freetoken runtime wheel"
 uv "${BUILD_ARGS[@]}" .
 
-# CLEAN wiped old wheels, so the sole freetoken-*.whl (NOT the kernel-cache) is what we just built.
-rt_whl="$(find "$OUT_DIR" -maxdepth 1 -name 'freetoken-*.whl' \
-  ! -name 'freetoken-kernel-cache-*.whl' -print | sort | tail -1)"
-[ -n "$rt_whl" ] && strip_wheel "$rt_whl"
+# Select the wheel THIS interpreter just built by its cp tag -- a matrix build runs
+# this script once per interpreter with CLEAN=0, so several freetoken-*.whl coexist.
+cptag="$("$PYTHON_BIN" -c 'import sys; print(f"cp{sys.version_info[0]}{sys.version_info[1]}")')"
+rt_whl="$(find "$OUT_DIR" -maxdepth 1 -name "freetoken-*-${cptag}-*.whl" -printf '%T@ %p\n' \
+  | sort -n | tail -1 | cut -d' ' -f2-)"
+[ -n "$rt_whl" ] || die "runtime wheel for $cptag not found in $OUT_DIR"
+strip_wheel "$rt_whl"
 
-say "building freetoken-kernel-cache wheel"
-warn_arch_override
-export FREETOKEN_KERNEL_CACHE_VERBOSE="${FREETOKEN_KERNEL_CACHE_VERBOSE:-1}"
-export FREETOKEN_KERNEL_CACHE_BUILD_DIR="${FREETOKEN_KERNEL_CACHE_BUILD_DIR:-$ROOT/build/freetoken-kernel-cache}"
-uv "${BUILD_ARGS[@]}" freetoken-kernel-cache
+if enabled "$SKIP_KERNEL_CACHE"; then
+  say "skipping freetoken-kernel-cache wheel (FREETOKEN_BUILD_SKIP_KERNEL_CACHE)"
+else
+  say "building freetoken-kernel-cache wheel"
+  warn_arch_override
+  export FREETOKEN_KERNEL_CACHE_VERBOSE="${FREETOKEN_KERNEL_CACHE_VERBOSE:-1}"
+  export FREETOKEN_KERNEL_CACHE_BUILD_DIR="${FREETOKEN_KERNEL_CACHE_BUILD_DIR:-$ROOT/build/freetoken-kernel-cache}"
+  uv "${BUILD_ARGS[@]}" freetoken-kernel-cache
+fi
 
 say "wheels written to $OUT_DIR"
 find "$OUT_DIR" -maxdepth 1 -type f \

scripts/ci/manylinux-build.sh

@@ -15,7 +15,17 @@
 #   FT_CI_CACHE_DIR    persistent cache dir on the host, holds the uv binary and
 #                      uv's package cache across builds (default: ~/.cache/freetoken-ci)
 #   FT_OUT_DIR         host dir that receives the wheels (default: <repo>/dist)
-#   FREETOKEN_BUILD_*  passed through to scripts/build-release-wheels.sh
+#   FT_PYTHON_MATRIX   space-separated cp tags to build the runtime wheel for
+#                      (default: cp312 -- the nightly/Desktop channel is cp312-only;
+#                      the release lane passes "cp310 cp311 cp312 cp313")
+#   FT_MANYLINUX_RETAG retag runtime wheels linux_x86_64 -> detected manylinux (default: 0).
+#                      Release/PyPI lane only: shipped Desktops resolve the nightly
+#                      release's assets by name and expect linux_x86_64.
+#   FREETOKEN_BUILD_NO_STAMP / _RELEASE / _DEV_STAMP / _STRIP and
+#   FREETOKEN_KERNEL_CACHE_* are forwarded into the container. Other
+#   FREETOKEN_BUILD_* vars are NOT: _CLEAN is set by this script per matrix
+#   iteration, and the rest (_KEEP_TEMP, _NO_ISOLATION, _OUT_DIR, ...) only
+#   make sense when driving build-release-wheels.sh directly.
 set -euo pipefail
 
 say() { printf '\033[1;36m==>\033[0m %s\n' "$*"; }
@@ -33,9 +43,13 @@ if [[ -z "${FT_IN_CONTAINER:-}" ]]; then
     -e FT_HOST_UID="$(id -u)" \
     -e FT_HOST_GID="$(id -g)" \
     -e FREETOKEN_BUILD_NO_STAMP="${FREETOKEN_BUILD_NO_STAMP:-}" \
+    -e FREETOKEN_BUILD_RELEASE="${FREETOKEN_BUILD_RELEASE:-}" \
+    -e FREETOKEN_BUILD_DEV_STAMP="${FREETOKEN_BUILD_DEV_STAMP:-}" \
     -e FREETOKEN_BUILD_STRIP="${FREETOKEN_BUILD_STRIP:-}" \
     -e FREETOKEN_KERNEL_CACHE_SPECS="${FREETOKEN_KERNEL_CACHE_SPECS:-}" \
     -e FREETOKEN_KERNEL_CACHE_VERBOSE="${FREETOKEN_KERNEL_CACHE_VERBOSE:-}" \
+    -e FT_PYTHON_MATRIX="${FT_PYTHON_MATRIX:-}" \
+    -e FT_MANYLINUX_RETAG="${FT_MANYLINUX_RETAG:-}" \
     -v "$ROOT:/workspace" \
     -v "$CACHE_DIR:/ci-cache" \
     -v "$OUT_DIR:/ci-out" \
@@ -65,20 +79,51 @@ if [[ ! -x /ci-cache/bin/uv ]]; then
 fi
 export UV_CACHE_DIR=/ci-cache/uv
 
-# Build venv is throwaway (recreated per build from the warm uv cache) so stale
-# build deps can never linger; only the cache dir persists across builds.
-VENV=/tmp/build-venv
-PYBIN=/opt/python/cp312-cp312/bin/python
-say "creating build venv"
-uv venv --quiet --python "$PYBIN" "$VENV"
-# torch must come from the cu130 index (PyPI's torch is a different CUDA variant
-# and would tag the kernel-cache wheel wrong); everything else is plain PyPI.
-uv pip install --quiet --python "$VENV/bin/python" \
-  --index-url https://download.pytorch.org/whl/cu130 "torch>=2.11,<2.12"
-uv pip install --quiet --python "$VENV/bin/python" \
-  "setuptools>=77" wheel ninja "apache-tvm-ffi==0.1.13.post3"
+MATRIX="${FT_PYTHON_MATRIX:-cp312}"
+RETAG="${FT_MANYLINUX_RETAG:-0}"
 
-export FREETOKEN_BUILD_PYTHON="$VENV/bin/python"
 export FREETOKEN_BUILD_OUT_DIR=/ci-out
-# No exec: the ownership trap above must still fire after the build returns.
-bash scripts/build-release-wheels.sh
+# One clean here instead of per-invocation: with several interpreters, each
+# build-release-wheels.sh run would otherwise wipe the previous ABI's wheel.
+rm -f /ci-out/freetoken-*.whl /ci-out/freetoken_kernel_cache-*.whl /ci-out/freetoken-kernel-cache-*.whl
+export FREETOKEN_BUILD_CLEAN=0
+
+# Build venvs are throwaway (recreated per build from the warm uv cache) so stale
+# build deps can never linger; only the cache dir persists across builds.
+first=1
+for cptag in $MATRIX; do
+  PYBIN="/opt/python/${cptag}-${cptag}/bin/python"
+  [[ -x "$PYBIN" ]] || { echo "no such interpreter in the builder image: $PYBIN" >&2; exit 1; }
+  VENV="/tmp/build-venv-$cptag"
+  say "creating build venv ($cptag)"
+  uv venv --quiet --python "$PYBIN" "$VENV"
+  # Provenance pin: PyPI's torch 2.11.0 is itself the cu130 build, but this index
+  # serves ONLY cu130 wheels, so the resolve can never pick a different-CUDA torch
+  # and tag the kernel-cache wheel wrong; everything else is plain PyPI.
+  uv pip install --quiet --python "$VENV/bin/python" \
+    --index-url https://download.pytorch.org/whl/cu130 "torch>=2.11,<2.12"
+  uv pip install --quiet --python "$VENV/bin/python" \
+    "setuptools>=77" wheel ninja "apache-tvm-ffi==0.1.13.post3"
+
+  export FREETOKEN_BUILD_PYTHON="$VENV/bin/python"
+  # The kernel-cache wheel is py3-none: build it once, with the first interpreter.
+  export FREETOKEN_BUILD_SKIP_KERNEL_CACHE="$((1 - first))"
+  first=0
+  # No exec: the ownership trap above must still fire after the build returns.
+  bash scripts/build-release-wheels.sh
+done
+
+# Release/PyPI lane only (see the header note on FT_MANYLINUX_RETAG). The glob
+# leaves the kernel-cache wheel alone: freetoken_* does not match freetoken-*.
+case " 1 true yes on " in *" $(printf '%s' "$RETAG" | tr '[:upper:]' '[:lower:]') "*)
+  say "retagging runtime wheels to their detected manylinux policy"
+  uv pip install --quiet --python "$VENV/bin/python" "auditwheel==6.6.0"
+  found=0
+  for whl in /ci-out/freetoken-*linux_x86_64.whl; do
+    [[ -e "$whl" ]] || continue
+    "$VENV/bin/python" scripts/ci/retag-manylinux.py "$whl"
+    found=1
+  done
+  [[ "$found" == 1 ]] || { echo "FT_MANYLINUX_RETAG set but no linux_x86_64 runtime wheels in /ci-out" >&2; exit 1; }
+  ;;
+esac

scripts/ci/retag-manylinux.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""Retag `linux_x86_64` runtime wheels to their detected manylinux policy.
+
+setuptools tags C-extension wheels `linux_<arch>`, which PyPI rejects at upload.
+The CI build already runs inside the pytorch manylinux_2_28 container, so the
+binaries meet the policy — only the tag is missing. This asks auditwheel which
+policy the wheel's symbol versions actually satisfy and rewrites the tag.
+
+Two deliberate choices:
+  * `sym_policy`, not the overall policy: the extensions intentionally leave
+    libtorch/libcudart as external NEEDED entries (provided by the installed
+    torch), and the overall policy grades any non-whitelisted external lib as
+    plain `linux`. `auditwheel repair` is equally unusable here — it would try
+    to graft libtorch into the wheel. Detection approach follows vLLM's
+    detect-manylinux-tag.py (Apache-2.0).
+  * Detected tag, not a hard-coded one, with a ceiling: a wheel built outside
+    the container (host glibc 2.3x) must fail loudly instead of shipping a tag
+    that lies about its glibc floor.
+
+`wheel tags` rewrites filename + WHEEL metadata + RECORD consistently; renaming
+the file alone would leave the wheel internally inconsistent.
+
+Requires: auditwheel==6.6.0 (the analyze_wheel_abi signature is version
+specific), wheel.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+from auditwheel.error import NonPlatformWheelError, WheelToolsError
+from auditwheel.wheel_abi import analyze_wheel_abi
+from auditwheel.wheeltools import get_wheel_architecture, get_wheel_libc
+from packaging.utils import InvalidWheelFilename
+
+
+def detect_platform_tag(wheel: Path) -> str:
+    try:
+        arch = get_wheel_architecture(wheel.name)
+    except (WheelToolsError, NonPlatformWheelError, InvalidWheelFilename):
+        arch = None
+    try:
+        libc = get_wheel_libc(wheel.name)
+    except (WheelToolsError, InvalidWheelFilename):
+        libc = None
+    winfo = analyze_wheel_abi(
+        libc,
+        arch,
+        wheel,
+        frozenset(),
+        disable_isa_ext_check=False,
+        allow_graft=False,
+    )
+    # Deliberately not winfo.overall_policy: that folds in the external-library
+    # check, and libtorch/libcudart are not on any manylinux whitelist, so it
+    # always collapses to linux_x86_64. sym_policy (glibc symbols) and
+    # machine_policy (required ISA extensions) are the two components that do
+    # apply to us. min() by priority picks the stricter one -- the same
+    # combinator auditwheel itself uses -- so a future global -march= that
+    # raises the ISA floor cannot be silently retagged as broadly compatible.
+    return min(winfo.sym_policy, winfo.machine_policy).name
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("wheels", nargs="+", type=Path)
+    parser.add_argument(
+        "--max-glibc",
+        default="2.28",
+        help="highest glibc version the detected tag may require (default: 2.28)",
+    )
+    args = parser.parse_args()
+    m = re.fullmatch(r"(\d+)\.(\d+)", args.max_glibc)
+    if not m:
+        print(f"error: --max-glibc must look like '2.28' (got '{args.max_glibc}')", file=sys.stderr)
+        return 1
+    ceiling = (int(m[1]), int(m[2]))
+
+    for whl in args.wheels:
+        if not whl.is_file():
+            print(f"error: no such wheel: {whl}", file=sys.stderr)
+            return 1
+        tag = detect_platform_tag(whl)
+        m = re.fullmatch(r"manylinux_(\d+)_(\d+)_(\w+)", tag)
+        if not m or (int(m[1]), int(m[2])) > ceiling:
+            print(
+                f"error: {whl.name}: detected policy '{tag}' exceeds "
+                f"manylinux_{ceiling[0]}_{ceiling[1]} — this wheel was not "
+                "built in the manylinux container and must not be retagged",
+                file=sys.stderr,
+            )
+            return 2
+        subprocess.run(
+            [sys.executable, "-m", "wheel", "tags", "--platform-tag", tag, "--remove", str(whl)],
+            check=True,
+            stdout=subprocess.DEVNULL,
+        )
+        print(f"{whl.name} -> {tag}", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())