1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433# /// script
# requires-python = ">=3.13"
# dependencies = [
# "packaging",
# "pyyaml",
# ]
# ///
import argparse
import json
import sys
from typing import Any
import yaml
from packaging.version import Version
CI_TARGETS_YAML = "ci-targets.yaml"
CI_RUNNERS_YAML = "ci-runners.yaml"
CI_EXTRA_SKIP_LABELS = ["documentation"]
CI_MATRIX_SIZE_LIMIT = 256 # The maximum size of a matrix in GitHub Actions
# Docker images for building toolchains and dependencies
DOCKER_BUILD_IMAGES = [
{"name": "build", "arch": "x86_64"},
{"name": "build.cross", "arch": "x86_64"},
{"name": "build.cross-riscv64", "arch": "x86_64"},
{"name": "build.debian9", "arch": "aarch64"},
{"name": "gcc", "arch": "x86_64"},
{"name": "gcc.debian9", "arch": "aarch64"},
]
def crate_artifact_name(platform: str, arch: str) -> str:
return f"crate-{platform}-{arch}"
def meets_conditional_version(version: str, min_version: str) -> bool:
return Version(version) >= Version(min_version)
def parse_labels(labels: str | None) -> dict[str, set[str]]:
"""Parse labels into a dict of category filters."""
if not labels:
return {}
result: dict[str, set[str]] = {
"platform": set(),
"python": set(),
"build": set(),
"arch": set(),
"libc": set(),
"directives": set(),
}
for label in labels.split(","):
label = label.strip()
# Handle special labels
if label in CI_EXTRA_SKIP_LABELS:
result["directives"].add("skip")
continue
if not label or ":" not in label:
continue
category, value = label.split(":", 1)
if category == "ci":
category = "directives"
if category in result:
result[category].add(value)
return result
def should_include_entry(entry: dict[str, str], filters: dict[str, set[str]]) -> bool:
"""Check if an entry satisfies the label filters."""
if filters.get("directives") and "skip" in filters["directives"]:
return False
if filters.get("platform") and entry["platform"] not in filters["platform"]:
return False
if filters.get("python") and entry["python"] not in filters["python"]:
return False
if filters.get("arch") and entry["arch"] not in filters["arch"]:
return False
if (
filters.get("libc")
and entry.get("libc")
and entry["libc"] not in filters["libc"]
):
return False
if filters.get("build"):
build_options = set(entry.get("build_options", "").split("+"))
if not all(f in build_options for f in filters["build"]):
return False
return True
def generate_docker_matrix_entries(
runners: dict[str, Any],
platform_filter: str | None = None,
) -> list[dict[str, str]]:
"""Generate matrix entries for docker image builds."""
if platform_filter and platform_filter != "linux":
return []
matrix_entries = []
for image in DOCKER_BUILD_IMAGES:
# Find appropriate runner for Linux platform with the specified architecture
runner = find_runner(runners, "linux", image["arch"], False)
entry = {
"name": image["name"],
"arch": image["arch"],
"runner": runner,
}
matrix_entries.append(entry)
return matrix_entries
def generate_crate_build_matrix_entries(
python_entries: list[dict[str, str]],
runners: dict[str, Any],
config: dict[str, Any],
force_crate_build: bool = False,
platform_filter: str | None = None,
) -> list[dict[str, str]]:
"""Generate matrix entries for crate builds based on python build matrix."""
needed_builds = set()
for entry in python_entries:
# The crate build will need to match the runner's architecture
runner = runners[entry["runner"]]
needed_builds.add((entry["platform"], runner["arch"]))
# If forcing crate build, also include all possible native builds
if force_crate_build:
for platform, platform_config in config.items():
# Filter by platform if specified
if platform_filter and platform != platform_filter:
continue
for target_config in platform_config.values():
# Only include if native (run: true means native)
if not target_config.get("run"):
continue
arch = target_config["arch"]
needed_builds.add((platform, arch))
# Create matrix entries for each needed build
return [
{
"platform": platform,
"arch": arch,
# Use the GitHub runner for Windows, because the Depot one is
# missing a Rust toolchain. On Linux, it's important that the the
# `python-build` runner matches the `crate-build` runner because of
# GLIBC version mismatches.
"runner": find_runner(
runners, platform, arch, True if platform == "windows" else False
),
"crate_artifact_name": crate_artifact_name(
platform,
arch,
),
}
for platform, arch in needed_builds
if not platform_filter or platform == platform_filter
]
def generate_python_build_matrix_entries(
config: dict[str, Any],
runners: dict[str, Any],
platform_filter: str | None = None,
label_filters: dict[str, set[str]] | None = None,
) -> list[dict[str, str]]:
"""Generate matrix entries for python builds."""
matrix_entries = []
for platform, platform_config in config.items():
if platform_filter and platform != platform_filter:
continue
for target_triple, target_config in platform_config.items():
add_python_build_entries_for_config(
matrix_entries,
target_triple,
target_config,
platform,
runners,
label_filters.get("directives", set()) if label_filters else set(),
)
# Apply label filters if present
if label_filters:
matrix_entries = [
entry
for entry in matrix_entries
if should_include_entry(entry, label_filters)
]
return matrix_entries
def find_runner(runners: dict[str, Any], platform: str, arch: str, free: bool) -> str:
# Find a matching platform first
match_platform = [
runner
for runner in runners
if runners[runner]["platform"] == platform and runners[runner]["free"] == free
]
# Then, find a matching architecture
match_arch = [
runner for runner in match_platform if runners[runner]["arch"] == arch
]
# If there's a matching architecture, use that
if match_arch:
return match_arch[0]
# Otherwise, use the first with a matching platform
if match_platform:
return match_platform[0]
raise RuntimeError(
f"No runner found for platform {platform!r} and arch {arch!r} with free={free}"
)
def add_python_build_entries_for_config(
matrix_entries: list[dict[str, str]],
target_triple: str,
config: dict[str, Any],
platform: str,
runners: dict[str, Any],
directives: set[str],
) -> None:
"""Add python build matrix entries for a specific target configuration."""
python_versions = config["python_versions"]
build_options = config["build_options"]
arch = config["arch"]
runner = find_runner(runners, platform, arch, False)
# Create base entry that will be used for all variants
base_entry = {
"arch": arch,
"target_triple": target_triple,
"platform": platform,
"runner": runner,
# If `run` is in the config, use that — otherwise, default to if the
# runner architecture matches the build architecture
"run": str(config.get("run", runners[runner]["arch"] == arch)).lower(),
# Use the crate artifact built for the runner's architecture
"crate_artifact_name": crate_artifact_name(platform, runners[runner]["arch"]),
}
# Add optional fields if they exist
if "arch_variant" in config:
base_entry["arch_variant"] = config["arch_variant"]
if "libc" in config:
base_entry["libc"] = config["libc"]
if "vcvars" in config:
base_entry["vcvars"] = config["vcvars"]
if "dry-run" in directives:
base_entry["dry-run"] = "true"
# Process regular build options
for python_version in python_versions:
for build_option in build_options:
entry = base_entry.copy()
entry.update(
{
"python": python_version,
"build_options": build_option,
}
)
matrix_entries.append(entry)
# Process conditional build options (e.g., freethreaded)
for conditional in config.get("build_options_conditional", []):
min_version = conditional["minimum-python-version"]
for python_version in python_versions:
if not meets_conditional_version(python_version, min_version):
continue
for build_option in conditional["options"]:
entry = base_entry.copy()
entry.update(
{
"python": python_version,
"build_options": build_option,
}
)
matrix_entries.append(entry)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Generate a JSON matrix for building distributions in CI"
)
parser.add_argument(
"--platform",
choices=["darwin", "linux", "windows"],
help="Filter matrix entries by platform",
)
parser.add_argument(
"--max-shards",
type=int,
default=0,
help="The maximum number of shards allowed; set to zero to disable ",
)
parser.add_argument(
"--labels",
help="Comma-separated list of labels to filter by (e.g., 'platform:darwin,python:3.13,build:debug'), all must match.",
)
parser.add_argument(
"--free-runners",
action="store_true",
help="If only free runners should be used.",
)
parser.add_argument(
"--force-crate-build",
action="store_true",
help="Force crate builds to be included even without python builds.",
)
parser.add_argument(
"--matrix-type",
choices=["python-build", "docker-build", "crate-build", "all"],
default="all",
help="Which matrix types to generate (default: all)",
)
return parser.parse_args()
def main() -> None:
args = parse_args()
labels = parse_labels(args.labels)
with open(CI_TARGETS_YAML) as f:
config = yaml.safe_load(f)
with open(CI_RUNNERS_YAML) as f:
runners = yaml.safe_load(f)
# If only free runners are allowed, reduce to a subset
if args.free_runners:
runners = {
runner: runner_config
for runner, runner_config in runners.items()
if runner_config.get("free")
}
result = {}
# Generate python build entries
python_entries = generate_python_build_matrix_entries(
config,
runners,
args.platform,
labels,
)
# Output python-build matrix if requested
if args.matrix_type in ["python-build", "all"]:
if args.max_shards:
python_build_matrix = {}
shards = (len(python_entries) // CI_MATRIX_SIZE_LIMIT) + 1
if shards > args.max_shards:
print(
f"error: python-build matrix of size {len(python_entries)} requires {shards} shards, but the maximum is {args.max_shards}; consider increasing `--max-shards`",
file=sys.stderr,
)
sys.exit(1)
for shard in range(args.max_shards):
shard_entries = python_entries[
shard * CI_MATRIX_SIZE_LIMIT : (shard + 1) * CI_MATRIX_SIZE_LIMIT
]
python_build_matrix[str(shard)] = {"include": shard_entries}
result["python-build"] = python_build_matrix
else:
if len(python_entries) > CI_MATRIX_SIZE_LIMIT:
print(
f"warning: python-build matrix of size {len(python_entries)} exceeds limit of {CI_MATRIX_SIZE_LIMIT} but sharding is not enabled; consider setting `--max-shards`",
file=sys.stderr,
)
result["python-build"] = {"include": python_entries}
# Generate docker-build matrix if requested
# Only include docker builds if there are Linux python builds
if args.matrix_type in ["docker-build", "all"]:
# Check if we have any Linux python builds
has_linux_builds = any(
entry.get("platform") == "linux" for entry in python_entries
)
# If no platform filter or explicitly requesting docker-build only, include docker builds
# Otherwise, only include if there are Linux python builds
if args.matrix_type == "docker-build" or has_linux_builds:
docker_entries = generate_docker_matrix_entries(
runners,
args.platform,
)
result["docker-build"] = {"include": docker_entries}
# Generate crate-build matrix if requested
if args.matrix_type in ["crate-build", "all"]:
crate_entries = generate_crate_build_matrix_entries(
python_entries,
runners,
config,
args.force_crate_build,
args.platform,
)
result["crate-build"] = {"include": crate_entries}
print(json.dumps(result))
if __name__ == "__main__":
main()