From d9c52ff1db2653f034be41b76e9611d605205b2b Mon Sep 17 00:00:00 2001 From: George Fu Date: Wed, 4 Nov 2026 16:44:48 +0000 Subject: [PATCH 1/3] Changelog entries for client generation --- .gitignore | 3 ++ Makefile | 2 + codegen/gen_client.py | 88 +++++++++++++++++++++++++++++++++----- codegen/smithy-python.toml | 3 ++ 4 files changed, 85 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index f3dbe4d9..5c16ff0b 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ dist/ # scratch workspace /workspace + +# Codegen sync changelog text, read by `make clients` +/CODEGEN_SYNC_CHANGELOG.md diff --git a/Makefile b/Makefile index 22f2b071..ecb57417 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,8 @@ test-integration: # Regenerate every client from codegen/aws-models with codegen built from the # smithy-python commit in codegen/smithy-python.toml. Each client is cleared first, # keeping only the paths in gen_client.py's KEEP, then scripts/client-overlays is applied. +# Requires CODEGEN_SYNC_CHANGELOG.md in the repo root, written within 8 hours; each +# changed client gets a changelog fragment from it, prefixed with the commit range. clients: SMITHY_PYTHON=$(abspath $(SMITHY_PYTHON)) uv run python codegen/gen_client.py --all --pinned --batch-size $(BATCH_SIZE) diff --git a/codegen/gen_client.py b/codegen/gen_client.py index ba303eba..3ad75c2d 100644 --- a/codegen/gen_client.py +++ b/codegen/gen_client.py @@ -5,6 +5,8 @@ accepts dynamodb, aws_sdk_dynamodb, aws-sdk-dynamodb, or bedrock-runtime forms. --pinned builds codegen from the smithy-python commit in codegen/smithy-python.toml; without it, the codegen last published to the local Maven cache is used. + --pinned requires a CODEGEN_SYNC_CHANGELOG.md in the repo root written within the + last 8 hours; each client whose output changed gets a changelog fragment from it. """ import argparse @@ -14,6 +16,7 @@ import shutil import subprocess import sys +import time import tomllib from collections.abc import Iterator from contextlib import contextmanager @@ -29,6 +32,10 @@ DEFAULT_BATCH_SIZE = 25 POINTER = CODEGEN_DIR / "smithy-python.toml" PINNED_DIR = CODEGEN_DIR / ".smithy-python" +SYNC_CHANGELOG = REPO / "CODEGEN_SYNC_CHANGELOG.md" +SMITHY_PYTHON_COMMITS = "https://github.com/smithy-lang/smithy-python/commits" +# An older file is assumed to be left over from a previous `make clients`. +SYNC_CHANGELOG_MAX_AGE_SECONDS = 8 * 60 * 60 # Paths codegen never emits; they survive the pre-generation delete. A path codegen # does emit but that must differ from codegen output belongs in client-overlays. @@ -76,27 +83,62 @@ def main() -> None: sys.exit("no clients selected") if args.pinned: - with _pinned_codegen(_pinned_commit()) as gradle_args: - _generate(models, args.batch_size, gradle_args) + previous, commit = _pinned_commits() + sync_entry = _sync_entry(previous, commit) + with _pinned_codegen(commit) as gradle_args: + _generate(models, args.batch_size, gradle_args, sync_entry) else: - _generate(models, args.batch_size, []) + _generate(models, args.batch_size, [], None) -def _generate(models: list[Path], batch_size: int, gradle_args: list[str]) -> None: +def _generate( + models: list[Path], + batch_size: int, + gradle_args: list[str], + sync_entry: str | None, +) -> None: docs_deps = _docs_deps() for start in range(0, len(models), batch_size): batch = models[start : start + batch_size] _build(batch, gradle_args) for model in batch: - _install_client(model.stem, docs_deps) + _install_client(model.stem, docs_deps, sync_entry) -def _pinned_commit() -> str: +def _pinned_commits() -> tuple[str, str]: with POINTER.open("rb") as f: - commit = tomllib.load(f)["commit"] - if not re.fullmatch(r"[0-9a-f]{40}", commit): - sys.exit(f"{POINTER}: commit must be a full 40-character hash, got {commit!r}") - return commit + pointer = tomllib.load(f) + commits = [] + for key in ("previous_commit", "commit"): + value = pointer.get(key) + if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{40}", value): + sys.exit( + f"{POINTER}: {key} must be a full 40-character hash, got {value!r}" + ) + commits.append(value) + previous, commit = commits + if previous == commit: + sys.exit(f"{POINTER}: previous_commit and commit must differ") + return previous, commit + + +def _sync_entry(previous: str, commit: str) -> str: + if not SYNC_CHANGELOG.exists(): + SYNC_CHANGELOG.touch() + sys.exit( + f"created empty {SYNC_CHANGELOG}; " + "write the changelog text for this codegen sync" + ) + age = time.time() - SYNC_CHANGELOG.stat().st_mtime + if age > SYNC_CHANGELOG_MAX_AGE_SECONDS: + sys.exit( + f"{SYNC_CHANGELOG} is {age / 3600:.1f} hours old; " + "write the changelog text for this codegen sync" + ) + text = SYNC_CHANGELOG.read_text().strip() + if not text: + sys.exit(f"{SYNC_CHANGELOG} is empty") + return f"Codegen sync for {SMITHY_PYTHON_COMMITS}/{previous}..{commit} {text}" @contextmanager @@ -242,7 +284,7 @@ def _build(models: list[Path], gradle_args: list[str]) -> None: ) -def _install_client(slug: str, docs_deps: list[str]) -> None: +def _install_client(slug: str, docs_deps: list[str], sync_entry: str | None) -> None: name = f"aws-sdk-{slug}" projection = ( CODEGEN_DIR @@ -271,6 +313,30 @@ def _install_client(slug: str, docs_deps: list[str]) -> None: services = ",".join(CLIENTS_REQUIRING_BOTO3[name]) _uv_add(dest, "test", ["boto3", f"types-boto3-lite[{services}]"]) print(f"wrote {dest}") + if sync_entry is not None and _changed(dest): + _write_sync_fragment(dest, name, sync_entry) + + +def _changed(dest: Path) -> bool: + out = subprocess.run( + ["git", "status", "--porcelain", "--", ".", ":(exclude).changes"], + cwd=dest, + check=True, + capture_output=True, + text=True, + ).stdout + return bool(out.strip()) + + +def _write_sync_fragment(dest: Path, name: str, description: str) -> None: + # A fixed name lets a rerun of `make clients` overwrite rather than duplicate. + fragment = ( + dest / ".changes" / "next-release" / f"{name}-enhancement-codegen-sync.json" + ) + fragment.parent.mkdir(parents=True, exist_ok=True) + entry = {"type": "enhancement", "description": description} + fragment.write_text(json.dumps(entry, indent=2) + "\n") + print(f"wrote {fragment}") def _relative_files(root: Path) -> set[str]: diff --git a/codegen/smithy-python.toml b/codegen/smithy-python.toml index 6d7079aa..af07b80f 100644 --- a/codegen/smithy-python.toml +++ b/codegen/smithy-python.toml @@ -1,3 +1,6 @@ # The smithy-python commit `make clients` builds codegen from. May be any commit # reachable from the colocated smithy-python checkout, including a local one. commit = "b8eceadb0573bd0c757def9a2b26fe7bf938b239" +# The commit this pointer moved from; with `commit` it bounds the range linked +# in codegen sync changelog entries. +previous_commit = "6c2dfa2c689c2352985ffb05eef1bfc5e3a697e7" From c84ad531d42b2f0a0b5f87ce51db69790b1a4409 Mon Sep 17 00:00:00 2001 From: George Fu Date: Wed, 4 Nov 2026 17:35:14 +0000 Subject: [PATCH 2/3] Include the smithy-python commit range in codegen sync changelog errors --- codegen/gen_client.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/codegen/gen_client.py b/codegen/gen_client.py index 3ad75c2d..3837006c 100644 --- a/codegen/gen_client.py +++ b/codegen/gen_client.py @@ -123,22 +123,18 @@ def _pinned_commits() -> tuple[str, str]: def _sync_entry(previous: str, commit: str) -> str: + changes = f"{SMITHY_PYTHON_COMMITS}/{previous}..{commit}" + reminder = f"write the changelog text for this codegen sync of {changes}" if not SYNC_CHANGELOG.exists(): SYNC_CHANGELOG.touch() - sys.exit( - f"created empty {SYNC_CHANGELOG}; " - "write the changelog text for this codegen sync" - ) + sys.exit(f"created empty {SYNC_CHANGELOG}; {reminder}") age = time.time() - SYNC_CHANGELOG.stat().st_mtime if age > SYNC_CHANGELOG_MAX_AGE_SECONDS: - sys.exit( - f"{SYNC_CHANGELOG} is {age / 3600:.1f} hours old; " - "write the changelog text for this codegen sync" - ) + sys.exit(f"{SYNC_CHANGELOG} is {age / 3600:.1f} hours old; {reminder}") text = SYNC_CHANGELOG.read_text().strip() if not text: - sys.exit(f"{SYNC_CHANGELOG} is empty") - return f"Codegen sync for {SMITHY_PYTHON_COMMITS}/{previous}..{commit} {text}" + sys.exit(f"{SYNC_CHANGELOG} is empty; {reminder}") + return f"Codegen sync for {changes} {text}" @contextmanager From 3181e11b0b3c4fc099cb49362cb5478e33432765 Mon Sep 17 00:00:00 2001 From: George Fu Date: Wed, 4 Nov 2026 19:26:21 +0000 Subject: [PATCH 3/3] Separate clients and fragments make targets --- Makefile | 9 ++-- codegen/gen_client.py | 84 +++++------------------------------- codegen/gen_fragments.py | 87 ++++++++++++++++++++++++++++++++++++++ codegen/smithy-python.toml | 2 +- 4 files changed, 105 insertions(+), 77 deletions(-) create mode 100644 codegen/gen_fragments.py diff --git a/Makefile b/Makefile index ecb57417..086d5682 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ SMITHY_PYTHON ?= ../smithy-python # Clients per gradle build in `make clients`; lower it if the build runs out of heap. BATCH_SIZE ?= 25 -.PHONY: docs docs-serve docs-clean docs-install docs-lock venv gen clients publish-codegen aio-check static-analysis test-unit test-functional test-integration +.PHONY: docs docs-serve docs-clean docs-install docs-lock venv gen clients fragments publish-codegen aio-check static-analysis test-unit test-functional test-integration # Each client and package runs in its own .venv. Dependencies outside this repo, # including smithy-python, resolve from PyPI at the versions each one declares. @@ -57,11 +57,14 @@ test-integration: # Regenerate every client from codegen/aws-models with codegen built from the # smithy-python commit in codegen/smithy-python.toml. Each client is cleared first, # keeping only the paths in gen_client.py's KEEP, then scripts/client-overlays is applied. -# Requires CODEGEN_SYNC_CHANGELOG.md in the repo root, written within 8 hours; each -# changed client gets a changelog fragment from it, prefixed with the commit range. clients: SMITHY_PYTHON=$(abspath $(SMITHY_PYTHON)) uv run python codegen/gen_client.py --all --pinned --batch-size $(BATCH_SIZE) +# Only for a smithy-python pointer move: writes a changelog fragment from +# CODEGEN_SYNC_CHANGELOG.md for every generated client. +fragments: + uv run --no-project --python $(PYTHON_VERSION) python codegen/gen_fragments.py + ########################## Granular targets below ########################## # Publishes the smithy-python codegen jars to the local Maven cache so `gen` diff --git a/codegen/gen_client.py b/codegen/gen_client.py index 3837006c..ba303eba 100644 --- a/codegen/gen_client.py +++ b/codegen/gen_client.py @@ -5,8 +5,6 @@ accepts dynamodb, aws_sdk_dynamodb, aws-sdk-dynamodb, or bedrock-runtime forms. --pinned builds codegen from the smithy-python commit in codegen/smithy-python.toml; without it, the codegen last published to the local Maven cache is used. - --pinned requires a CODEGEN_SYNC_CHANGELOG.md in the repo root written within the - last 8 hours; each client whose output changed gets a changelog fragment from it. """ import argparse @@ -16,7 +14,6 @@ import shutil import subprocess import sys -import time import tomllib from collections.abc import Iterator from contextlib import contextmanager @@ -32,10 +29,6 @@ DEFAULT_BATCH_SIZE = 25 POINTER = CODEGEN_DIR / "smithy-python.toml" PINNED_DIR = CODEGEN_DIR / ".smithy-python" -SYNC_CHANGELOG = REPO / "CODEGEN_SYNC_CHANGELOG.md" -SMITHY_PYTHON_COMMITS = "https://github.com/smithy-lang/smithy-python/commits" -# An older file is assumed to be left over from a previous `make clients`. -SYNC_CHANGELOG_MAX_AGE_SECONDS = 8 * 60 * 60 # Paths codegen never emits; they survive the pre-generation delete. A path codegen # does emit but that must differ from codegen output belongs in client-overlays. @@ -83,58 +76,27 @@ def main() -> None: sys.exit("no clients selected") if args.pinned: - previous, commit = _pinned_commits() - sync_entry = _sync_entry(previous, commit) - with _pinned_codegen(commit) as gradle_args: - _generate(models, args.batch_size, gradle_args, sync_entry) + with _pinned_codegen(_pinned_commit()) as gradle_args: + _generate(models, args.batch_size, gradle_args) else: - _generate(models, args.batch_size, [], None) + _generate(models, args.batch_size, []) -def _generate( - models: list[Path], - batch_size: int, - gradle_args: list[str], - sync_entry: str | None, -) -> None: +def _generate(models: list[Path], batch_size: int, gradle_args: list[str]) -> None: docs_deps = _docs_deps() for start in range(0, len(models), batch_size): batch = models[start : start + batch_size] _build(batch, gradle_args) for model in batch: - _install_client(model.stem, docs_deps, sync_entry) + _install_client(model.stem, docs_deps) -def _pinned_commits() -> tuple[str, str]: +def _pinned_commit() -> str: with POINTER.open("rb") as f: - pointer = tomllib.load(f) - commits = [] - for key in ("previous_commit", "commit"): - value = pointer.get(key) - if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{40}", value): - sys.exit( - f"{POINTER}: {key} must be a full 40-character hash, got {value!r}" - ) - commits.append(value) - previous, commit = commits - if previous == commit: - sys.exit(f"{POINTER}: previous_commit and commit must differ") - return previous, commit - - -def _sync_entry(previous: str, commit: str) -> str: - changes = f"{SMITHY_PYTHON_COMMITS}/{previous}..{commit}" - reminder = f"write the changelog text for this codegen sync of {changes}" - if not SYNC_CHANGELOG.exists(): - SYNC_CHANGELOG.touch() - sys.exit(f"created empty {SYNC_CHANGELOG}; {reminder}") - age = time.time() - SYNC_CHANGELOG.stat().st_mtime - if age > SYNC_CHANGELOG_MAX_AGE_SECONDS: - sys.exit(f"{SYNC_CHANGELOG} is {age / 3600:.1f} hours old; {reminder}") - text = SYNC_CHANGELOG.read_text().strip() - if not text: - sys.exit(f"{SYNC_CHANGELOG} is empty; {reminder}") - return f"Codegen sync for {changes} {text}" + commit = tomllib.load(f)["commit"] + if not re.fullmatch(r"[0-9a-f]{40}", commit): + sys.exit(f"{POINTER}: commit must be a full 40-character hash, got {commit!r}") + return commit @contextmanager @@ -280,7 +242,7 @@ def _build(models: list[Path], gradle_args: list[str]) -> None: ) -def _install_client(slug: str, docs_deps: list[str], sync_entry: str | None) -> None: +def _install_client(slug: str, docs_deps: list[str]) -> None: name = f"aws-sdk-{slug}" projection = ( CODEGEN_DIR @@ -309,30 +271,6 @@ def _install_client(slug: str, docs_deps: list[str], sync_entry: str | None) -> services = ",".join(CLIENTS_REQUIRING_BOTO3[name]) _uv_add(dest, "test", ["boto3", f"types-boto3-lite[{services}]"]) print(f"wrote {dest}") - if sync_entry is not None and _changed(dest): - _write_sync_fragment(dest, name, sync_entry) - - -def _changed(dest: Path) -> bool: - out = subprocess.run( - ["git", "status", "--porcelain", "--", ".", ":(exclude).changes"], - cwd=dest, - check=True, - capture_output=True, - text=True, - ).stdout - return bool(out.strip()) - - -def _write_sync_fragment(dest: Path, name: str, description: str) -> None: - # A fixed name lets a rerun of `make clients` overwrite rather than duplicate. - fragment = ( - dest / ".changes" / "next-release" / f"{name}-enhancement-codegen-sync.json" - ) - fragment.parent.mkdir(parents=True, exist_ok=True) - entry = {"type": "enhancement", "description": description} - fragment.write_text(json.dumps(entry, indent=2) + "\n") - print(f"wrote {fragment}") def _relative_files(root: Path) -> set[str]: diff --git a/codegen/gen_fragments.py b/codegen/gen_fragments.py new file mode 100644 index 00000000..8647a414 --- /dev/null +++ b/codegen/gen_fragments.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Write a codegen sync changelog fragment for every generated client. + +Usage: python codegen/gen_fragments.py + Only for a smithy-python pointer move; the text comes from CODEGEN_SYNC_CHANGELOG.md + in the repo root. +""" + +import json +import re +import sys +import time +import tomllib +from pathlib import Path + +CODEGEN_DIR = Path(__file__).resolve().parent +REPO = CODEGEN_DIR.parent +CLIENTS = REPO / "clients" +MODELS = CODEGEN_DIR / "aws-models" +POINTER = CODEGEN_DIR / "smithy-python.toml" +SYNC_CHANGELOG = REPO / "CODEGEN_SYNC_CHANGELOG.md" +SMITHY_PYTHON_COMMITS = "https://github.com/smithy-lang/smithy-python/commits" +# An older file is assumed to be left over from a previous sync. +SYNC_CHANGELOG_MAX_AGE_SECONDS = 8 * 60 * 60 + + +def main() -> None: + previous, commit = _pointer_commits() + description = _description(previous, commit) + for name in _generated_clients(): + _write_fragment(name, commit, description) + + +def _pointer_commits() -> tuple[str, str]: + with POINTER.open("rb") as f: + pointer = tomllib.load(f) + commits = [] + for key in ("previous_commit", "commit"): + value = pointer.get(key) + if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{40}", value): + sys.exit( + f"{POINTER}: {key} must be a full 40-character hash, got {value!r}" + ) + commits.append(value) + previous, commit = commits + if previous == commit: + sys.exit(f"{POINTER}: previous_commit and commit must differ") + return previous, commit + + +def _description(previous: str, commit: str) -> str: + changes = f"{SMITHY_PYTHON_COMMITS}/{previous}..{commit}" + reminder = f"write the changelog text for this codegen sync of {changes}" + if not SYNC_CHANGELOG.exists(): + SYNC_CHANGELOG.touch() + sys.exit(f"created empty {SYNC_CHANGELOG}; {reminder}") + age = time.time() - SYNC_CHANGELOG.stat().st_mtime + if age > SYNC_CHANGELOG_MAX_AGE_SECONDS: + sys.exit(f"{SYNC_CHANGELOG} is {age / 3600:.1f} hours old; {reminder}") + text = SYNC_CHANGELOG.read_text().strip() + if not text: + sys.exit(f"{SYNC_CHANGELOG} is empty; {reminder}") + link = f"[smithy-python {previous[:7]}..{commit[:7]}]({changes})" + return f"Codegen sync for {link}: {text}" + + +def _generated_clients() -> list[str]: + return [f"aws-sdk-{model.stem}" for model in sorted(MODELS.glob("*.json"))] + + +def _write_fragment(name: str, commit: str, description: str) -> None: + # Keyed by target commit: a rerun for the same pin overwrites, a later sync adds. + fragment = ( + CLIENTS + / name + / ".changes" + / "next-release" + / f"{name}-enhancement-codegen-sync-{commit}.json" + ) + fragment.parent.mkdir(parents=True, exist_ok=True) + entry = {"type": "enhancement", "description": description} + fragment.write_text(json.dumps(entry, indent=2) + "\n") + print(f"wrote {fragment}") + + +if __name__ == "__main__": + main() diff --git a/codegen/smithy-python.toml b/codegen/smithy-python.toml index af07b80f..dab7d93c 100644 --- a/codegen/smithy-python.toml +++ b/codegen/smithy-python.toml @@ -3,4 +3,4 @@ commit = "b8eceadb0573bd0c757def9a2b26fe7bf938b239" # The commit this pointer moved from; with `commit` it bounds the range linked # in codegen sync changelog entries. -previous_commit = "6c2dfa2c689c2352985ffb05eef1bfc5e3a697e7" +previous_commit = "505312562bf78d45265145ae3ae9f874c00464d1"