Update build from v0.1.1 to v0.1.2

.github/workflows/ci.yaml

@@ -5,6 +5,7 @@ on:
     branches:
       - "**"
       - "!dependabot/**"
+      - "!update-build/**"
     tags:
       - 'v[0-9]+\.[0-9]+\.[0-9]+-?**'
   pull_request: {}
@@ -33,7 +34,7 @@ jobs:
       - name: Install cargo binstall
         uses: cargo-bins/cargo-binstall@main
       - name: Install tools
-        # the versions pinned in tools/Cargo.toml, on the path for later steps
+        # the versions pinned in build/tools/Cargo.toml and tools/Cargo.toml, on the path for later steps
         run: |
           make tools
           make -s tools-path >> "${GITHUB_PATH}"

build/MANIFEST

@@ -1,21 +1,22 @@
 0edf07b7ec073ce666d847f9c9976ad68822d80d7c1651739459ff27f8b3fa0f  .github/workflows/bump-version.yaml
-fb1340584ef4774b44a80acc8112ef7fa5243dc61d63f19a1bd09bd64c2e7a94  .github/workflows/ci.yaml
+3cf7bd99aca63aac045bbfcfd2e3912696817fc5473ff47b1f0e50bf5d03da33  .github/workflows/ci.yaml
 b09c55d4207eb188036d3a7ba118c68b2547e958362d898714408af3a1c4afa6  .github/workflows/release.yaml
 0a5da65531ed134a43f0aba94aacccd84707d548c348eb31868365c8c400978a  .github/workflows/update-build.yaml
 f9b6ad1d205adf8198aecdf9130fe149a362cbe805c077f19c17dc5562b60405  build/README.md
 1c97ff9af5103fd562f230f58a49bd88a65d0539ad69be0cabd1d91885a1b609  build/build.mk
-b2a557bcf59210b2e859cdea7786d8ecf141383ac62e599970cfde251c8ed44b  build/components.mk
+51af1124aa1fe472baa000dbe157879af75462971400af774f81dcb8eb15b561  build/components.mk
 acb1b32ba485536dc0b834cb351f41a6804339e4beab0b0391d149d069724a89  build/local.jq
-7a92f0051f300b0e22f571e34d441c6cd93ccbfbecb3bf3cdbff7083f8124bdd  build/pinned.jq
+5560aa0d6be4878992a77441c0631da5ae4d52cc65f52185b2af333f70a58b59  build/pinned.jq
 df556e7ce9919c72815db01ac06ef541ecb1b9f920facb51ccbb635d17ee656a  build/publish.mk
 2e98011760ad48ff0d706df309f95a067e0edc82c9ec55292cd2522d4e347c47  build/scripts/bump-version.sh
 c651f8d58218a8ad6475857a70364e2df9e5607aa066800f19d0b65ec97a02da  build/scripts/check-build.sh
 45c5f4b5e7a4525c6da2780687b705536f2a57f40c18eda0ec84015c5c38df3e  build/scripts/create-commit.sh
 9cef942742a92e6c8845b72ca59ba9c13154415956c1be7dab72d61acacbfc2e  build/scripts/publish.sh
-d8d869c6df5dbaa0a11356e34e5e82b64587a47969b085f14b7037d96b2e108e  build/scripts/update-build.sh
+d91382922ff27332373518e782a6f003081db04809b5a87d00d77531d9f95c6b  build/scripts/update-build.sh
 0477a5e4ee8fdb61e6cf67205c43aafe52be25501b8e9d41e72ce805b5b3c0df  build/scripts/wac-deps.sh
 58fb46d7754b0a8316350e2733cc27baf2e85502cd26ca36e1091207b10af5e4  build/scripts/wit-canonical.sh
 44f65271d65b043e9affcbaddd9e958ecb33a60e945d7ce869e6408925620626  build/sync.mk
-0a52ef86f584169f9f2e0fd5f67ac9add84b8186fbb21d197508373eaa3ed9bc  build/tool-versions
-e605a028dbbacf58f82a975dad1479a439e5c92c8591a8bd1242b6153c4d8372  build/tools.mk
+c08767afa3a60b96ab908ab8cf2169eb03eeb5947b9d1b9c3dc8444eae52cabc  build/tools.mk
+d3a8914a14ca9ffae55c2352b72bd86f9b482beef1918c401743a27581bb52f5  build/tools/Cargo.toml
+dab14a274377b620391d4979812ef34997e07b01ffd6b90e6df3b37131e3a3eb  build/tools/lib.rs
 7010449b71c86de391f25e32a8bf06419f1a277479108aa31250b8aead56e45d  build/wit.mk

build/VERSION

@@ -1 +1 @@
-v0.1.1
+v0.1.2

build/components.mk

@@ -37,9 +37,9 @@ endif
 # turn the core module cargo built into a component
 componentize = $(WASM_TOOLS) component new $(1) -o $(2)
 
-# the release build of a component is optimized again for size with wasm-opt when it is pinned in
-# tools/Cargo.toml, `COMPONENT_OPTIMIZE=` builds without.
-COMPONENT_OPTIMIZE ?= $(filter wasm-opt,$(TOOLS))
+# the release build of a component is optimized again for size with wasm-opt when set, e.g.
+# `COMPONENT_OPTIMIZE := true`, the build provides wasm-opt to every repository
+COMPONENT_OPTIMIZE ?=
 
 # the wasm features rust enables for wasm32-unknown-unknown and wasm32-wasip3, `rustc --print cfg
 # --target wasm32-unknown-unknown`. The stripped modules don't say which features they use, wasm-opt

build/pinned.jq

@@ -1,6 +1,14 @@
-# generates target/tools/pinned.mk from `cargo metadata --no-deps` of tools/Cargo.toml, see build/tools.mk
-.packages[0] as $tools
-| ($tools.dependencies[].name | select($tools.metadata.bins[.] == null) | error("\(.) is missing from [package.metadata.bins] in tools/Cargo.toml")),
-  "TOOLS := \([$tools.dependencies[].name] | join(" "))",
-  "TOOLS_PINNED := \([$tools.dependencies[] | "\(.name)@\(.req | ltrimstr("="))"] | join(" "))",
-  ($tools.metadata.bins[][] | "\(ascii_upcase | gsub("-"; "_")) := $(TOOLS_BIN)/\(.)")
+# generates target/tools/pinned.mk from `cargo metadata --no-deps` of build/tools/Cargo.toml, the
+# tools the build provides, followed by that of the repository's tools/Cargo.toml when it has one,
+# slurped, see build/tools.mk. A tool pinned by the repository overrides the build's.
+def pinned($manifest):
+  .packages[0] as $tools
+  | $tools.dependencies[]
+  | {name, version: (.req | ltrimstr("=")),
+     bins: ($tools.metadata.bins[.name] // error("\(.name) is missing from [package.metadata.bins] in \($manifest)"))};
+([.[0] | pinned("build/tools/Cargo.toml")] + [.[1] // empty | pinned("tools/Cargo.toml")])
+| reduce .[] as $tool ({}; .[$tool.name] = $tool)
+| [.[]] | sort_by(.name)
+| "TOOLS := \(map(.name) | join(" "))",
+  "TOOLS_PINNED := \(map("\(.name)@\(.version)") | join(" "))",
+  (.[].bins[] | "\(ascii_upcase | gsub("-"; "_")) := $(TOOLS_BIN)/\(.)")

build/scripts/update-build.sh

@@ -8,8 +8,9 @@
 # build/MANIFEST, copies in the files of the release, listed in its SYNC file, and records the
 # version in build/VERSION and a checksum of each file in build/MANIFEST, checked by
 # build/scripts/check-build.sh. The current version the repository keeps in a synced file, the
-# default of the bump-version workflow, is carried forward. A tool pinned in tools/Cargo.toml to a version older than the
-# release's build/tool-versions requires is raised to that version.
+# default of the bump-version workflow, is carried forward. A tool the repository's tools/Cargo.toml
+# overrides is removed from it once the release's build/tools/Cargo.toml pins the same version or a
+# newer one.
 #
 # <version>         the release to update to, e.g. v0.2.0, the latest release by default
 # BUILD_REPOSITORY  the GitHub repository of the build, defaults to componentized/build
@@ -141,16 +142,20 @@ while read -r path; do
     echo "$(checksum "$path")  ${path}"
 done < "$synced" > build/MANIFEST
 
-# raise the tools pinned to a version older than the build requires, e.g. `wac-cli = "=0.11.0"`
-while read -r tool required; do
-    [[ -z "$tool" || "$tool" == \#* ]] && continue
-    pinned=$(sed -n "s/^${tool} = \"=\(.*\)\"$/\1/p" tools/Cargo.toml 2> /dev/null || true)
-    [[ -z "$pinned" ]] && continue
-    if [[ "$pinned" != "$required" && "$(printf '%s\n%s\n' "$pinned" "$required" | sort -V | head -n 1)" == "$pinned" ]]; then
-        perl -pi -e 's/^(\Q'"${tool}"'\E = "=)[^"]*(")/${1}'"${required}"'${2}/' tools/Cargo.toml
-        echo "updated ${tool} from ${pinned} to ${required} in tools/Cargo.toml, required by the build"
-    fi
-done < build/tool-versions
+# remove the overrides of the tools the build provides that the build has caught up with, pinned to
+# the build's version or an older one, e.g. `wasm-tools = "=1.262.0"` when the build pins 1.263.0. A
+# newer version stays, the repository is ahead of the build. The tool's dependency and its line in
+# [package.metadata.bins] both start with its name, e.g. `wasm-tools = ["wasm-tools"]`.
+if [[ -f tools/Cargo.toml ]]; then
+    while read -r tool provided; do
+        pinned=$(sed -n "s/^${tool} = \"=\(.*\)\"$/\1/p" tools/Cargo.toml)
+        [[ -z "$pinned" ]] && continue
+        if [[ "$(printf '%s\n%s\n' "$pinned" "$provided" | sort -V | tail -n 1)" == "$provided" ]]; then
+            perl -ni -e 'print unless /^\Q'"${tool}"'\E = /' tools/Cargo.toml
+            echo "removed ${tool} ${pinned} from tools/Cargo.toml, the build provides ${provided}"
+        fi
+    done < <(sed -n 's/^\([A-Za-z0-9_-]*\) = "=\(.*\)"$/\1 \2/p' build/tools/Cargo.toml)
+fi
 
 echo "updated the build to ${BUILD_REPOSITORY} ${version}"
 exit

build/tools.mk

@@ -4,18 +4,28 @@ TOOLS_BIN := $(abspath $(TOOLS_DIR))/bin
 # for the scripts and tests the recipes run, they look up the tools on the PATH
 export PATH := $(TOOLS_BIN):$(PATH)
 
-# the tools pinned in tools/Cargo.toml, generated when it changes, make restarts after generating it.
-# Defines TOOLS, TOOLS_PINNED with the version of each tool, e.g. `wkg@0.16.1`, and a variable for
-# each binary a tool installs, e.g. `WKG := $(TOOLS_BIN)/wkg`. Recipes run the pinned tools by path,
-# make 3.81 runs a simple command without a shell and looks it up on its own PATH, before the export
-# above, which would find a tool already on the system.
+# the tools the build provides, pinned in build/tools/Cargo.toml, and those the repository pins in
+# its own tools/Cargo.toml, optional: a tool it pins overrides the build's, others it only uses
+# itself. Generated when either changes, make restarts after generating it. Defines TOOLS,
+# TOOLS_PINNED with the version of each tool, e.g. `wkg@0.16.1`, and a variable for each binary a
+# tool installs, e.g. `WKG := $(TOOLS_BIN)/wkg`. Recipes run the pinned tools by path, make 3.81 runs
+# a simple command without a shell and looks it up on its own PATH, before the export above, which
+# would find a tool already on the system. A binary set on the command line is run in its place,
+# e.g. `make WKG=../wkg/target/release/wkg`.
+TOOLS_MANIFESTS := build/tools/Cargo.toml $(wildcard tools/Cargo.toml)
 TOOLS_MK := target/tools/pinned.mk
 include $(TOOLS_MK)
+# generated again when tools/Cargo.toml is added or removed, it records the manifests it read
+ifneq ($(strip $(TOOLS_MANIFESTS_PINNED)),$(strip $(TOOLS_MANIFESTS)))
+$(TOOLS_MK): tools-manifests-changed
+.PHONY: tools-manifests-changed
+endif
 
-# the tools built from this repository rather than pinned in tools/Cargo.toml, e.g. a cli the
+# the tools built from this repository rather than pinned, e.g. a cli the
 # components are built with, `LOCAL_TOOLS := componentized-constants-cli`. Each is a package of the
 # workspace, installed with the pinned tools, and installed again when its sources change, its own or
-# those of the workspace packages it depends on. A local tool replaces a pinned tool of the same name.
+# those of the workspace packages it depends on. A local tool replaces a pinned tool of the same name,
+# one the build provides too, e.g. componentized/constants builds its own constants cli.
 # Generated when the workspace changes, defines LOCAL_TOOL_PATH_<package>, LOCAL_TOOL_SOURCES_<package>
 # and a variable for each binary the package installs, like the pinned tools, e.g.
 # `CONSTANTS := $(TOOLS_BIN)/constants`.
@@ -37,11 +47,12 @@ CARGO_INSTALL := $(if $(shell command -v cargo-binstall 2> /dev/null),cargo bins
 clean-tools:
 	rm -rf ${TOOLS_DIR}
 
-# each tool is a dependency of the tools package, the binaries it installs are listed in
-# [package.metadata.bins]
-$(TOOLS_MK): tools/Cargo.toml build/pinned.jq
+# each tool is a dependency of a tools package, the binaries it installs are listed in
+# [package.metadata.bins]. The repository's tools/Cargo.toml is read last, overriding the build's.
+$(TOOLS_MK): $(TOOLS_MANIFESTS) build/pinned.jq
 	@mkdir -p $(@D)
-	@set -o pipefail; cargo metadata --manifest-path tools/Cargo.toml --no-deps --format-version 1 | jq -r -f build/pinned.jq > $@.tmp
+	@set -o pipefail; for manifest in $(TOOLS_MANIFESTS); do cargo metadata --manifest-path $$manifest --no-deps --format-version 1; done | jq -rs -f build/pinned.jq > $@.tmp
+	@echo "TOOLS_MANIFESTS_PINNED := $(TOOLS_MANIFESTS)" >> $@.tmp
 	@mv $@.tmp $@
 
 # the packages of the workspace, generated again when LOCAL_TOOLS changes. The workspace's members may
@@ -62,7 +73,7 @@ tool = $(TOOLS_DIR)/.installed/$(1)@$(call tool_version,$(1))
 # among the prerequisites, it may be order-only.
 tool_prerequisite = $(if $(filter $(1),$(LOCAL_TOOLS)),$(call tool,$(1)),| $(call tool,$(1)))
 
-.PHONY: tools ## Install the cli tools pinned in tools/Cargo.toml, and those built from the workspace
+.PHONY: tools ## Install the cli tools the build provides, those pinned in tools/Cargo.toml, and those built from the workspace
 tools: $(foreach name,$(sort $(TOOLS) $(LOCAL_TOOLS)),$(call tool,$(name)))
 
 .PHONY: tools-path ## Print the directory of the installed tools for this platform, to add to the PATH

build/tools/Cargo.toml

@@ -0,0 +1,36 @@
+# The versions of the cli tools the build provides, installed by `make tools` with cargo binstall.
+#
+# This package is never built, it lists the tools as dependencies so dependabot bumps them in
+# componentized/build. Each tool is pinned to an exact version with `=`, the Makefile installs that
+# version. A repository overrides a tool in its own tools/Cargo.toml, with a newer version, or builds
+# it from its workspace with LOCAL_TOOLS, and adds the tools only it uses there, see build/tools.mk.
+[package]
+name = "build-tools"
+version = "0.0.0"
+edition = "2024"
+license = "Apache-2.0"
+publish = false
+
+[lib]
+path = "lib.rs"
+
+# the binaries each tool installs, the Makefile runs each by path from a variable named after it,
+# e.g. `$(WASM_TOOLS)` for `wasm-tools`
+[package.metadata.bins]
+componentized-constants-cli = ["constants"]
+componentized-static-config-cli = ["static-config"]
+wac-cli = ["wac"]
+wasm-opt = ["wasm-opt"]
+wasm-tools = ["wasm-tools"]
+wkg = ["wkg"]
+
+[dependencies]
+componentized-constants-cli = "=0.2.0"
+componentized-static-config-cli = "=0.3.0"
+wac-cli = "=0.12.0"
+wasm-opt = "=0.116.1"
+wasm-tools = "=1.263.0"
+wkg = "=0.16.1"
+
+# not a member of the repository's workspace
+[workspace]

build/tools/lib.rs

@@ -0,0 +1 @@
+//! Never built, see `Cargo.toml`.

tools/Cargo.toml

@@ -15,22 +15,10 @@ path = "lib.rs"
 # the binaries each tool installs, the Makefile runs each by path from a variable named after it,
 # e.g. `$(STATIC_CONFIG)` for `static-config`
 [package.metadata.bins]
-componentized-constants-cli = ["constants"]
-componentized-static-config-cli = ["static-config"]
-wac-cli = ["wac"]
-wasm-opt = ["wasm-opt"]
-wasm-tools = ["wasm-tools"]
 wasmtime-cli = ["wasmtime"]
-wkg = ["wkg"]
 
 [dependencies]
-componentized-constants-cli = "=0.2.0"
-componentized-static-config-cli = "=0.3.0"
-wac-cli = "=0.12.0"
-wasm-opt = "=0.116.1"
-wasm-tools = "=1.263.0"
 wasmtime-cli = "=50.0.1"
-wkg = "=0.16.1"
 
 # not a member of the repository's workspace
 [workspace]