diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b608217..0387d90 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -216,6 +216,10 @@ jobs: # the repository variable DRY_RUN=true goes through publishing without publishing anything: # nothing is pushed or signed, and no release is drafted DRY_RUN: ${{ vars.DRY_RUN == 'true' }} + outputs: + # the version published, and whether anything was, for the directory-scan job + version: ${{ steps.version.outputs.VERSION }} + published: ${{ steps.publish.outputs.published == 'true' }} steps: - name: Dry run if: env.DRY_RUN == 'true' @@ -229,7 +233,7 @@ jobs: make tools make -s tools-path >> "${GITHUB_PATH}" - name: Install cosign - uses: sigstore/cosign-installer@v4.1.2 + uses: sigstore/cosign-installer@v4.1.3 - name: Download components.tar uses: actions/download-artifact@v8 with: @@ -250,6 +254,12 @@ jobs: if: startsWith(github.ref, 'refs/tags/') id: tag_version run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT + - name: Get publish version + # the interface version on main, the tag on a tag, without a leading `v`, e.g. 0.1.0 for v0.1.0 + id: version + env: + VERSION: ${{ case(github.ref == 'refs/heads/main', steps.interface_version.outputs.VERSION, steps.tag_version.outputs.VERSION) }} + run: echo "VERSION=${VERSION#v}" >> $GITHUB_OUTPUT - name: Check tag matches the workspace version # cargo binstall finds the cli archives at the release tagged with the cli crate's version, # and publishing the release publishes the crates to crates.io at that version. A repository @@ -271,11 +281,16 @@ jobs: - name: Publish components to gchr.io # only publish tags and pre-releases from main if: startsWith(github.ref, 'refs/tags/') || (github.ref == 'refs/heads/main' && contains(steps.interface_version.outputs.VERSION, '-')) - run: make publish + id: publish + run: | + make publish + if [ "${DRY_RUN}" != "true" ] ; then + echo "published=true" >> "${GITHUB_OUTPUT}" + fi env: PUBLISH_LOG: "${{ runner.temp }}/published.txt" REPOSITORY: "ghcr.io/${{ github.repository }}" - VERSION: "${{ case(github.ref == 'refs/heads/main', steps.interface_version.outputs.VERSION, steps.tag_version.outputs.VERSION) }}" + VERSION: "${{ steps.version.outputs.VERSION }}" - name: Draft release notes if: startsWith(github.ref, 'refs/tags/') || (github.ref == 'refs/heads/main' && contains(steps.interface_version.outputs.VERSION, '-')) run: | @@ -302,3 +317,28 @@ jobs: ${{ needs.settings.outputs.cli != '' && 'dist/*' || '' }} fail_on_unmatched_files: true token: ${{ secrets.GITHUB_TOKEN }} + + # asks componentized/directory to scan the components just published. The organization variable + # COMPONENTIZED_DIRECTORY_SCAN=true limits it to componentized repositories, a repository outside + # it uses the build without the custodian's access to the directory. + directory-scan: + # !cancelled() rather than the default success(), which is false when the cli is skipped + if: ${{ !cancelled() && vars.COMPONENTIZED_DIRECTORY_SCAN == 'true' && needs.publish.result == 'success' && needs.publish.outputs.published == 'true' }} + needs: publish + # the app token is used, not the GITHUB_TOKEN + permissions: {} + runs-on: ubuntu-latest + steps: + - name: Create directory token + id: directory-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ secrets.CUSTODIAN_CLIENT_ID }} + private-key: ${{ secrets.CUSTODIAN_PRIVATE_KEY }} + repositories: directory + permission-contents: write + - name: Request a directory scan + env: + GH_TOKEN: ${{ steps.directory-token.outputs.token }} + VERSION: ${{ needs.publish.outputs.version }} + run: gh api repos/componentized/directory/dispatches -f event_type=release -f "client_payload[only]=${GITHUB_REPOSITORY}" -f "client_payload[version]=${VERSION}" diff --git a/build/MANIFEST b/build/MANIFEST index 454fb0c..09b2bef 100644 --- a/build/MANIFEST +++ b/build/MANIFEST @@ -1,22 +1,22 @@ -1c97ff9af5103fd562f230f58a49bd88a65d0539ad69be0cabd1d91885a1b609 build/build.mk -51af1124aa1fe472baa000dbe157879af75462971400af774f81dcb8eb15b561 build/components.mk +0edf07b7ec073ce666d847f9c9976ad68822d80d7c1651739459ff27f8b3fa0f .github/workflows/bump-version.yaml +5b2adefd1b6278712f2054c62a9845fee41a717b6977a1202740ac176d0a3d54 .github/workflows/ci.yaml +b09c55d4207eb188036d3a7ba118c68b2547e958362d898714408af3a1c4afa6 .github/workflows/release.yaml +0a5da65531ed134a43f0aba94aacccd84707d548c348eb31868365c8c400978a .github/workflows/update-build.yaml +f9b6ad1d205adf8198aecdf9130fe149a362cbe805c077f19c17dc5562b60405 build/README.md +10fbbe21746d38d841dbc2ddf94b7daffbecd51715a516c694940b4d79f51067 build/build.mk +4d3b9ec8b5b4f8b58df9997a064d353f8792f1001f58d984ba80998748216a18 build/components.mk acb1b32ba485536dc0b834cb351f41a6804339e4beab0b0391d149d069724a89 build/local.jq 5560aa0d6be4878992a77441c0631da5ae4d52cc65f52185b2af333f70a58b59 build/pinned.jq df556e7ce9919c72815db01ac06ef541ecb1b9f920facb51ccbb635d17ee656a build/publish.mk -f9b6ad1d205adf8198aecdf9130fe149a362cbe805c077f19c17dc5562b60405 build/README.md 2e98011760ad48ff0d706df309f95a067e0edc82c9ec55292cd2522d4e347c47 build/scripts/bump-version.sh c651f8d58218a8ad6475857a70364e2df9e5607aa066800f19d0b65ec97a02da build/scripts/check-build.sh 45c5f4b5e7a4525c6da2780687b705536f2a57f40c18eda0ec84015c5c38df3e build/scripts/create-commit.sh -9cef942742a92e6c8845b72ca59ba9c13154415956c1be7dab72d61acacbfc2e build/scripts/publish.sh +c125f2fdad69e6ad5caa660659b51a420be867fea36cd274aeceac0c10eb1594 build/scripts/publish.sh d91382922ff27332373518e782a6f003081db04809b5a87d00d77531d9f95c6b build/scripts/update-build.sh 0477a5e4ee8fdb61e6cf67205c43aafe52be25501b8e9d41e72ce805b5b3c0df build/scripts/wac-deps.sh 58fb46d7754b0a8316350e2733cc27baf2e85502cd26ca36e1091207b10af5e4 build/scripts/wit-canonical.sh 44f65271d65b043e9affcbaddd9e958ecb33a60e945d7ce869e6408925620626 build/sync.mk +c08767afa3a60b96ab908ab8cf2169eb03eeb5947b9d1b9c3dc8444eae52cabc build/tools.mk d3a8914a14ca9ffae55c2352b72bd86f9b482beef1918c401743a27581bb52f5 build/tools/Cargo.toml dab14a274377b620391d4979812ef34997e07b01ffd6b90e6df3b37131e3a3eb build/tools/lib.rs -c08767afa3a60b96ab908ab8cf2169eb03eeb5947b9d1b9c3dc8444eae52cabc build/tools.mk 7010449b71c86de391f25e32a8bf06419f1a277479108aa31250b8aead56e45d build/wit.mk -0edf07b7ec073ce666d847f9c9976ad68822d80d7c1651739459ff27f8b3fa0f .github/workflows/bump-version.yaml -3cf7bd99aca63aac045bbfcfd2e3912696817fc5473ff47b1f0e50bf5d03da33 .github/workflows/ci.yaml -b09c55d4207eb188036d3a7ba118c68b2547e958362d898714408af3a1c4afa6 .github/workflows/release.yaml -0a5da65531ed134a43f0aba94aacccd84707d548c348eb31868365c8c400978a .github/workflows/update-build.yaml diff --git a/build/VERSION b/build/VERSION index 5366600..8c43fb4 100644 --- a/build/VERSION +++ b/build/VERSION @@ -1 +1 @@ -v0.1.2 +v0.1.4 diff --git a/build/build.mk b/build/build.mk index 05281af..1f57898 100644 --- a/build/build.mk +++ b/build/build.mk @@ -18,7 +18,8 @@ export WKG_CONFIG_FILE := $(abspath .config/wasm-pkg/config.toml) .DEFAULT_GOAL := all # the rust target components are built for, `wasm32-wasip3` builds components that run std threads on -# component model cooperative threads, see build/components.mk. A component built with cargo may name +# component model cooperative threads, see build/components.mk, `wasm32-wasip2` builds components +# rust links itself, e.g. a command. A component built with cargo may name # its own in its Cargo.toml, `COMPONENT_TARGET` given on the command line overrides it, e.g. # # [package.metadata.componentized] diff --git a/build/components.mk b/build/components.mk index 69964c9..e94505b 100644 --- a/build/components.mk +++ b/build/components.mk @@ -4,7 +4,7 @@ COMPONENTS_DIR := target/components # composition names without instantiating it, e.g. `import stderr: wasi:cli/stderr@0.3.0;` WAC_DEPS_DIR := target/wac-deps -COMPONENTS = $(sort $(foreach file,$(wildcard $(addprefix components/*/,wit/*.constants.wit *.properties *.wac *.wkg Cargo.toml)),$(word 2,$(subst /, ,$(file))))) +COMPONENTS = $(sort $(foreach file,$(wildcard $(addprefix components/*/,wit/*.constants.wit *.properties *.wac *.plug *.wkg Cargo.toml)),$(word 2,$(subst /, ,$(file))))) # the cargo profiles components are built with, `component` and `component-debug` when Cargo.toml # defines them, e.g. to build smaller components than the release profile a cli is built with @@ -13,8 +13,12 @@ COMPONENT_DEBUG_PROFILE ?= $(if $(shell grep -s '^\[profile\.component-debug\]' # the core module cargo built for a component with a profile, e.g. `$(call cargo_wasm,gate,dev)` for # target/wasm32-unknown-unknown/debug/gate.wasm, `$(call cargo_wasm,gate,release,.opt)` for the -# module wasm-opt optimized -cargo_wasm = target/$(call component_target,$(1))/$(if $(filter dev,$(2)),debug,$(2))/$(subst -,_,$(1))$(3).wasm +# module wasm-opt optimized. For wasm32-wasip2 cargo builds a component rather than a core module. +cargo_wasm = target/$(call component_target,$(1))/$(if $(filter dev,$(2)),debug,$(2))/$(call cargo_artifact,$(1))$(3).wasm + +# the name of the wasm file cargo builds for a package, a library's has `_` for each `-`, a binary +# keeps the package's name, e.g. a command for wasm32-wasip2 with a src/main.rs and no src/lib.rs +cargo_artifact = $(if $(and $(wildcard components/$(1)/src/main.rs),$(if $(wildcard components/$(1)/src/lib.rs),,bin)),$(1),$(subst -,_,$(1))) # the files the components cargo builds need besides the sources, e.g. a component the library # embeds, `CARGO_COMPONENT_DEPS := crates/componentized-static-config/constants-config.wasm` @@ -34,13 +38,19 @@ export CARGO_TARGET_WASM32_WASIP3_LINKER := $(WASI_SDK_PATH)/bin/wasm-component- export CARGO_TARGET_WASM32_WASIP3_RUSTFLAGS := -Clink-self-contained=n -Clink-arg=-L$(WASI_SYSROOT_LIB) -Clink-arg=$(WASI_SYSROOT_LIB)/crt1-reactor.o -Clink-arg=--export=cabi_realloc -Clink-arg=--skip-wit-component endif -# turn the core module cargo built into a component -componentize = $(WASM_TOOLS) component new $(1) -o $(2) +# turn the core module cargo built for a component into a component, e.g. `$(call +# componentize,gate,,)`. For wasm32-wasip2 rust links a component itself, with +# wasm-component-ld, which is copied. +componentize = $(if $(filter wasm32-wasip2,$(call component_target,$(1))),cp $(2) $(3),$(WASM_TOOLS) component new $(2) -o $(3)) # the release build of a component is optimized again for size with wasm-opt when set, e.g. -# `COMPONENT_OPTIMIZE := true`, the build provides wasm-opt to every repository +# `COMPONENT_OPTIMIZE := true`, the build provides wasm-opt to every repository. wasm-opt optimizes +# core modules, a component built for wasm32-wasip2 is not optimized again. COMPONENT_OPTIMIZE ?= +# whether the release build of a component is optimized again, e.g. `$(call component_optimize,gate)` +component_optimize = $(if $(COMPONENT_OPTIMIZE),$(filter-out wasm32-wasip2,$(call component_target,$(1)))) + # the wasm features rust enables for wasm32-unknown-unknown and wasm32-wasip3, `rustc --print cfg # --target wasm32-unknown-unknown`. The stripped modules don't say which features they use, wasm-opt # is limited to these so it doesn't introduce instructions rust wouldn't emit. @@ -105,6 +115,26 @@ ${COMPONENTS_DIR}/$1/$1.wasm: components/$1/$1.wac $$(foreach component,$$(WAC_D ${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.wac $$(foreach component,$$(WAC_DEPS_$1),$${COMPONENTS_DIR}/$$(component)/$$(component).debug.wasm) ${COMPONENTS_DIR}/$1/README.md ${WAC_DEPS_DIR} | $(call tool,wac-cli) $(WAC) compose --deps-dir ${WAC_DEPS_DIR} $$(foreach component,$$(WAC_DEPS_$1),-d local:$$(component)=$${COMPONENTS_DIR}/$$(component)/$$(component).debug.wasm) -o ${COMPONENTS_DIR}/$1/$1.debug.wasm components/$1/$1.wac +else ifneq ($(wildcard components/$1/$1.plug),) + +# the components plugged together with `wac plug`, each a component of the repository named on a line +# of its own, the socket first, then each plug, `#` starts a comment, e.g. +# +# greeter +# defaults +# +# Where the socket and a plug import different versions of the same interface, e.g. wasi:io@0.2.6 and +# wasi:io@0.2.12, the plugged component imports the newer one, a composition the one it imports first. +PLUG_COMPONENTS_$1 := $$(shell sed 's/\#.*//' components/$1/$1.plug) +PLUG_SOCKET_$1 := $$(firstword $$(PLUG_COMPONENTS_$1)) +PLUG_PLUGS_$1 := $$(wordlist 2,$$(words $$(PLUG_COMPONENTS_$1)),$$(PLUG_COMPONENTS_$1)) + +${COMPONENTS_DIR}/$1/$1.wasm: components/$1/$1.plug $$(foreach component,$$(PLUG_COMPONENTS_$1),$${COMPONENTS_DIR}/$$(component)/$$(component).wasm) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wac-cli) + $(WAC) plug $${COMPONENTS_DIR}/$$(PLUG_SOCKET_$1)/$$(PLUG_SOCKET_$1).wasm $$(foreach component,$$(PLUG_PLUGS_$1),--plug $${COMPONENTS_DIR}/$$(component)/$$(component).wasm) -o ${COMPONENTS_DIR}/$1/$1.wasm + +${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.plug $$(foreach component,$$(PLUG_COMPONENTS_$1),$${COMPONENTS_DIR}/$$(component)/$$(component).debug.wasm) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wac-cli) + $(WAC) plug $${COMPONENTS_DIR}/$$(PLUG_SOCKET_$1)/$$(PLUG_SOCKET_$1).debug.wasm $$(foreach component,$$(PLUG_PLUGS_$1),--plug $${COMPONENTS_DIR}/$$(component)/$$(component).debug.wasm) -o ${COMPONENTS_DIR}/$1/$1.debug.wasm + else ifneq ($(wildcard components/$1/$1.wkg),) ${COMPONENTS_DIR}/$1/$1.wasm: components/$1/$1.wkg ${COMPONENTS_DIR}/$1/README.md | $(call tool,wkg) @@ -116,18 +146,18 @@ ${COMPONENTS_DIR}/$1/$1.debug.wasm: components/$1/$1.wkg ${COMPONENTS_DIR}/$1/RE # cargo is checked last, other strategies may have a Cargo.toml for tests of non-rust sources else ifneq ($(wildcard components/$1/Cargo.toml),) -${COMPONENTS_DIR}/$1/$1.wasm: $(CARGO_COMPONENT_DEPS) Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) $(shell find crates -type f 2> /dev/null) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wasm-tools) $(if $(COMPONENT_OPTIMIZE),$(call tool,wasm-opt)) $(if $(filter wasm32-wasip3,$(call component_target,$1)),$(WASI_SDK) $(WASI_LIBC)) +${COMPONENTS_DIR}/$1/$1.wasm: $(CARGO_COMPONENT_DEPS) Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) $(shell find crates -type f 2> /dev/null) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wasm-tools) $(if $(call component_optimize,$1),$(call tool,wasm-opt)) $(if $(filter wasm32-wasip3,$(call component_target,$1)),$(WASI_SDK) $(WASI_LIBC)) cargo build -p $1 --target $(call component_target,$1) --profile $(COMPONENT_PROFILE) -ifneq ($(COMPONENT_OPTIMIZE),) +ifneq ($(call component_optimize,$1),) $(WASM_OPT) -Oz $(WASM_OPT_FEATURES) $(call cargo_wasm,$1,$(COMPONENT_PROFILE)) -o $(call cargo_wasm,$1,$(COMPONENT_PROFILE),.opt) - $(call componentize,$(call cargo_wasm,$1,$(COMPONENT_PROFILE),.opt),${COMPONENTS_DIR}/$1/$1.wasm) + $(call componentize,$1,$(call cargo_wasm,$1,$(COMPONENT_PROFILE),.opt),${COMPONENTS_DIR}/$1/$1.wasm) else - $(call componentize,$(call cargo_wasm,$1,$(COMPONENT_PROFILE)),${COMPONENTS_DIR}/$1/$1.wasm) + $(call componentize,$1,$(call cargo_wasm,$1,$(COMPONENT_PROFILE)),${COMPONENTS_DIR}/$1/$1.wasm) endif ${COMPONENTS_DIR}/$1/$1.debug.wasm: $(CARGO_COMPONENT_DEPS) Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) $(shell find crates -type f 2> /dev/null) ${COMPONENTS_DIR}/$1/README.md | $(call tool,wasm-tools) $(if $(filter wasm32-wasip3,$(call component_target,$1)),$(WASI_SDK) $(WASI_LIBC)) cargo build -p $1 --target $(call component_target,$1) --profile $(COMPONENT_DEBUG_PROFILE) - $(call componentize,$(call cargo_wasm,$1,$(COMPONENT_DEBUG_PROFILE)),${COMPONENTS_DIR}/$1/$1.debug.wasm) + $(call componentize,$1,$(call cargo_wasm,$1,$(COMPONENT_DEBUG_PROFILE)),${COMPONENTS_DIR}/$1/$1.debug.wasm) endif diff --git a/build/scripts/publish.sh b/build/scripts/publish.sh index 5771999..407b173 100755 --- a/build/scripts/publish.sh +++ b/build/scripts/publish.sh @@ -15,7 +15,9 @@ # gate.wasm -> ${REPOSITORY}/gate:0.1.0 # gate.debug.wasm -> ${REPOSITORY}/gate:0.1.0_debug # -# VERSION the version to publish, a leading `v` is dropped from the tag, e.g. v0.1.0 +# VERSION the version to publish, with or without a leading `v`, e.g. v0.1.0 or 0.1.0. The +# org.opencontainers.image.version annotation always has the `v`, the tag never +# does, e.g. v0.1.0 is tagged 0.1.0 # REPOSITORY the repository to publish to, e.g. ghcr.io/componentized/http # GITHUB_REPOSITORY the GitHub repository the components are built from, e.g. componentized/http # COMPONENTS_DIR the directory the components are built into, defaults to target/components @@ -72,8 +74,9 @@ git diff --quiet HEAD || revision="${commit}+dirty" url="https://github.com/${GITHUB_REPOSITORY}/tree/${commit}" [[ -f "components/${component}/README.md" ]] && url="${url}/components/${component}" -component_version="$VERSION" -[[ -n "$debug" ]] && component_version="${VERSION}+debug" +# the version has a leading `v`, the tag does not, whether or not VERSION has one +component_version="v${VERSION#v}" +[[ -n "$debug" ]] && component_version="${component_version}+debug" tag="${component_version#v}" tag="${tag//+/_}"