From 44772f4c778f0852319085d945ac5534f4d5366e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zoe=20Faltib=C3=A0?= Date: Fri, 13 Nov 2026 18:23:55 +0100 Subject: [PATCH 1/5] demo: update service versions --- demo/compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/demo/compose.yaml b/demo/compose.yaml index 04f20ad..04bf53a 100644 --- a/demo/compose.yaml +++ b/demo/compose.yaml @@ -1,9 +1,9 @@ services: bitcoind: - image: registry.gitlab.com/hashbeam/docker/bitcoind:30.0 + image: registry.gitlab.com/hashbeam/docker/bitcoind:31.2 command: "-fallbackfee=0.0002" electrs: - image: registry.gitlab.com/hashbeam/docker/electrs:0.10.10 + image: registry.gitlab.com/hashbeam/docker/electrs:0.11.0 depends_on: - bitcoind proxy: From 249cc827f635bedd6fbfbe8f5530996b2de6bc09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zoe=20Faltib=C3=A0?= Date: Fri, 13 Nov 2026 18:24:04 +0100 Subject: [PATCH 2/5] add CI Add two GitHub Actions workflows: - Build: builds the wheel for each supported platform with the release build system, builds the sdist and checks the rust sources it packages still build, installs each wheel on its platform and calls into the bindings, then runs the demo notebook on regtest with the freshly built linux wheel - Lint: runs black, flake8 and pylint on the python scripts, plus shellcheck To support this: - the demo notebook funds its wallets and mines blocks on its own, through the new regtest.py helpers, so it can be executed unattended (and no longer requires switching to the console when run interactively) - services.sh gains a "test" command, that executes the notebook with nbconvert and fails if any cell raises, and waits for services with a timeout instead of looping forever - the demo image accepts a RGB_LIB_SPEC build arg, to install a locally built wheel instead of the released package - build.py can pass extra options to the cross image build (used to cache its layers in CI) and ignores GITHUB_REF_*, which the cross xtask would otherwise use to derive image tags - build_packages.py gains build_wheel(), to build a single platform --- .github/scripts/smoke_test.py | 18 +++ .github/workflows/build.yml | 240 ++++++++++++++++++++++++++++++++++ .github/workflows/lint.yml | 45 +++++++ .gitignore | 2 + CONTRIBUTING.md | 40 +++++- build.py | 19 ++- build_packages.py | 13 +- demo/.dockerignore | 2 + demo/Dockerfile | 11 +- demo/README.md | 43 +++++- demo/regtest.py | 79 +++++++++++ demo/rgb-lib.ipynb | 90 ++++++++++--- demo/services.sh | 73 ++++++++--- demo/wheels/.gitkeep | 0 14 files changed, 622 insertions(+), 53 deletions(-) create mode 100644 .github/scripts/smoke_test.py create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/lint.yml create mode 100644 demo/regtest.py create mode 100644 demo/wheels/.gitkeep diff --git a/.github/scripts/smoke_test.py b/.github/scripts/smoke_test.py new file mode 100644 index 0000000..7f66c9f --- /dev/null +++ b/.github/scripts/smoke_test.py @@ -0,0 +1,18 @@ +"""Script to check the installed rgb-lib package loads and works.""" + +import rgb_lib + + +def main(): + """Load the native library and generate a set of keys.""" + keys = rgb_lib.generate_keys( + rgb_lib.BitcoinNetwork.REGTEST, + rgb_lib.WitnessVersion.TAPROOT, + ) + if not keys.mnemonic: + raise RuntimeError("generate_keys returned no mnemonic") + print("native library loaded, keys generated") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..6aaa760 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,240 @@ +name: Build + +# SECURITY: the build job exposes the GitHub Actions runtime token (needed by +# docker buildx to access the cache), which allows writing to the cache. Never +# trigger this workflow on pull_request_target (or other events running +# untrusted code with access to the base repository's secrets and cache), as a +# pull request could then poison the cache used by master builds +on: + push: + branches: [ "master" ] + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + PYTHON_VERSION: "3.13" + +jobs: + # all platforms are built on linux, using cross + build: + name: Build ${{ matrix.platform }} wheel + runs-on: ubuntu-24.04 + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + platform: + - linux-x86_64 + - linux-aarch64 + - macosx-x86_64 + - macosx-arm64 + - win-amd64 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + submodules: recursive + + # the cross images and the rust build artifacts need a lot of space + - name: Free disk space + run: | + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc + df -h / + + - name: Setup python + uses: actions/setup-python@v7 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Setup rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry and cross xtask build + uses: Swatinem/rust-cache@v2 + with: + workspaces: cross + shared-key: cross-xtask + save-if: ${{ github.event_name != 'pull_request' }} + + - name: Install cross + uses: taiki-e/install-action@v2 + with: + tool: cross + + - name: Install poetry + run: pipx install poetry + + - name: Install build dependencies + run: poetry install --no-root --only=dev + + # the cross images are built with a dedicated builder, as the default + # one can't export layers to the GitHub Actions cache. It is not made the + # default as cross needs the docker daemon one for pre-build images + - name: Setup docker buildx + id: buildx + uses: docker/setup-buildx-action@v4 + with: + use: false + + # expose the runtime variables needed by buildx to access the cache. + # These include ACTIONS_RUNTIME_TOKEN, see the security note at the top + - name: Expose GitHub runtime + uses: crazy-max/ghaction-github-runtime@v4 + + - name: Setup cross + run: poetry run python -c 'import build_packages; build_packages.setup_cross()' + + - name: Build the wheel + env: + # pull requests don't write to the cache, to limit its size and to + # prevent cache poisoning, as their code is not trusted + CROSS_IMAGE_BUILD_OPTS: >- + --builder ${{ steps.buildx.outputs.name }} + --cache-from type=gha,scope=cross-image-${{ matrix.platform }} + ${{ github.event_name != 'pull_request' && format('--cache-to type=gha,scope=cross-image-{0},mode=max,ignore-error=true', matrix.platform) || '' }} + run: | + poetry run python -c \ + 'import build_packages; build_packages.build_wheel("${{ matrix.platform }}")' + + # the retagging leaves the originally tagged wheel in place, only the + # retagged one (the one that gets published) is uploaded + - name: Upload the wheel + uses: actions/upload-artifact@v7 + with: + name: wheel-${{ matrix.platform }} + path: dist/*-py3-none-*.whl + if-no-files-found: error + retention-days: 7 + + # check the sdist is complete, as it's what source installations build from + sdist: + name: Build the source distribution + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Setup python + uses: actions/setup-python@v7 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Setup rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install poetry + run: pipx install poetry + + # the sdist ships the patched cross files, so cross is set up first + - name: Setup cross + run: python3 -c 'import build_packages; build_packages.setup_cross()' + + - name: Build the sdist + run: poetry build --format sdist + + # a source installation builds the rust library from the sdist contents, + # so all the required sources need to have been packaged + - name: Check the packaged rust sources build + run: | + mkdir -p "$RUNNER_TEMP/sdist" + tar xzf dist/*.tar.gz -C "$RUNNER_TEMP/sdist" --strip-components=1 + cargo check --locked \ + --manifest-path "$RUNNER_TEMP/sdist/rgb-lib/bindings/uniffi/Cargo.toml" + + - name: Upload the sdist + uses: actions/upload-artifact@v7 + with: + name: sdist + path: dist/*.tar.gz + if-no-files-found: error + retention-days: 7 + + # install each wheel on its platform and check the bindings load. The + # linux-x86_64 wheel is not here, as it's covered by the demo run + smoke-test: + name: Smoke test ${{ matrix.platform }} + needs: build + # run for the platforms that built even if others failed + if: ${{ !cancelled() }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - { platform: linux-aarch64, runner: ubuntu-24.04-arm } + - { platform: macosx-x86_64, runner: macos-15-intel } + - { platform: macosx-arm64, runner: macos-15 } + - { platform: win-amd64, runner: windows-2025 } + defaults: + run: + shell: bash + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup python + uses: actions/setup-python@v7 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Download the wheel + uses: actions/download-artifact@v8 + with: + name: wheel-${{ matrix.platform }} + path: dist + + - name: Install the wheel + run: python -m pip install dist/*.whl + + - name: Load the bindings and call them + run: python .github/scripts/smoke_test.py + + # the regtest services need linux docker, so the demo only runs on + # linux-x86_64 + demo: + name: Run the demo on regtest (linux-x86_64) + needs: build + if: ${{ !cancelled() }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Download the wheel + uses: actions/download-artifact@v8 + with: + name: wheel-linux-x86_64 + path: demo/wheels + + # the demo image is built with the wheel instead of the released package, + # then the notebook is executed unattended against the regtest services + - name: Run the demo notebook + working-directory: demo + env: + CELL_TIMEOUT: 900 + run: | + wheel=$(ls wheels/*.whl) + echo "running the demo with $wheel" + RGB_LIB_SPEC="./$wheel" ./services.sh test + + - name: Upload the executed notebook + if: always() + uses: actions/upload-artifact@v7 + with: + name: executed-notebook + path: demo/rgb-lib-executed.ipynb + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..e807b7a --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,45 @@ +name: Lint + +on: + push: + branches: [ "master" ] + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup python + uses: actions/setup-python@v7 + with: + python-version: "3.13" + + - name: Install poetry + run: pipx install poetry + + - name: Install dev dependencies + run: poetry install --no-root --only=dev + + - name: Check formatting + run: poetry run black --check --diff *.py demo/*.py + + - name: Lint with flake8 + run: poetry run flake8 *.py demo/*.py + + - name: Lint with pylint + run: poetry run pylint *.py demo/*.py + + - name: Shellcheck + run: shellcheck demo/services.sh diff --git a/.gitignore b/.gitignore index 91800cd..7a7c9f2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ **/.vim/ **/__pycache__/ +/demo/rgb-lib-executed.ipynb +/demo/wheels/*.whl /dist/ /poetry.lock /rgb_lib/*rgblibuniffi.* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8dc380b..04e3c05 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,6 +72,14 @@ is complete, the generated files will be available in the `dist/` directory. Finally, the script will cleanup the build artifacts and restore the submodules to the committed revisions. +To build the wheel for a single platform (e.g. to test a change without +building all of them), run: + +```sh +poetry run python -c 'import build_packages; build_packages.setup_cross()' +poetry run python -c 'import build_packages; build_packages.build_wheel("linux-x86_64")' +``` + ### Wheel tags The `build.py` script contains the wheel tags associated to the supported @@ -105,7 +113,7 @@ the commits defined in the `build_packages.py` (`CROSS_COMMIT` and To format the code of the build scripts, from the project root run: ```sh -poetry run black *.py +poetry run black *.py demo/*.py ``` ## Lint @@ -113,10 +121,33 @@ poetry run black *.py To lint the code of the build scripts, from the project root run: ```sh -poetry run flake8 *.py -poetry run pylint *.py +poetry run flake8 *.py demo/*.py +poetry run pylint *.py demo/*.py ``` +## Continuous integration + +The `Build` workflow (`.github/workflows/build.yml`) checks the project still +builds and works, which is especially useful after an update of the `rgb-lib` +submodule. It: + +- builds the wheel for each supported platform, with the same build system used + for releases, and uploads them as artifacts +- builds the sdist and checks the packaged rust sources build, as source + installations are built from them (so a new file in rgb-lib that's not + matched by the `include` list in `pyproject.toml` fails the build) +- installs each wheel on its platform and checks the bindings load and work + (see `.github/scripts/smoke_test.py`) +- runs the [demo] notebook unattended, against regtest services and using the + freshly built linux wheel, so an API change that breaks the demo is caught + +Building the docker images used to cross-build the native library takes a long +time, so their layers are cached. Pull requests use the cache but don't write +to it, to avoid poisoning the one used by master builds. + +The `Lint` workflow (`.github/workflows/lint.yml`) runs the [format] and [lint] +checks for the python scripts, plus `shellcheck` for `demo/services.sh`. + ## Publish Publishing to PyPI is handled with [poetry] (version 2.0 or later). @@ -166,7 +197,10 @@ pip install --index-url https://test.pypi.org/simple/ rgb-lib [cargo]: https://github.com/rust-lang/cargo [cross]: https://github.com/cross-rs/cross +[demo]: demo/README.md [docker]: https://docs.docker.com/engine/install/ +[format]: #format +[lint]: #lint [packages]: #packages [platform compatibility tags]: https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/ [poetry]: https://github.com/python-poetry/poetry diff --git a/build.py b/build.py index 18c0263..c5b5da6 100644 --- a/build.py +++ b/build.py @@ -1,6 +1,6 @@ """Module to build the native library and the bindings.""" -from os import getenv +from os import environ, getenv from pathlib import Path from platform import machine from shutil import copy @@ -76,7 +76,7 @@ def build_docker_image(platform): f"{OSX_VERSION}/MacOSX{OSX_VERSION}.sdk.tar.xz" ) args.extend(["--build-arg", f"MACOS_SDK_URL={mac_sdk_url}"]) - run(args, cwd="cross", check=True) + run(args, cwd="cross", check=True, env=get_image_build_env()) def build_rust_library(platform): @@ -147,6 +147,21 @@ def get_current_platform(): return platform +def get_image_build_env(): + """Get the environment for the docker image build.""" + env = environ.copy() + # the cross xtask derives image tags from GITHUB_REF_*, failing on some + # refs (e.g. tags) even if --tag is passed + for var in ["GITHUB_REF_TYPE", "GITHUB_REF_NAME"]: + env.pop(var, None) + # extra docker build options (e.g. for layer caching in CI) are only + # applied to the image build, as cross also uses CROSS_BUILD_OPTS itself + image_build_opts = env.pop("CROSS_IMAGE_BUILD_OPTS", None) + if image_build_opts: + env["CROSS_BUILD_OPTS"] = image_build_opts + return env + + def get_lib_name(platform): """Get library name based on provided target.""" filename = PLATFORMS[platform]["filename"] diff --git a/build_packages.py b/build_packages.py index 9d0e38a..4bb2782 100644 --- a/build_packages.py +++ b/build_packages.py @@ -16,6 +16,14 @@ OUTPUT_DIR = Path("dist") +def build_wheel(platform): + """Build the wheel for the given platform.""" + print(f"\n\n\n==== bulding wheel for platform {platform}") + environ["PLATFORM"] = platform + run(["poetry", "build", "--format", "wheel"], check=True) + retag_wheel(platform) + + def cleanup(platform): """Cleanup the build artifacts for the given platform.""" print("Cleaning up build artifacts") @@ -54,10 +62,7 @@ def main(): setup_cross() # build platform-specific wheels for platform in PLATFORMS: - print(f"\n\n\n==== bulding wheel for platform {platform}") - environ["PLATFORM"] = platform - run(["poetry", "build", "--format", "wheel"], check=True) - retag_wheel(platform) + build_wheel(platform) cleanup(platform) # build the platform-independent sdist print("\n\n\n==== bulding source distribution") diff --git a/demo/.dockerignore b/demo/.dockerignore index 5502e26..d4578e6 100644 --- a/demo/.dockerignore +++ b/demo/.dockerignore @@ -1,4 +1,6 @@ * !rgb-lib.ipynb +!regtest.py !sample.png +!wheels diff --git a/demo/Dockerfile b/demo/Dockerfile index e1d81d8..a00dcf2 100644 --- a/demo/Dockerfile +++ b/demo/Dockerfile @@ -6,15 +6,22 @@ RUN apt-get update \ && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* ENV USER="jupyter" PIP_BREAK_SYSTEM_PACKAGES=1 +ENV PATH="/home/jupyter/.local/bin:$PATH" RUN adduser --disabled-login --gecos "$USER user" $USER WORKDIR /home/$USER USER $USER +# requirement specifier for the rgb-lib package, override to test a local build +# (e.g. ./wheels/rgb_lib--py3-none-manylinux_2_34_x86_64.whl) +ARG RGB_LIB_SPEC="rgb-lib==0.3.0b7" + +COPY --chown=$USER:$USER wheels/ ./wheels/ + RUN python3 -m pip install --no-cache-dir --no-warn-script-location \ - jupyterlab matplotlib python-magic qrcode rgb-lib==0.3.0b7 + jupyterlab matplotlib python-magic qrcode "$RGB_LIB_SPEC" -COPY --chown=$USER:$USER rgb-lib.ipynb sample.png ./ +COPY --chown=$USER:$USER rgb-lib.ipynb regtest.py sample.png ./ CMD ["/home/jupyter/.local/bin/jupyter-lab", "--ip=0.0.0.0"] diff --git a/demo/README.md b/demo/README.md index 8dfb23e..1efa03b 100644 --- a/demo/README.md +++ b/demo/README.md @@ -19,23 +19,24 @@ Start regtest blockchain services along with jupyter with: Get the link from the console output (`http://localhost:8888/...`) and open it in a web browser. -Once services are running, a regtest bitcoin address can be funded from the -console with: +Executing each code cell in the notebook from top to bottom reproduces an +example of wallet creation, asset issuance and asset transfer, with alternating +sender and receiver roles. The notebook funds its wallets and mines the +required blocks on its own, via the helpers in `regtest.py`. + +The same operations are also available from the console, to fund a regtest +bitcoin address: ```sh ./services.sh fund ``` -and a regtest block can be mined with: +or to mine a regtest block: ```sh ./services.sh mine ``` -Executing each code cell in the notebook from top to bottom reproduces an -example of wallet creation, asset issuance and asset transfer, with alternating -sender and receiver roles. - Once done with the example, close the jupyter browser page and stop all services (this will also delete all data produced by services and the demo) with: @@ -50,3 +51,31 @@ it can be deleted with: ```sh docker image rm rgb-lib-python-demo ``` + +## Unattended run + +The demo can also be executed without user interaction (this is what CI does to +check the bindings still work), with: + +```sh +./services.sh test +``` + +This builds the image, starts the services, then executes the notebook with +`nbconvert`, failing if any cell raises an error. The executed notebook is +saved as `rgb-lib-executed.ipynb` and the services are stopped on completion. + +By default the demo uses the released `rgb-lib` package. To test a locally +built wheel instead, copy it to the `wheels/` directory and set the +`RGB_LIB_SPEC` environment variable to its path inside the container: + +```sh +cp ../dist/rgb_lib--py3-none-manylinux_2_34_x86_64.whl wheels/ +RGB_LIB_SPEC=./wheels/rgb_lib--py3-none-manylinux_2_34_x86_64.whl \ + ./services.sh test +``` + +The same variable works with `./services.sh build`, to then run the demo +interactively against the local wheel. The maximum execution time allowed for a +single notebook cell (600 seconds by default) can be changed with the +`CELL_TIMEOUT` environment variable. diff --git a/demo/regtest.py b/demo/regtest.py new file mode 100644 index 0000000..49105ae --- /dev/null +++ b/demo/regtest.py @@ -0,0 +1,79 @@ +"""Module to operate the regtest services used by the demo. + +The notebook runs in a container alongside the services defined in +compose.yaml, so bitcoind is called via its RPC interface and electrs via the +electrum protocol. This allows the notebook to fund wallets and mine blocks on +its own, without the need to call services.sh from the console. +""" + +from base64 import b64encode +from json import dumps, load, loads +from socket import create_connection +from time import monotonic, sleep +from urllib.request import Request, urlopen + +BITCOIND_URL = "http://bitcoind:18443" +BITCOIND_USER = "user" +BITCOIND_PASSWORD = "default_password" +ELECTRS_ADDRESS = ("electrs", 50001) +MINER_WALLET = "miner" +RPC_TIMEOUT = 30 +SYNC_TIMEOUT = 120 + + +def _bitcoind_rpc(method, params=None, wallet=None): + """Call the given method on the bitcoind RPC interface.""" + url = BITCOIND_URL + (f"/wallet/{wallet}" if wallet else "") + credentials = f"{BITCOIND_USER}:{BITCOIND_PASSWORD}".encode() + request = Request( + url, + data=dumps({"id": "demo", "method": method, "params": params or []}).encode(), + headers={ + "Authorization": "Basic " + b64encode(credentials).decode(), + "Content-Type": "application/json", + }, + ) + with urlopen(request, timeout=RPC_TIMEOUT) as response: + result = load(response) + if result.get("error"): + raise RuntimeError(f'bitcoind RPC error: {result["error"]}') + return result["result"] + + +def _electrs_height(): + """Get the height of the chain tip indexed by electrs.""" + request = dumps({"id": 0, "method": "blockchain.headers.subscribe", "params": []}) + with create_connection(ELECTRS_ADDRESS, timeout=RPC_TIMEOUT) as sock: + sock.sendall(request.encode() + b"\n") + data = b"" + while not data.endswith(b"\n"): + chunk = sock.recv(4096) + if not chunk: + raise RuntimeError("electrs closed the connection") + data += chunk + return loads(data)["result"]["height"] + + +def _wait_electrs(height): + """Wait for electrs to have indexed the chain up to the given height.""" + deadline = monotonic() + SYNC_TIMEOUT + while _electrs_height() < height: + if monotonic() > deadline: + raise RuntimeError(f"electrs did not index block {height} in time") + sleep(1) + + +def fund(address, amount=1): + """Send the given amount of bitcoins to the given address and confirm it.""" + txid = _bitcoind_rpc("sendtoaddress", [address, amount], wallet=MINER_WALLET) + print(f"sent {amount} BTC to {address} (txid {txid})") + mine() + + +def mine(blocks=1): + """Mine the requested number of blocks, then wait for electrs to sync.""" + address = _bitcoind_rpc("getnewaddress", wallet=MINER_WALLET) + _bitcoind_rpc("generatetoaddress", [blocks, address]) + height = _bitcoind_rpc("getblockcount") + _wait_electrs(height) + print(f"mined {blocks} block(s), chain height is now {height}") diff --git a/demo/rgb-lib.ipynb b/demo/rgb-lib.ipynb index abf6d14..9dc2aa5 100644 --- a/demo/rgb-lib.ipynb +++ b/demo/rgb-lib.ipynb @@ -35,7 +35,9 @@ "import qrcode\n", "from IPython.display import Image, display\n", "\n", - "import rgb_lib" + "import rgb_lib\n", + "\n", + "import regtest" ] }, { @@ -153,11 +155,7 @@ "id": "e957d882-0243-419f-a304-e6f96d8a5f88", "metadata": {}, "source": [ - "### get a Bitcoin address to add some funds\n", - "fund the address with some regtest bitcoins\n", - "```shell\n", - "./services.sh fund
\n", - "```" + "### get a Bitcoin address to add some funds" ] }, { @@ -176,6 +174,26 @@ "print(recv_address)" ] }, + { + "cell_type": "markdown", + "id": "4e2b0701-8a0c-456f-b072-d2059f94f21a", + "metadata": {}, + "source": [ + "### fund the address with some regtest bitcoins" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3239a9d-4fcb-4d00-9ba7-e9e90cf609a7", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "regtest.fund(recv_address)" + ] + }, { "cell_type": "markdown", "id": "732d07ca-2073-4fac-a6da-ddcf111cd0f0", @@ -337,11 +355,7 @@ "id": "cf019f80-7532-4f69-b25b-8b7c68e7a7bd", "metadata": {}, "source": [ - "### get a Bitcoin address to add some funds\n", - "fund the address with some regtest bitcoins\n", - "```shell\n", - "./services.sh fund
\n", - "```" + "### get a Bitcoin address to add some funds" ] }, { @@ -360,6 +374,26 @@ "plot.imshow(send_image,cmap='gray')" ] }, + { + "cell_type": "markdown", + "id": "b16721b2-a7f7-44d5-842e-4a118785fb29", + "metadata": {}, + "source": [ + "### fund the address with some regtest bitcoins" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1caafad1-a6c6-4674-a04a-590e03511e1a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "regtest.fund(send_address)" + ] + }, { "cell_type": "markdown", "id": "5f9ed276-4220-4f1c-a811-64b4d25f4f75", @@ -561,10 +595,19 @@ "metadata": {}, "source": [ "### transaction confirmation\n", - "for the operation to progress to a final state, mine a block\n", - "```shell\n", - "./services.sh mine\n", - "```" + "mine a block, for the operation to progress to a final state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52a2066f-8ef5-4bb3-85b0-9c58d89d2666", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "regtest.mine()" ] }, { @@ -876,10 +919,19 @@ "source": [ "---\n", "### transaction confirmation\n", - "for the transfer to progress to a final state, mine a block\n", - "```shell\n", - "./services.sh mine\n", - "```" + "mine a block, for the transfer to progress to a final state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "161f5b91-dd97-40f3-9694-eb19bcd9fed3", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "regtest.mine()" ] }, { diff --git a/demo/services.sh b/demo/services.sh index 2aaabe1..5e146a6 100755 --- a/demo/services.sh +++ b/demo/services.sh @@ -13,8 +13,32 @@ if ! $COMPOSE >/dev/null; then fi BCLI="$COMPOSE exec -T -u blits bitcoind bitcoin-cli -regtest" +# notebook files (inside the jupyter container) +NOTEBOOK="rgb-lib.ipynb" +NOTEBOOK_OUT="rgb-lib-executed.ipynb" +# max execution time for a single notebook cell, in seconds +CELL_TIMEOUT="${CELL_TIMEOUT:-600}" +# max time to wait for a service to complete startup, in seconds +TIMEOUT=120 + +_wait_for() { + local service="$1" pattern="$2" start_time + start_time=$(date +%s) + until $COMPOSE logs "$service" |grep -q "$pattern"; do + if [ $(($(date +%s) - start_time)) -gt $TIMEOUT ]; then + echo "timeout waiting for $service to start, logs follow" + $COMPOSE logs "$service" + _die "could not start services" + fi + sleep 1 + done +} + build() { - $COMPOSE build jupyter + local args=() + # requirement specifier for the rgb-lib package (e.g. a local wheel) + [ -n "$RGB_LIB_SPEC" ] && args+=("--build-arg" "RGB_LIB_SPEC=$RGB_LIB_SPEC") + $COMPOSE build "${args[@]}" jupyter } start() { @@ -26,33 +50,24 @@ start() { _die "port $port is already bound, services can't be started" fi done - $COMPOSE up -d + $COMPOSE up -d || _die "could not start services" # wait for bitcoind to be up - until $COMPOSE logs bitcoind |grep -q 'Bound to'; do - sleep 1 - done + _wait_for bitcoind 'Bound to' # prepare bitcoin funds $BCLI createwallet miner >/dev/null mine 103 >/dev/null # wait for electrs to have completed startup - until $COMPOSE logs electrs |grep -q 'finished full compaction'; do - sleep 1 - done + _wait_for electrs 'finished full compaction' # wait for proxy to have completed startup - until $COMPOSE logs proxy |grep -q 'App is running at http://localhost:3000'; do - sleep 1 - done - + _wait_for proxy 'App is running at http://localhost:3000' # wait for jupyter to have completed startup local jupyter_str='http://127.0.0.1:8888/lab?token=' - until $COMPOSE logs jupyter |grep -q "$jupyter_str"; do - sleep 1 - done + _wait_for jupyter "$jupyter_str" local link link=$($COMPOSE logs jupyter \ |grep "${jupyter_str}" |tail -1 |awk '{print $NF}') @@ -60,7 +75,7 @@ start() { echo echo "open jupyter by pointing a browser to the following link:" # insert notebook path and show link in console - echo "${link//\?/\/tree\/rgb-lib.ipynb?}" + echo "${link//\?/\/tree\/${NOTEBOOK}?}" } stop() { @@ -80,9 +95,35 @@ mine() { $BCLI -rpcwallet=miner -generate "$blocks" } +_execute_notebook() { + $COMPOSE exec -T jupyter jupyter nbconvert --to notebook --execute \ + --ExecutePreprocessor.timeout="$CELL_TIMEOUT" \ + --output "$NOTEBOOK_OUT" "$NOTEBOOK" +} + +run_test() { + build || _die "demo image build failed" + start + echo + echo "executing the notebook (this will take a while)" + if ! _execute_notebook; then + echo + echo "notebook execution failed, service logs follow" + $COMPOSE logs + stop + _die "demo test failed" + fi + # save the executed notebook, so it can be inspected + $COMPOSE cp "jupyter:/home/jupyter/${NOTEBOOK_OUT}" "./${NOTEBOOK_OUT}" + stop + echo + echo "demo test completed successfully (see ./${NOTEBOOK_OUT})" +} + [ -n "$1" ] || _die "command required" case $1 in build|start|stop) "$1";; fund|mine) "$@";; + test) run_test;; *) _die "unrecognized command";; esac diff --git a/demo/wheels/.gitkeep b/demo/wheels/.gitkeep new file mode 100644 index 0000000..e69de29 From effe3ab6a6d4b8253fa114befbb922925f516359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zoe=20Faltib=C3=A0?= Date: Fri, 13 Nov 2026 18:57:22 +0100 Subject: [PATCH 3/5] fix cross build with recent rust Switch the cross submodule to the RGB-Tools fork, which patches xargo before installing it, as recent rust versions fail to build it. --- .gitmodules | 2 +- build_packages.py | 2 +- cross | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 94098b8..e5479c3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,4 +3,4 @@ url = https://github.com/RGB-Tools/rgb-lib.git [submodule "cross"] path = cross - url = https://github.com/nicbus/cross + url = https://github.com/RGB-Tools/cross diff --git a/build_packages.py b/build_packages.py index 4bb2782..7c2670b 100644 --- a/build_packages.py +++ b/build_packages.py @@ -11,7 +11,7 @@ from build import OSX_VERSION, PLATFORMS, PYTHON_LIB_DIR, UNIFFI_DIR, get_lib_name -CROSS_COMMIT = "43a1220" +CROSS_COMMIT = "2870a0c" CROSS_TOOLCHAIN_COMMIT = "d139724" OUTPUT_DIR = Path("dist") diff --git a/cross b/cross index 43a1220..2870a0c 160000 --- a/cross +++ b/cross @@ -1 +1 @@ -Subproject commit 43a12200e1b21653fafb942d14e2699ccb9e3c11 +Subproject commit 2870a0cef9308febf8b4b75a83542d9a3d319ffe From cffa89f247c7b700ed81777efd371d2ff788fba8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zoe=20Faltib=C3=A0?= Date: Fri, 13 Nov 2026 18:57:56 +0100 Subject: [PATCH 4/5] fix cross reset with shallow submodules The cross-toolchains commit the build script resets to is newer than the one pinned by the cross submodule, so it's missing when submodules are cloned shallow (e.g. by actions/checkout). Fetch the cross and cross-toolchains commits if missing, using full hashes as fetching by short hash is not supported. --- build_packages.py | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/build_packages.py b/build_packages.py index 7c2670b..ae9fc87 100644 --- a/build_packages.py +++ b/build_packages.py @@ -6,13 +6,14 @@ from pathlib import Path from re import sub from shutil import copy -from subprocess import run +from subprocess import DEVNULL, run from tomllib import load from build import OSX_VERSION, PLATFORMS, PYTHON_LIB_DIR, UNIFFI_DIR, get_lib_name -CROSS_COMMIT = "2870a0c" -CROSS_TOOLCHAIN_COMMIT = "d139724" +# full commit hashes, as fetching by short hash is not supported +CROSS_COMMIT = "2870a0cef9308febf8b4b75a83542d9a3d319ffe" +CROSS_TOOLCHAIN_COMMIT = "d139724ffa5d4bf5355e05eb9ada2b21cb2b9c44" OUTPUT_DIR = Path("dist") @@ -24,6 +25,33 @@ def build_wheel(platform): retag_wheel(platform) +def checkout_commit(commit, path): + """Checkout the given commit in the repository at the given path.""" + # the commit can be missing (e.g. with shallow submodules, as in CI), in + # that case fetch it, keeping the clone shallow only if it already is + missing = run( + ["git", "cat-file", "-e", f"{commit}^{{commit}}"], + cwd=path, + check=False, + stdout=DEVNULL, + stderr=DEVNULL, + ).returncode + if missing: + shallow = run( + ["git", "rev-parse", "--is-shallow-repository"], + cwd=path, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + args = ["git", "fetch"] + if shallow == "true": + args.extend(["--depth", "1"]) + args.extend(["origin", commit]) + run(args, cwd=path, check=True) + run(["git", "reset", "--hard", commit], cwd=path, check=True) + + def cleanup(platform): """Cleanup the build artifacts for the given platform.""" print("Cleaning up build artifacts") @@ -158,12 +186,8 @@ def retag_wheel(platform): def setup_cross(): """Setup cross for cross-building.""" reset_submodules() - run(["git", "reset", "--hard", CROSS_COMMIT], cwd="cross", check=True) - run( - ["git", "reset", "--hard", CROSS_TOOLCHAIN_COMMIT], - cwd="cross/docker/cross-toolchains", - check=True, - ) + checkout_commit(CROSS_COMMIT, "cross") + checkout_commit(CROSS_TOOLCHAIN_COMMIT, "cross/docker/cross-toolchains") patch_cross_files() configure_cross() From 2634c0f571907e48cba4571aa6048f537d74c6ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zoe=20Faltib=C3=A0?= Date: Fri, 13 Nov 2026 19:54:40 +0100 Subject: [PATCH 5/5] fix qemu download in the native cross image The native image (used for the host platform target) builds qemu from source, downloading util-linux and libcap from kernel.org, whose origin host is not reachable from the GitHub Actions runners, so the build fails after exhausting the curl retries. Patch the downloads to use the kernel.org CDN, which is reachable. --- build_packages.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build_packages.py b/build_packages.py index ae9fc87..eac7253 100644 --- a/build_packages.py +++ b/build_packages.py @@ -119,6 +119,13 @@ def patch_cross_files(): (r"archive_{2020,2021,2022,2023,2024}", r"archive_2025"), ] patch_files(files, substitutions) + # patch qemu.sh to download from the kernel.org CDN, as the main site is + # not reachable from some networks (e.g. the GitHub Actions runners) + files = ["cross/docker/qemu.sh"] + substitutions = [ + (r"https://(www\.)?kernel\.org/pub", r"https://cdn.kernel.org/pub"), + ] + patch_files(files, substitutions) # patch wine version for updated ubuntu files = ["cross/docker/wine.sh"] substitutions = [