diff --git a/.github/scripts/smoke_test.py b/.github/scripts/smoke_test.py new file mode 100644 index 0000000..7f66c9f --- /dev/null +++ b/.github/scripts/smoke_test.py @@ -0,0 +1,18 @@ +"""Script to check the installed rgb-lib package loads and works.""" + +import rgb_lib + + +def main(): + """Load the native library and generate a set of keys.""" + keys = rgb_lib.generate_keys( + rgb_lib.BitcoinNetwork.REGTEST, + rgb_lib.WitnessVersion.TAPROOT, + ) + if not keys.mnemonic: + raise RuntimeError("generate_keys returned no mnemonic") + print("native library loaded, keys generated") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..6aaa760 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,240 @@ +name: Build + +# SECURITY: the build job exposes the GitHub Actions runtime token (needed by +# docker buildx to access the cache), which allows writing to the cache. Never +# trigger this workflow on pull_request_target (or other events running +# untrusted code with access to the base repository's secrets and cache), as a +# pull request could then poison the cache used by master builds +on: + push: + branches: [ "master" ] + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + PYTHON_VERSION: "3.13" + +jobs: + # all platforms are built on linux, using cross + build: + name: Build ${{ matrix.platform }} wheel + runs-on: ubuntu-24.04 + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + platform: + - linux-x86_64 + - linux-aarch64 + - macosx-x86_64 + - macosx-arm64 + - win-amd64 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + submodules: recursive + + # the cross images and the rust build artifacts need a lot of space + - name: Free disk space + run: | + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc + df -h / + + - name: Setup python + uses: actions/setup-python@v7 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Setup rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry and cross xtask build + uses: Swatinem/rust-cache@v2 + with: + workspaces: cross + shared-key: cross-xtask + save-if: ${{ github.event_name != 'pull_request' }} + + - name: Install cross + uses: taiki-e/install-action@v2 + with: + tool: cross + + - name: Install poetry + run: pipx install poetry + + - name: Install build dependencies + run: poetry install --no-root --only=dev + + # the cross images are built with a dedicated builder, as the default + # one can't export layers to the GitHub Actions cache. It is not made the + # default as cross needs the docker daemon one for pre-build images + - name: Setup docker buildx + id: buildx + uses: docker/setup-buildx-action@v4 + with: + use: false + + # expose the runtime variables needed by buildx to access the cache. + # These include ACTIONS_RUNTIME_TOKEN, see the security note at the top + - name: Expose GitHub runtime + uses: crazy-max/ghaction-github-runtime@v4 + + - name: Setup cross + run: poetry run python -c 'import build_packages; build_packages.setup_cross()' + + - name: Build the wheel + env: + # pull requests don't write to the cache, to limit its size and to + # prevent cache poisoning, as their code is not trusted + CROSS_IMAGE_BUILD_OPTS: >- + --builder ${{ steps.buildx.outputs.name }} + --cache-from type=gha,scope=cross-image-${{ matrix.platform }} + ${{ github.event_name != 'pull_request' && format('--cache-to type=gha,scope=cross-image-{0},mode=max,ignore-error=true', matrix.platform) || '' }} + run: | + poetry run python -c \ + 'import build_packages; build_packages.build_wheel("${{ matrix.platform }}")' + + # the retagging leaves the originally tagged wheel in place, only the + # retagged one (the one that gets published) is uploaded + - name: Upload the wheel + uses: actions/upload-artifact@v7 + with: + name: wheel-${{ matrix.platform }} + path: dist/*-py3-none-*.whl + if-no-files-found: error + retention-days: 7 + + # check the sdist is complete, as it's what source installations build from + sdist: + name: Build the source distribution + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Setup python + uses: actions/setup-python@v7 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Setup rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install poetry + run: pipx install poetry + + # the sdist ships the patched cross files, so cross is set up first + - name: Setup cross + run: python3 -c 'import build_packages; build_packages.setup_cross()' + + - name: Build the sdist + run: poetry build --format sdist + + # a source installation builds the rust library from the sdist contents, + # so all the required sources need to have been packaged + - name: Check the packaged rust sources build + run: | + mkdir -p "$RUNNER_TEMP/sdist" + tar xzf dist/*.tar.gz -C "$RUNNER_TEMP/sdist" --strip-components=1 + cargo check --locked \ + --manifest-path "$RUNNER_TEMP/sdist/rgb-lib/bindings/uniffi/Cargo.toml" + + - name: Upload the sdist + uses: actions/upload-artifact@v7 + with: + name: sdist + path: dist/*.tar.gz + if-no-files-found: error + retention-days: 7 + + # install each wheel on its platform and check the bindings load. The + # linux-x86_64 wheel is not here, as it's covered by the demo run + smoke-test: + name: Smoke test ${{ matrix.platform }} + needs: build + # run for the platforms that built even if others failed + if: ${{ !cancelled() }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - { platform: linux-aarch64, runner: ubuntu-24.04-arm } + - { platform: macosx-x86_64, runner: macos-15-intel } + - { platform: macosx-arm64, runner: macos-15 } + - { platform: win-amd64, runner: windows-2025 } + defaults: + run: + shell: bash + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup python + uses: actions/setup-python@v7 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Download the wheel + uses: actions/download-artifact@v8 + with: + name: wheel-${{ matrix.platform }} + path: dist + + - name: Install the wheel + run: python -m pip install dist/*.whl + + - name: Load the bindings and call them + run: python .github/scripts/smoke_test.py + + # the regtest services need linux docker, so the demo only runs on + # linux-x86_64 + demo: + name: Run the demo on regtest (linux-x86_64) + needs: build + if: ${{ !cancelled() }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Download the wheel + uses: actions/download-artifact@v8 + with: + name: wheel-linux-x86_64 + path: demo/wheels + + # the demo image is built with the wheel instead of the released package, + # then the notebook is executed unattended against the regtest services + - name: Run the demo notebook + working-directory: demo + env: + CELL_TIMEOUT: 900 + run: | + wheel=$(ls wheels/*.whl) + echo "running the demo with $wheel" + RGB_LIB_SPEC="./$wheel" ./services.sh test + + - name: Upload the executed notebook + if: always() + uses: actions/upload-artifact@v7 + with: + name: executed-notebook + path: demo/rgb-lib-executed.ipynb + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..e807b7a --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,45 @@ +name: Lint + +on: + push: + branches: [ "master" ] + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup python + uses: actions/setup-python@v7 + with: + python-version: "3.13" + + - name: Install poetry + run: pipx install poetry + + - name: Install dev dependencies + run: poetry install --no-root --only=dev + + - name: Check formatting + run: poetry run black --check --diff *.py demo/*.py + + - name: Lint with flake8 + run: poetry run flake8 *.py demo/*.py + + - name: Lint with pylint + run: poetry run pylint *.py demo/*.py + + - name: Shellcheck + run: shellcheck demo/services.sh diff --git a/.gitignore b/.gitignore index 91800cd..7a7c9f2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ **/.vim/ **/__pycache__/ +/demo/rgb-lib-executed.ipynb +/demo/wheels/*.whl /dist/ /poetry.lock /rgb_lib/*rgblibuniffi.* diff --git a/.gitmodules b/.gitmodules index 94098b8..e5479c3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,4 +3,4 @@ url = https://github.com/RGB-Tools/rgb-lib.git [submodule "cross"] path = cross - url = https://github.com/nicbus/cross + url = https://github.com/RGB-Tools/cross diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8dc380b..04e3c05 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,6 +72,14 @@ is complete, the generated files will be available in the `dist/` directory. Finally, the script will cleanup the build artifacts and restore the submodules to the committed revisions. +To build the wheel for a single platform (e.g. to test a change without +building all of them), run: + +```sh +poetry run python -c 'import build_packages; build_packages.setup_cross()' +poetry run python -c 'import build_packages; build_packages.build_wheel("linux-x86_64")' +``` + ### Wheel tags The `build.py` script contains the wheel tags associated to the supported @@ -105,7 +113,7 @@ the commits defined in the `build_packages.py` (`CROSS_COMMIT` and To format the code of the build scripts, from the project root run: ```sh -poetry run black *.py +poetry run black *.py demo/*.py ``` ## Lint @@ -113,10 +121,33 @@ poetry run black *.py To lint the code of the build scripts, from the project root run: ```sh -poetry run flake8 *.py -poetry run pylint *.py +poetry run flake8 *.py demo/*.py +poetry run pylint *.py demo/*.py ``` +## Continuous integration + +The `Build` workflow (`.github/workflows/build.yml`) checks the project still +builds and works, which is especially useful after an update of the `rgb-lib` +submodule. It: + +- builds the wheel for each supported platform, with the same build system used + for releases, and uploads them as artifacts +- builds the sdist and checks the packaged rust sources build, as source + installations are built from them (so a new file in rgb-lib that's not + matched by the `include` list in `pyproject.toml` fails the build) +- installs each wheel on its platform and checks the bindings load and work + (see `.github/scripts/smoke_test.py`) +- runs the [demo] notebook unattended, against regtest services and using the + freshly built linux wheel, so an API change that breaks the demo is caught + +Building the docker images used to cross-build the native library takes a long +time, so their layers are cached. Pull requests use the cache but don't write +to it, to avoid poisoning the one used by master builds. + +The `Lint` workflow (`.github/workflows/lint.yml`) runs the [format] and [lint] +checks for the python scripts, plus `shellcheck` for `demo/services.sh`. + ## Publish Publishing to PyPI is handled with [poetry] (version 2.0 or later). @@ -166,7 +197,10 @@ pip install --index-url https://test.pypi.org/simple/ rgb-lib [cargo]: https://github.com/rust-lang/cargo [cross]: https://github.com/cross-rs/cross +[demo]: demo/README.md [docker]: https://docs.docker.com/engine/install/ +[format]: #format +[lint]: #lint [packages]: #packages [platform compatibility tags]: https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/ [poetry]: https://github.com/python-poetry/poetry diff --git a/build.py b/build.py index 18c0263..c5b5da6 100644 --- a/build.py +++ b/build.py @@ -1,6 +1,6 @@ """Module to build the native library and the bindings.""" -from os import getenv +from os import environ, getenv from pathlib import Path from platform import machine from shutil import copy @@ -76,7 +76,7 @@ def build_docker_image(platform): f"{OSX_VERSION}/MacOSX{OSX_VERSION}.sdk.tar.xz" ) args.extend(["--build-arg", f"MACOS_SDK_URL={mac_sdk_url}"]) - run(args, cwd="cross", check=True) + run(args, cwd="cross", check=True, env=get_image_build_env()) def build_rust_library(platform): @@ -147,6 +147,21 @@ def get_current_platform(): return platform +def get_image_build_env(): + """Get the environment for the docker image build.""" + env = environ.copy() + # the cross xtask derives image tags from GITHUB_REF_*, failing on some + # refs (e.g. tags) even if --tag is passed + for var in ["GITHUB_REF_TYPE", "GITHUB_REF_NAME"]: + env.pop(var, None) + # extra docker build options (e.g. for layer caching in CI) are only + # applied to the image build, as cross also uses CROSS_BUILD_OPTS itself + image_build_opts = env.pop("CROSS_IMAGE_BUILD_OPTS", None) + if image_build_opts: + env["CROSS_BUILD_OPTS"] = image_build_opts + return env + + def get_lib_name(platform): """Get library name based on provided target.""" filename = PLATFORMS[platform]["filename"] diff --git a/build_packages.py b/build_packages.py index 9d0e38a..eac7253 100644 --- a/build_packages.py +++ b/build_packages.py @@ -6,16 +6,52 @@ from pathlib import Path from re import sub from shutil import copy -from subprocess import run +from subprocess import DEVNULL, run from tomllib import load from build import OSX_VERSION, PLATFORMS, PYTHON_LIB_DIR, UNIFFI_DIR, get_lib_name -CROSS_COMMIT = "43a1220" -CROSS_TOOLCHAIN_COMMIT = "d139724" +# full commit hashes, as fetching by short hash is not supported +CROSS_COMMIT = "2870a0cef9308febf8b4b75a83542d9a3d319ffe" +CROSS_TOOLCHAIN_COMMIT = "d139724ffa5d4bf5355e05eb9ada2b21cb2b9c44" OUTPUT_DIR = Path("dist") +def build_wheel(platform): + """Build the wheel for the given platform.""" + print(f"\n\n\n==== bulding wheel for platform {platform}") + environ["PLATFORM"] = platform + run(["poetry", "build", "--format", "wheel"], check=True) + retag_wheel(platform) + + +def checkout_commit(commit, path): + """Checkout the given commit in the repository at the given path.""" + # the commit can be missing (e.g. with shallow submodules, as in CI), in + # that case fetch it, keeping the clone shallow only if it already is + missing = run( + ["git", "cat-file", "-e", f"{commit}^{{commit}}"], + cwd=path, + check=False, + stdout=DEVNULL, + stderr=DEVNULL, + ).returncode + if missing: + shallow = run( + ["git", "rev-parse", "--is-shallow-repository"], + cwd=path, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + args = ["git", "fetch"] + if shallow == "true": + args.extend(["--depth", "1"]) + args.extend(["origin", commit]) + run(args, cwd=path, check=True) + run(["git", "reset", "--hard", commit], cwd=path, check=True) + + def cleanup(platform): """Cleanup the build artifacts for the given platform.""" print("Cleaning up build artifacts") @@ -54,10 +90,7 @@ def main(): setup_cross() # build platform-specific wheels for platform in PLATFORMS: - print(f"\n\n\n==== bulding wheel for platform {platform}") - environ["PLATFORM"] = platform - run(["poetry", "build", "--format", "wheel"], check=True) - retag_wheel(platform) + build_wheel(platform) cleanup(platform) # build the platform-independent sdist print("\n\n\n==== bulding source distribution") @@ -86,6 +119,13 @@ def patch_cross_files(): (r"archive_{2020,2021,2022,2023,2024}", r"archive_2025"), ] patch_files(files, substitutions) + # patch qemu.sh to download from the kernel.org CDN, as the main site is + # not reachable from some networks (e.g. the GitHub Actions runners) + files = ["cross/docker/qemu.sh"] + substitutions = [ + (r"https://(www\.)?kernel\.org/pub", r"https://cdn.kernel.org/pub"), + ] + patch_files(files, substitutions) # patch wine version for updated ubuntu files = ["cross/docker/wine.sh"] substitutions = [ @@ -153,12 +193,8 @@ def retag_wheel(platform): def setup_cross(): """Setup cross for cross-building.""" reset_submodules() - run(["git", "reset", "--hard", CROSS_COMMIT], cwd="cross", check=True) - run( - ["git", "reset", "--hard", CROSS_TOOLCHAIN_COMMIT], - cwd="cross/docker/cross-toolchains", - check=True, - ) + checkout_commit(CROSS_COMMIT, "cross") + checkout_commit(CROSS_TOOLCHAIN_COMMIT, "cross/docker/cross-toolchains") patch_cross_files() configure_cross() diff --git a/cross b/cross index 43a1220..2870a0c 160000 --- a/cross +++ b/cross @@ -1 +1 @@ -Subproject commit 43a12200e1b21653fafb942d14e2699ccb9e3c11 +Subproject commit 2870a0cef9308febf8b4b75a83542d9a3d319ffe diff --git a/demo/.dockerignore b/demo/.dockerignore index 5502e26..d4578e6 100644 --- a/demo/.dockerignore +++ b/demo/.dockerignore @@ -1,4 +1,6 @@ * !rgb-lib.ipynb +!regtest.py !sample.png +!wheels diff --git a/demo/Dockerfile b/demo/Dockerfile index e1d81d8..a00dcf2 100644 --- a/demo/Dockerfile +++ b/demo/Dockerfile @@ -6,15 +6,22 @@ RUN apt-get update \ && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* ENV USER="jupyter" PIP_BREAK_SYSTEM_PACKAGES=1 +ENV PATH="/home/jupyter/.local/bin:$PATH" RUN adduser --disabled-login --gecos "$USER user" $USER WORKDIR /home/$USER USER $USER +# requirement specifier for the rgb-lib package, override to test a local build +# (e.g. ./wheels/rgb_lib--py3-none-manylinux_2_34_x86_64.whl) +ARG RGB_LIB_SPEC="rgb-lib==0.3.0b7" + +COPY --chown=$USER:$USER wheels/ ./wheels/ + RUN python3 -m pip install --no-cache-dir --no-warn-script-location \ - jupyterlab matplotlib python-magic qrcode rgb-lib==0.3.0b7 + jupyterlab matplotlib python-magic qrcode "$RGB_LIB_SPEC" -COPY --chown=$USER:$USER rgb-lib.ipynb sample.png ./ +COPY --chown=$USER:$USER rgb-lib.ipynb regtest.py sample.png ./ CMD ["/home/jupyter/.local/bin/jupyter-lab", "--ip=0.0.0.0"] diff --git a/demo/README.md b/demo/README.md index 8dfb23e..1efa03b 100644 --- a/demo/README.md +++ b/demo/README.md @@ -19,23 +19,24 @@ Start regtest blockchain services along with jupyter with: Get the link from the console output (`http://localhost:8888/...`) and open it in a web browser. -Once services are running, a regtest bitcoin address can be funded from the -console with: +Executing each code cell in the notebook from top to bottom reproduces an +example of wallet creation, asset issuance and asset transfer, with alternating +sender and receiver roles. The notebook funds its wallets and mines the +required blocks on its own, via the helpers in `regtest.py`. + +The same operations are also available from the console, to fund a regtest +bitcoin address: ```sh ./services.sh fund ``` -and a regtest block can be mined with: +or to mine a regtest block: ```sh ./services.sh mine ``` -Executing each code cell in the notebook from top to bottom reproduces an -example of wallet creation, asset issuance and asset transfer, with alternating -sender and receiver roles. - Once done with the example, close the jupyter browser page and stop all services (this will also delete all data produced by services and the demo) with: @@ -50,3 +51,31 @@ it can be deleted with: ```sh docker image rm rgb-lib-python-demo ``` + +## Unattended run + +The demo can also be executed without user interaction (this is what CI does to +check the bindings still work), with: + +```sh +./services.sh test +``` + +This builds the image, starts the services, then executes the notebook with +`nbconvert`, failing if any cell raises an error. The executed notebook is +saved as `rgb-lib-executed.ipynb` and the services are stopped on completion. + +By default the demo uses the released `rgb-lib` package. To test a locally +built wheel instead, copy it to the `wheels/` directory and set the +`RGB_LIB_SPEC` environment variable to its path inside the container: + +```sh +cp ../dist/rgb_lib--py3-none-manylinux_2_34_x86_64.whl wheels/ +RGB_LIB_SPEC=./wheels/rgb_lib--py3-none-manylinux_2_34_x86_64.whl \ + ./services.sh test +``` + +The same variable works with `./services.sh build`, to then run the demo +interactively against the local wheel. The maximum execution time allowed for a +single notebook cell (600 seconds by default) can be changed with the +`CELL_TIMEOUT` environment variable. diff --git a/demo/compose.yaml b/demo/compose.yaml index 04f20ad..04bf53a 100644 --- a/demo/compose.yaml +++ b/demo/compose.yaml @@ -1,9 +1,9 @@ services: bitcoind: - image: registry.gitlab.com/hashbeam/docker/bitcoind:30.0 + image: registry.gitlab.com/hashbeam/docker/bitcoind:31.2 command: "-fallbackfee=0.0002" electrs: - image: registry.gitlab.com/hashbeam/docker/electrs:0.10.10 + image: registry.gitlab.com/hashbeam/docker/electrs:0.11.0 depends_on: - bitcoind proxy: diff --git a/demo/regtest.py b/demo/regtest.py new file mode 100644 index 0000000..49105ae --- /dev/null +++ b/demo/regtest.py @@ -0,0 +1,79 @@ +"""Module to operate the regtest services used by the demo. + +The notebook runs in a container alongside the services defined in +compose.yaml, so bitcoind is called via its RPC interface and electrs via the +electrum protocol. This allows the notebook to fund wallets and mine blocks on +its own, without the need to call services.sh from the console. +""" + +from base64 import b64encode +from json import dumps, load, loads +from socket import create_connection +from time import monotonic, sleep +from urllib.request import Request, urlopen + +BITCOIND_URL = "http://bitcoind:18443" +BITCOIND_USER = "user" +BITCOIND_PASSWORD = "default_password" +ELECTRS_ADDRESS = ("electrs", 50001) +MINER_WALLET = "miner" +RPC_TIMEOUT = 30 +SYNC_TIMEOUT = 120 + + +def _bitcoind_rpc(method, params=None, wallet=None): + """Call the given method on the bitcoind RPC interface.""" + url = BITCOIND_URL + (f"/wallet/{wallet}" if wallet else "") + credentials = f"{BITCOIND_USER}:{BITCOIND_PASSWORD}".encode() + request = Request( + url, + data=dumps({"id": "demo", "method": method, "params": params or []}).encode(), + headers={ + "Authorization": "Basic " + b64encode(credentials).decode(), + "Content-Type": "application/json", + }, + ) + with urlopen(request, timeout=RPC_TIMEOUT) as response: + result = load(response) + if result.get("error"): + raise RuntimeError(f'bitcoind RPC error: {result["error"]}') + return result["result"] + + +def _electrs_height(): + """Get the height of the chain tip indexed by electrs.""" + request = dumps({"id": 0, "method": "blockchain.headers.subscribe", "params": []}) + with create_connection(ELECTRS_ADDRESS, timeout=RPC_TIMEOUT) as sock: + sock.sendall(request.encode() + b"\n") + data = b"" + while not data.endswith(b"\n"): + chunk = sock.recv(4096) + if not chunk: + raise RuntimeError("electrs closed the connection") + data += chunk + return loads(data)["result"]["height"] + + +def _wait_electrs(height): + """Wait for electrs to have indexed the chain up to the given height.""" + deadline = monotonic() + SYNC_TIMEOUT + while _electrs_height() < height: + if monotonic() > deadline: + raise RuntimeError(f"electrs did not index block {height} in time") + sleep(1) + + +def fund(address, amount=1): + """Send the given amount of bitcoins to the given address and confirm it.""" + txid = _bitcoind_rpc("sendtoaddress", [address, amount], wallet=MINER_WALLET) + print(f"sent {amount} BTC to {address} (txid {txid})") + mine() + + +def mine(blocks=1): + """Mine the requested number of blocks, then wait for electrs to sync.""" + address = _bitcoind_rpc("getnewaddress", wallet=MINER_WALLET) + _bitcoind_rpc("generatetoaddress", [blocks, address]) + height = _bitcoind_rpc("getblockcount") + _wait_electrs(height) + print(f"mined {blocks} block(s), chain height is now {height}") diff --git a/demo/rgb-lib.ipynb b/demo/rgb-lib.ipynb index abf6d14..9dc2aa5 100644 --- a/demo/rgb-lib.ipynb +++ b/demo/rgb-lib.ipynb @@ -35,7 +35,9 @@ "import qrcode\n", "from IPython.display import Image, display\n", "\n", - "import rgb_lib" + "import rgb_lib\n", + "\n", + "import regtest" ] }, { @@ -153,11 +155,7 @@ "id": "e957d882-0243-419f-a304-e6f96d8a5f88", "metadata": {}, "source": [ - "### get a Bitcoin address to add some funds\n", - "fund the address with some regtest bitcoins\n", - "```shell\n", - "./services.sh fund
\n", - "```" + "### get a Bitcoin address to add some funds" ] }, { @@ -176,6 +174,26 @@ "print(recv_address)" ] }, + { + "cell_type": "markdown", + "id": "4e2b0701-8a0c-456f-b072-d2059f94f21a", + "metadata": {}, + "source": [ + "### fund the address with some regtest bitcoins" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3239a9d-4fcb-4d00-9ba7-e9e90cf609a7", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "regtest.fund(recv_address)" + ] + }, { "cell_type": "markdown", "id": "732d07ca-2073-4fac-a6da-ddcf111cd0f0", @@ -337,11 +355,7 @@ "id": "cf019f80-7532-4f69-b25b-8b7c68e7a7bd", "metadata": {}, "source": [ - "### get a Bitcoin address to add some funds\n", - "fund the address with some regtest bitcoins\n", - "```shell\n", - "./services.sh fund
\n", - "```" + "### get a Bitcoin address to add some funds" ] }, { @@ -360,6 +374,26 @@ "plot.imshow(send_image,cmap='gray')" ] }, + { + "cell_type": "markdown", + "id": "b16721b2-a7f7-44d5-842e-4a118785fb29", + "metadata": {}, + "source": [ + "### fund the address with some regtest bitcoins" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1caafad1-a6c6-4674-a04a-590e03511e1a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "regtest.fund(send_address)" + ] + }, { "cell_type": "markdown", "id": "5f9ed276-4220-4f1c-a811-64b4d25f4f75", @@ -561,10 +595,19 @@ "metadata": {}, "source": [ "### transaction confirmation\n", - "for the operation to progress to a final state, mine a block\n", - "```shell\n", - "./services.sh mine\n", - "```" + "mine a block, for the operation to progress to a final state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52a2066f-8ef5-4bb3-85b0-9c58d89d2666", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "regtest.mine()" ] }, { @@ -876,10 +919,19 @@ "source": [ "---\n", "### transaction confirmation\n", - "for the transfer to progress to a final state, mine a block\n", - "```shell\n", - "./services.sh mine\n", - "```" + "mine a block, for the transfer to progress to a final state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "161f5b91-dd97-40f3-9694-eb19bcd9fed3", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "regtest.mine()" ] }, { diff --git a/demo/services.sh b/demo/services.sh index 2aaabe1..5e146a6 100755 --- a/demo/services.sh +++ b/demo/services.sh @@ -13,8 +13,32 @@ if ! $COMPOSE >/dev/null; then fi BCLI="$COMPOSE exec -T -u blits bitcoind bitcoin-cli -regtest" +# notebook files (inside the jupyter container) +NOTEBOOK="rgb-lib.ipynb" +NOTEBOOK_OUT="rgb-lib-executed.ipynb" +# max execution time for a single notebook cell, in seconds +CELL_TIMEOUT="${CELL_TIMEOUT:-600}" +# max time to wait for a service to complete startup, in seconds +TIMEOUT=120 + +_wait_for() { + local service="$1" pattern="$2" start_time + start_time=$(date +%s) + until $COMPOSE logs "$service" |grep -q "$pattern"; do + if [ $(($(date +%s) - start_time)) -gt $TIMEOUT ]; then + echo "timeout waiting for $service to start, logs follow" + $COMPOSE logs "$service" + _die "could not start services" + fi + sleep 1 + done +} + build() { - $COMPOSE build jupyter + local args=() + # requirement specifier for the rgb-lib package (e.g. a local wheel) + [ -n "$RGB_LIB_SPEC" ] && args+=("--build-arg" "RGB_LIB_SPEC=$RGB_LIB_SPEC") + $COMPOSE build "${args[@]}" jupyter } start() { @@ -26,33 +50,24 @@ start() { _die "port $port is already bound, services can't be started" fi done - $COMPOSE up -d + $COMPOSE up -d || _die "could not start services" # wait for bitcoind to be up - until $COMPOSE logs bitcoind |grep -q 'Bound to'; do - sleep 1 - done + _wait_for bitcoind 'Bound to' # prepare bitcoin funds $BCLI createwallet miner >/dev/null mine 103 >/dev/null # wait for electrs to have completed startup - until $COMPOSE logs electrs |grep -q 'finished full compaction'; do - sleep 1 - done + _wait_for electrs 'finished full compaction' # wait for proxy to have completed startup - until $COMPOSE logs proxy |grep -q 'App is running at http://localhost:3000'; do - sleep 1 - done - + _wait_for proxy 'App is running at http://localhost:3000' # wait for jupyter to have completed startup local jupyter_str='http://127.0.0.1:8888/lab?token=' - until $COMPOSE logs jupyter |grep -q "$jupyter_str"; do - sleep 1 - done + _wait_for jupyter "$jupyter_str" local link link=$($COMPOSE logs jupyter \ |grep "${jupyter_str}" |tail -1 |awk '{print $NF}') @@ -60,7 +75,7 @@ start() { echo echo "open jupyter by pointing a browser to the following link:" # insert notebook path and show link in console - echo "${link//\?/\/tree\/rgb-lib.ipynb?}" + echo "${link//\?/\/tree\/${NOTEBOOK}?}" } stop() { @@ -80,9 +95,35 @@ mine() { $BCLI -rpcwallet=miner -generate "$blocks" } +_execute_notebook() { + $COMPOSE exec -T jupyter jupyter nbconvert --to notebook --execute \ + --ExecutePreprocessor.timeout="$CELL_TIMEOUT" \ + --output "$NOTEBOOK_OUT" "$NOTEBOOK" +} + +run_test() { + build || _die "demo image build failed" + start + echo + echo "executing the notebook (this will take a while)" + if ! _execute_notebook; then + echo + echo "notebook execution failed, service logs follow" + $COMPOSE logs + stop + _die "demo test failed" + fi + # save the executed notebook, so it can be inspected + $COMPOSE cp "jupyter:/home/jupyter/${NOTEBOOK_OUT}" "./${NOTEBOOK_OUT}" + stop + echo + echo "demo test completed successfully (see ./${NOTEBOOK_OUT})" +} + [ -n "$1" ] || _die "command required" case $1 in build|start|stop) "$1";; fund|mine) "$@";; + test) run_test;; *) _die "unrecognized command";; esac diff --git a/demo/wheels/.gitkeep b/demo/wheels/.gitkeep new file mode 100644 index 0000000..e69de29