diff --git a/.cargo/config.toml b/.cargo/config.toml index 755c9d30397..5ee0dda9f02 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,6 +3,7 @@ rustflags = ["--cfg", "tokio_unstable"] [alias] bump-versions = "run -p upgrade-version --" +ci = "run -p ci --" [target.x86_64-pc-windows-msvc] # Use a different linker. Otherwise, the build fails with some obscure linker error that diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 052126c865f..a576f9e1971 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,9 +25,11 @@ jobs: strategy: matrix: include: - - { runner: spacetimedb-runner, smoketest_args: --docker } - - { runner: windows-latest, smoketest_args: --no-build-cli } - runner: [ spacetimedb-runner, windows-latest ] + - runner: spacetimedb-runner + smoketest_args: '--docker -x clear_database replication' + - runner: windows-latest + smoketest_args: '--no-build-cli -x clear_database replication' + runner: [spacetimedb-runner, windows-latest] runs-on: ${{ matrix.runner }} steps: - name: Find Git ref @@ -76,8 +78,8 @@ jobs: - name: Install python deps run: python -m pip install psycopg2-binary xmltodict - name: Run smoketests - # Note: clear_database and replication only work in private - run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication + run: cargo ci smoketests -- ${{ matrix.smoketest_args }} + - name: Stop containers (Linux) if: always() && runner.os == 'Linux' run: docker compose down @@ -118,30 +120,13 @@ jobs: with: run_install: true - - name: Create /stdb dir - run: | - sudo mkdir /stdb - sudo chmod 777 /stdb - - name: Build typescript module sdk working-directory: crates/bindings-typescript run: pnpm build - name: Run cargo test #Note: Unreal tests will be run separately - run: cargo test --all -- --skip unreal - - - name: Check that the test outputs are up-to-date - run: bash tools/check-diff.sh - - - name: Ensure C# autogen bindings are up-to-date - run: | - cargo run -p spacetimedb-codegen --example regen-csharp-moduledef - bash tools/check-diff.sh crates/bindings-csharp - - - name: C# bindings tests - working-directory: crates/bindings-csharp - run: dotnet test -warnaserror + run: cargo ci test lints: name: Lints @@ -157,30 +142,8 @@ jobs: with: global-json-file: global.json - - name: Run cargo fmt - run: cargo fmt --all -- --check - - - name: Run cargo clippy - run: cargo clippy --all --tests --benches -- -D warnings - - - name: Run C# formatting check - working-directory: crates/bindings-csharp - run: | - dotnet tool restore - dotnet csharpier --check . - - - name: Run `cargo doc` for bindings crate - # `bindings` is the only crate we care strongly about documenting, - # since we link to its docs.rs from our website. - # We won't pass `--no-deps`, though, - # since we want everything reachable through it to also work. - # This includes `sats` and `lib`. - working-directory: crates/bindings - env: - # Make `cargo doc` exit with error on warnings, most notably broken links - RUSTDOCFLAGS: '--deny warnings' - run: | - cargo doc + - name: Run ci lint + run: cargo ci lint wasm_bindings: name: Build and test wasm bindings @@ -192,20 +155,7 @@ jobs: - run: echo ::add-matcher::.github/workflows/rust_matcher.json - name: Run bindgen tests - run: cargo test -p spacetimedb-codegen - - # Make sure the `Cargo.lock` file reflects the latest available versions. - # This is what users would end up with on a fresh module, so we want to - # catch any compile errors arising from a different transitive closure - # of dependencies than what is in the workspace lock file. - # - # For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714 - - name: Update dependencies - run: cargo update - - - name: Build module-test - run: cargo run -p spacetimedb-cli -- build --project-path modules/module-test - + run: cargo ci wasm-bindings publish_checks: name: Check that packages are publishable @@ -258,19 +208,9 @@ jobs: run: sudo apt install -y libssl-dev - name: Build spacetimedb-update - run: cargo build --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update - - - name: Run self-install env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - run: | - ROOT_DIR="$(mktemp -d)" - # NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`. - # My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it - # happens very frequently on the `macos-runner`, but we haven't seen it on any others). - cargo run --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update -- self-install --root-dir="${ROOT_DIR}" --yes - "${ROOT_DIR}"/spacetime --root-dir="${ROOT_DIR}" help + run: cargo ci update-flow --target=${{ matrix.target }} --github-token-auth unreal_engine_tests: name: Unreal Engine Tests @@ -292,26 +232,26 @@ jobs: # without this (reassigning env vars and stuff), but was unable to get it to work and it felt like an uphill battle. options: --user 0:0 steps: -# Uncomment this before merging so that it will run properly if run manually through the GH actions flow. It was playing weird with rolled back -# commits though. -# - name: Find Git ref -# env: -# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# shell: bash -# run: | -# PR_NUMBER="${{ github.event.inputs.pr_number || null }}" -# if test -n "${PR_NUMBER}"; then -# GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" -# else -# GIT_REF="${{ github.ref }}" -# fi -# echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" + # Uncomment this before merging so that it will run properly if run manually through the GH actions flow. It was playing weird with rolled back + # commits though. + # - name: Find Git ref + # env: + # GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # shell: bash + # run: | + # PR_NUMBER="${{ github.event.inputs.pr_number || null }}" + # if test -n "${PR_NUMBER}"; then + # GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" + # else + # GIT_REF="${{ github.ref }}" + # fi + # echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - name: Checkout sources uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} - uses: dsherret/rust-toolchain-file@v1 - - name: Run Unreal Engine tests + - name: Install unreal engine test dependencies working-directory: sdks/unreal env: UE_ROOT_PATH: /home/ue4/UnrealEngine @@ -320,43 +260,39 @@ jobs: apt-get update apt-get install -y acl curl ca-certificates - REPO="$GITHUB_WORKSPACE" - # Let ue4 read/write the workspace & tool caches without changing ownership - for p in "$REPO" "${RUNNER_TEMP:-/__t}" "${RUNNER_TOOL_CACHE:-/__t}"; do - [ -d "$p" ] && setfacl -R -m u:ue4:rwX -m d:u:ue4:rwX "$p" || true - done + - name: Run Unreal Engine tests + working-directory: sdks/unreal + env: + UE_ROOT_PATH: /home/ue4/UnrealEngine + run: | + cargo ci unreal-tests - # Rust tool caches live under the runner tool cache so they persist - export CARGO_HOME="${RUNNER_TOOL_CACHE:-/__t}/cargo" - export RUSTUP_HOME="${RUNNER_TOOL_CACHE:-/__t}/rustup" - mkdir -p "$CARGO_HOME" "$RUSTUP_HOME" - chown -R ue4:ue4 "$CARGO_HOME" "$RUSTUP_HOME" - - # Make sure the UE build script is executable (and parents traversable) - UE_DIR="${UE_ROOT_PATH:-/home/ue4/UnrealEngine}" - chmod a+rx "$UE_DIR" "$UE_DIR/Engine" "$UE_DIR/Engine/Build" "$UE_DIR/Engine/Build/BatchFiles/Linux" || true - chmod a+rx "$UE_DIR/Engine/Build/BatchFiles/Linux/Build.sh" || true - - # Run the build & tests as ue4 (who owns the UE tree) - sudo -E -H -u ue4 env \ - HOME=/home/ue4 \ - XDG_CONFIG_HOME=/home/ue4/.config \ - CARGO_HOME="$CARGO_HOME" \ - RUSTUP_HOME="$RUSTUP_HOME" \ - PATH="$CARGO_HOME/bin:$PATH" \ - bash -lc ' - set -euxo pipefail - # Install rustup for ue4 if needed (uses the shared caches) - if ! command -v cargo >/dev/null 2>&1; then - curl -sSf https://sh.rustup.rs | sh -s -- -y - fi - rustup show >/dev/null - git config --global --add safe.directory "$GITHUB_WORKSPACE" || true - - cd "$GITHUB_WORKSPACE/sdks/unreal" - cargo --version - cargo test - ' + ci_command_docs: + name: Check CI command docs + runs-on: ubuntu-latest + steps: + - name: Find Git ref + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + run: | + PR_NUMBER="${{ github.event.inputs.pr_number || null }}" + if test -n "${PR_NUMBER}"; then + GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" + else + GIT_REF="${{ github.ref }}" + fi + echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" + + - name: Checkout sources + uses: actions/checkout@v4 + with: + ref: ${{ env.GIT_REF }} + + - uses: dsherret/rust-toolchain-file@v1 + + - name: Check for docs change + run: cargo ci self-docs --check cli_docs: name: Check CLI docs @@ -407,12 +343,4 @@ jobs: - name: Check for docs change run: | - cargo run --features markdown-docs -p spacetimedb-cli > docs/docs/cli-reference.md - pnpm format - git status - if git diff --exit-code HEAD; then - echo "No docs changes detected" - else - echo "It looks like the CLI docs have changed:" - exit 1 - fi + cargo ci cli-docs diff --git a/Cargo.lock b/Cargo.lock index e6258738d43..99184cce126 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -186,6 +186,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ar_archive_writer" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c269894b6fe5e9d7ada0cf69b5bf847ff35bc25fc271f08e1d080fce80339a" +dependencies = [ + "object 0.32.2", +] + [[package]] name = "arbitrary" version = "1.4.2" @@ -249,7 +258,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -260,7 +269,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -505,7 +514,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -592,7 +601,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -680,7 +689,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -783,9 +792,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.41" +version = "1.2.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" +checksum = "739eb0f94557554b3ca9a86d2d37bebd49c5e6d0c1d2bda35ba5bdac830befc2" dependencies = [ "find-msvc-tools", "jobserver", @@ -854,6 +863,18 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "ci" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap 4.5.50", + "duct", + "log", + "regex", +] + [[package]] name = "ciborium" version = "0.2.2" @@ -973,7 +994,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -1549,7 +1570,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -1560,7 +1581,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -1614,9 +1635,9 @@ checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" [[package]] name = "deranged" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41953f86f8a05768a6cda24def994fd2f424b04ec5c719cf89989779f199071" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" dependencies = [ "powerfmt", "serde_core", @@ -1630,7 +1651,7 @@ checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -1641,7 +1662,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -1654,7 +1675,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -1674,7 +1695,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", "unicode-xid", ] @@ -1715,7 +1736,7 @@ dependencies = [ "diplomat_core", "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -1735,7 +1756,7 @@ dependencies = [ "serde", "smallvec", "strck", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -1754,16 +1775,7 @@ version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" dependencies = [ - "dirs-sys 0.4.1", -] - -[[package]] -name = "dirs" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" -dependencies = [ - "dirs-sys 0.5.0", + "dirs-sys", ] [[package]] @@ -1774,22 +1786,10 @@ checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" dependencies = [ "libc", "option-ext", - "redox_users 0.4.6", + "redox_users", "windows-sys 0.48.0", ] -[[package]] -name = "dirs-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" -dependencies = [ - "libc", - "option-ext", - "redox_users 0.5.2", - "windows-sys 0.61.2", -] - [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -1797,7 +1797,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", - "redox_users 0.4.6", + "redox_users", "winapi", ] @@ -1809,7 +1809,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -1902,6 +1902,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" +[[package]] +name = "endian-type" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "869b0adbda23651a9c5c0c3d270aac9fcb52e8622a8f2b17e57802d7791962f2" + [[package]] name = "enum-as-inner" version = "0.6.1" @@ -1911,7 +1917,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -1931,7 +1937,7 @@ checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -1944,7 +1950,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -1965,7 +1971,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -2142,9 +2148,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flate2" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" dependencies = [ "crc32fast", "miniz_oxide", @@ -2298,7 +2304,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -2591,11 +2597,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2819,7 +2825,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -3202,20 +3208,20 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.16" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "is_terminal_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itertools" @@ -3427,7 +3433,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -3656,9 +3662,9 @@ dependencies = [ [[package]] name = "memmap2" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843a98750cd611cc2965a8213b53b43e715f13c37a9e096c6408e69990961db7" +checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" dependencies = [ "libc", ] @@ -3761,7 +3767,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -4032,6 +4038,15 @@ dependencies = [ "objc2", ] +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + [[package]] name = "object" version = "0.36.7" @@ -4061,9 +4076,9 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "once_cell_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "oorandom" @@ -4094,7 +4109,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -4105,9 +4120,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-src" -version = "300.5.3+3.5.4" +version = "300.5.4+3.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6bad8cd0233b63971e232cc9c5e83039375b8586d2312f31fda85db8f888c2" +checksum = "a507b3792995dae9b0df8a1c1e3771e8418b7c2d9f0baeba32e6fe8b06c7cb72" dependencies = [ "cc", ] @@ -4237,7 +4252,7 @@ checksum = "05bbaa5b6b98826bb62b164406f703bee72c5287af9986f9c863fa8ea992b476" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -4281,7 +4296,7 @@ dependencies = [ "phf", "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -4870,7 +4885,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -4899,7 +4914,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -4963,17 +4978,16 @@ dependencies = [ [[package]] name = "pnp" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a10a726fb86dab6571b148c0f52cf619a4aabf0ac4fcf578bd4cd2178fb0e6d0" +checksum = "7adbc1ab7344e1e77be663e91cb129e989e398c319df7a9b8dbda9dd6758df38" dependencies = [ "byteorder", "concurrent_lru", - "dirs 6.0.0", "fancy-regex", "miniz_oxide", "pathdiff", - "radix_trie", + "radix_trie 0.3.0", "rustc-hash", "serde", "serde_json", @@ -5007,7 +5021,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -5101,7 +5115,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -5139,9 +5153,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.101" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" dependencies = [ "unicode-ident", ] @@ -5163,14 +5177,13 @@ dependencies = [ [[package]] name = "proptest" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb0be07becd10686a0bb407298fb425360a5c44a663774406340c59a22de4ce" +checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40" dependencies = [ "bit-set", "bit-vec", "bitflags 2.10.0", - "lazy_static", "num-traits", "rand 0.9.2", "rand_chacha 0.9.0", @@ -5189,7 +5202,7 @@ checksum = "4ee1c9ac207483d5e7db4940700de86a9aae46ef90c48b57f99fe7edb8345e49" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -5212,7 +5225,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -5223,10 +5236,11 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" [[package]] name = "psm" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e66fcd288453b748497d8fb18bccc83a16b0518e3906d4b8df0a8d42d93dbb1c" +checksum = "d11f2fedc3b7dafdc2851bc52f277377c5473d378859be234bc7ebb593144d01" dependencies = [ + "ar_archive_writer", "cc", ] @@ -5267,7 +5281,7 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -5361,7 +5375,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" dependencies = [ - "endian-type", + "endian-type 0.1.2", + "nibble_vec", +] + +[[package]] +name = "radix_trie" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b4431027dcd37fc2a73ef740b5f233aa805897935b8bce0195e41bbf9a3289a" +dependencies = [ + "endian-type 0.2.0", "nibble_vec", ] @@ -5528,17 +5552,6 @@ dependencies = [ "thiserror 1.0.69", ] -[[package]] -name = "redox_users" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" -dependencies = [ - "getrandom 0.2.16", - "libredox", - "thiserror 2.0.17", -] - [[package]] name = "ref-cast" version = "1.0.25" @@ -5556,7 +5569,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -5802,7 +5815,7 @@ checksum = "bd83f5f173ff41e00337d97f6572e416d022ef8a19f371817259ae960324c482" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -6248,9 +6261,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.33" +version = "0.23.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "751e04a496ca00bb97a5e043158d23d66b5aabf2e1d5aa2a0aaebb1aafe6f82c" +checksum = "6a9586e9ee2b4f8fab52a0048ca7334d7024eef48e2cb9407e3497bb7cab7fa7" dependencies = [ "once_cell", "rustls-pki-types", @@ -6270,9 +6283,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" dependencies = [ "zeroize", ] @@ -6321,7 +6334,7 @@ dependencies = [ "log", "memchr", "nix 0.26.4", - "radix_trie", + "radix_trie 0.2.1", "scopeguard", "unicode-segmentation", "unicode-width 0.1.14", @@ -6490,9 +6503,9 @@ dependencies = [ [[package]] name = "self_cell" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749" +checksum = "16c2f82143577edb4921b71ede051dac62ca3c16084e918bf7b40c96ae10eb33" [[package]] name = "semver" @@ -6537,7 +6550,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -6597,9 +6610,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.15.0" +version = "3.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6093cd8c01b25262b84927e0f7151692158fab02d961e04c979d3903eba7ecc5" +checksum = "aa66c845eee442168b2c8134fec70ac50dc20e760769c8ba0ad1319ca1959b04" dependencies = [ "base64 0.22.1", "chrono", @@ -6616,14 +6629,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.15.0" +version = "3.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7e6c180db0816026a61afa1cff5344fb7ebded7e4d3062772179f2501481c27" +checksum = "b91a903660542fced4e99881aa481bdbaec1634568ee02e0b8bd57c64cb38955" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -6648,7 +6661,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -6738,9 +6751,9 @@ dependencies = [ [[package]] name = "signal-hook-mio" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" dependencies = [ "libc", "mio 0.8.11", @@ -6968,7 +6981,7 @@ dependencies = [ "proc-macro2", "quote", "spacetimedb-primitives 1.6.0", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -6982,7 +6995,7 @@ dependencies = [ "proc-macro2", "quote", "spacetimedb-primitives 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -7014,7 +7027,7 @@ dependencies = [ "clap-markdown", "colored", "convert_case 0.6.0", - "dirs 5.0.1", + "dirs", "duct", "email_address", "flate2", @@ -7212,7 +7225,7 @@ dependencies = [ "crossbeam-channel", "crossbeam-queue", "derive_more 0.99.20", - "dirs 5.0.1", + "dirs", "enum-as-inner", "enum-map", "env_logger 0.10.2", @@ -7525,7 +7538,7 @@ version = "1.6.0" dependencies = [ "anyhow", "chrono", - "dirs 5.0.1", + "dirs", "fs2", "itoa", "junction", @@ -7782,7 +7795,7 @@ dependencies = [ "async-trait", "axum", "clap 4.5.50", - "dirs 5.0.1", + "dirs", "futures", "hostname", "http 1.3.1", @@ -8130,7 +8143,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -8151,9 +8164,12 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "sugar_path" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8230d5b8a65a6d4d4a7e5ee8dbdd9312ba447a8b8329689a390a0945d69b57ce" +checksum = "48abcb2199ce37819c20dc7a72dc09e3263a00e598ff5089fe5fda92e0f63c37" +dependencies = [ + "smallvec", +] [[package]] name = "syn" @@ -8168,9 +8184,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.107" +version = "2.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a26dbd934e5451d21ef060c018dae56fc073894c5a7896f882928a76e6d081b" +checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917" dependencies = [ "proc-macro2", "quote", @@ -8212,7 +8228,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -8327,9 +8343,9 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "target-triple" -version = "0.1.4" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac9aa371f599d22256307c24a9d748c041e548cbf599f35d890f9d365361790" +checksum = "591ef38edfb78ca4771ee32cf494cb8771944bee237a9b91fc9c1424ac4b777b" [[package]] name = "tempdir" @@ -8454,7 +8470,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -8465,7 +8481,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -8620,7 +8636,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -8916,7 +8932,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -9043,9 +9059,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "trybuild" -version = "1.0.112" +version = "1.0.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d66678374d835fe847e0dc8348fde2ceb5be4a7ec204437d8367f0d8df266a5" +checksum = "559b6a626c0815c942ac98d434746138b4f89ddd6a1b8cbb168c6845fb3376c5" dependencies = [ "glob", "serde", @@ -9074,7 +9090,7 @@ checksum = "ee6ff59666c9cbaec3533964505d39154dc4e0a56151fdea30a09ed0301f62e2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", "termcolor", ] @@ -9165,9 +9181,9 @@ checksum = "81b79ad29b5e19de4260020f8919b443b2ef0277d242ce532ec7b7a2cc8b6007" [[package]] name = "unicode-ident" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" +checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06" [[package]] name = "unicode-linebreak" @@ -9435,7 +9451,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", "wasm-bindgen-shared", ] @@ -9470,7 +9486,7 @@ checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -9527,7 +9543,7 @@ checksum = "3961bf864c790b5a06939f8f36d2a1a6be5bf0f926ddc25fb159b1766f2874db" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", "synstructure 0.13.2", "thiserror 1.0.69", ] @@ -9641,7 +9657,7 @@ dependencies = [ "anyhow", "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", "wasmtime-component-util", "wasmtime-wit-bindgen", "wit-parser", @@ -9758,7 +9774,7 @@ checksum = "6879a8e168aef3fe07335343b7fbede12fa494215e83322e173d4018e124a846" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -9974,7 +9990,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -9985,7 +10001,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -10518,7 +10534,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", "synstructure 0.13.2", ] @@ -10539,7 +10555,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -10559,7 +10575,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", "synstructure 0.13.2", ] @@ -10580,7 +10596,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] @@ -10613,7 +10629,7 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.107", + "syn 2.0.108", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 30f1ddde62f..fba9bd7b471 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ members = [ "sdks/rust/tests/test-client", "sdks/rust/tests/test-counter", "sdks/rust/tests/connect_disconnect_client", + "tools/ci", "tools/upgrade-version", "tools/license-check", "crates/bindings-typescript/test-app/server", diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml new file mode 100644 index 00000000000..e907526a42c --- /dev/null +++ b/tools/ci/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ci" +version = "0.1.0" +edition.workspace = true + +[dependencies] +log.workspace = true +anyhow.workspace = true +chrono = { workspace = true, features=["clock"] } +clap.workspace = true +regex.workspace = true +duct.workspace = true diff --git a/tools/ci/README.md b/tools/ci/README.md new file mode 100644 index 00000000000..0a4a83b1264 --- /dev/null +++ b/tools/ci/README.md @@ -0,0 +1,164 @@ +# SpacetimeDB's cargo ci + +## Overview + +This document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options. + +## `cargo ci` + +SpacetimeDB CI tasks + +This tool provides several subcommands for automating CI workflows in SpacetimeDB. + +It may be invoked via `cargo ci `, or simply `cargo ci` to run all subcommands in sequence. It is mostly designed to be run in CI environments via the github workflows, but can also be run locally + +**Usage:** +```bash +Usage: cargo ci [OPTIONS] [COMMAND] +``` + +**Options:** + +- `--skip`: Skip specified subcommands when running all + +When no subcommand is specified, all subcommands are run in sequence. This option allows specifying subcommands to skip when running all. For example, to skip the `unreal-tests` subcommand, use `--skip unreal-tests`. + +- `--help`: Print help (see a summary with '-h') + +### `test` + +Runs tests + +Runs rust tests, codegens csharp sdk and runs csharp tests. This does not include Unreal tests. This expects to run in a clean git state. + +**Usage:** +```bash +Usage: test +``` + +**Options:** + +- `--help`: Print help (see a summary with '-h') + +### `lint` + +Lints the codebase + +Runs rustfmt, clippy, csharpier and generates rust docs to ensure there are no warnings. + +**Usage:** +```bash +Usage: lint +``` + +**Options:** + +- `--help`: Print help (see a summary with '-h') + +### `wasm-bindings` + +Tests Wasm bindings + +Runs tests for the codegen crate and builds a test module with the wasm bindings. + +**Usage:** +```bash +Usage: wasm-bindings +``` + +**Options:** + +- `--help`: Print help (see a summary with '-h') + +### `smoketests` + +Runs smoketests + +Executes the smoketests suite with some default exclusions. + +**Usage:** +```bash +Usage: smoketests [ARGS]... +``` + +**Options:** + +- `args`: Additional arguments to pass to the smoketests runner. These are usually set by the CI environment, such as `-- --docker` +- `--help`: Print help (see a summary with '-h') + +### `update-flow` + +Tests the update flow + +Tests the self-update flow by building the spacetimedb-update binary for the specified target, by default the current target, and performing a self-install into a temporary directory. + +**Usage:** +```bash +Usage: update-flow [OPTIONS] +``` + +**Options:** + +- `--target`: Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms. +- `--github-token-auth`: Whether to enable github token authentication feature when building the update binary. By default this is disabled. +- `--help`: Print help (see a summary with '-h') + +### `unreal-tests` + +Run Unreal Engine related tests + +This assumes the UE4 environment is already set up + +This is designed to run in the github actions environment, but should work locally if the Unreal environment is set up correctly. + +**Usage:** +```bash +Usage: unreal-tests +``` + +**Options:** + +- `--help`: Print help (see a summary with '-h') + +### `cli-docs` + +**Usage:** +```bash +Usage: cli-docs +``` + +**Options:** + +- `--help`: + +### `self-docs` + +**Usage:** +```bash +Usage: self-docs [OPTIONS] +``` + +**Options:** + +- `--check`: Only check for changes, do not generate the docs +- `--help`: Print help (see a summary with '-h') + +### `help` + +**Usage:** +```bash +Usage: help [COMMAND]... +``` + +**Options:** + +- `subcommand`: + + +--- + +This document is auto-generated by running: + +```bash +cargo ci self-docs +``` \ No newline at end of file diff --git a/tools/ci/build.rs b/tools/ci/build.rs new file mode 100644 index 00000000000..b821cf9a10d --- /dev/null +++ b/tools/ci/build.rs @@ -0,0 +1,4 @@ +#[allow(clippy::disallowed_macros)] +fn main() { + println!("cargo:rustc-env=TARGET={}", std::env::var("TARGET").unwrap_or_default()); +} diff --git a/tools/ci/src/ci_docs.rs b/tools/ci/src/ci_docs.rs new file mode 100644 index 00000000000..f6beb2eb8ef --- /dev/null +++ b/tools/ci/src/ci_docs.rs @@ -0,0 +1,68 @@ +use clap::{Command, CommandFactory}; + +use crate::Cli; + +pub fn generate_cli_docs() -> String { + let mut cli = Cli::command(); + let usage = generate_markdown(&mut cli, 2); + + format!( + "\ +# SpacetimeDB's cargo ci + +## Overview + +This document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options. + +{} +--- + +This document is auto-generated by running: + +```bash +cargo ci self-docs +```", + usage + ) +} + +fn generate_markdown(cmd: &mut Command, heading_level: usize) -> String { + let mut out = String::new(); + + let heading = "#".repeat(heading_level); + out.push_str(&format!("{} `{}`\n\n", heading, cmd.get_name())); + + if let Some(long_about) = cmd.get_long_about() { + out.push_str(&format!("{}\n\n", long_about)); + } + + out.push_str(&format!("**Usage:**\n```bash\n{}\n```\n\n", cmd.render_usage())); + + let mut options = String::new(); + for arg in cmd.get_arguments() { + let names = arg + .get_long() + .map(|l| format!("--{}", l)) + .or_else(|| arg.get_short().map(|s| format!("-{}", s))) + .unwrap_or_else(|| arg.get_id().to_string()); + let help = arg.get_long_help().unwrap_or_default(); + options.push_str(&format!( + "- `{}`: {}\n{}", + names, + help, + if help.to_string().lines().count() > 1 { "\n" } else { "" } + )); + } + + if !options.is_empty() { + out.push_str("**Options:**\n\n"); + out.push_str(&options); + out.push('\n'); + } + + for sub in cmd.get_subcommands_mut() { + out.push_str(&generate_markdown(sub, heading_level + 1)); + } + + out +} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs new file mode 100644 index 00000000000..b939b94e5d8 --- /dev/null +++ b/tools/ci/src/main.rs @@ -0,0 +1,251 @@ +use anyhow::{bail, Result}; +use clap::{CommandFactory, Parser, Subcommand}; +use duct::cmd; +use std::collections::HashMap; +use std::path::Path; +use std::{env, fs}; + +const README_PATH: &str = "tools/ci/README.md"; + +mod ci_docs; + +/// SpacetimeDB CI tasks +/// +/// This tool provides several subcommands for automating CI workflows in SpacetimeDB. +/// +/// It may be invoked via `cargo ci `, or simply `cargo ci` to run all subcommands in +/// sequence. It is mostly designed to be run in CI environments via the github workflows, but can +/// also be run locally +#[derive(Parser)] +#[command(name = "cargo ci", subcommand_required = false, arg_required_else_help = false)] +struct Cli { + #[command(subcommand)] + cmd: Option, + + /// Skip specified subcommands when running all + /// + /// When no subcommand is specified, all subcommands are run in sequence. This option allows + /// specifying subcommands to skip when running all. For example, to skip the `unreal-tests` + /// subcommand, use `--skip unreal-tests`. + #[arg(long)] + skip: Vec, +} + +#[derive(Subcommand)] +enum CiCmd { + /// Runs tests + /// + /// Runs rust tests, codegens csharp sdk and runs csharp tests. + /// This does not include Unreal tests. + /// This expects to run in a clean git state. + Test, + /// Lints the codebase + /// + /// Runs rustfmt, clippy, csharpier and generates rust docs to ensure there are no warnings. + Lint, + /// Tests Wasm bindings + /// + /// Runs tests for the codegen crate and builds a test module with the wasm bindings. + WasmBindings, + /// Runs smoketests + /// + /// Executes the smoketests suite with some default exclusions. + Smoketests { + #[arg( + trailing_var_arg = true, + long_help = "Additional arguments to pass to the smoketests runner. These are usually set by the CI environment, such as `-- --docker`" + )] + args: Vec, + }, + /// Tests the update flow + /// + /// Tests the self-update flow by building the spacetimedb-update binary for the specified + /// target, by default the current target, and performing a self-install into a temporary + /// directory. + UpdateFlow { + #[arg( + long, + long_help = "Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms." + )] + target: Option, + #[arg( + long, + default_value = "false", + long_help = "Whether to enable github token authentication feature when building the update binary. By default this is disabled." + )] + github_token_auth: bool, + }, + /// Run Unreal Engine related tests + /// + /// This assumes the UE4 environment is already set up + /// + /// This is designed to run in the github actions environment, but should work locally if the + /// Unreal environment is set up correctly. + UnrealTests, + /// Generates CLI documentation and checks for changes + CliDocs, + SelfDocs { + #[arg( + long, + default_value_t = false, + long_help = "Only check for changes, do not generate the docs" + )] + check: bool, + }, +} + +macro_rules! run { + ($cmdline:expr) => { + run_command($cmdline, &Vec::new()) + }; + ($cmdline:expr, $envs:expr) => { + run_command($cmdline, $envs) + }; +} + +fn run_all_clap_subcommands(skips: &[String]) -> Result<()> { + let subcmds = Cli::command() + .get_subcommands() + .map(|sc| sc.get_name().to_string()) + .collect::>(); + + for subcmd in subcmds { + if skips.contains(&subcmd) { + log::info!("skipping {subcmd} as requested"); + continue; + } + log::info!("executing cargo ci {subcmd}"); + run!(&format!("cargo ci {subcmd}"))?; + } + + Ok(()) +} + +fn run_command(cmdline: &str, additional_env: &[(&str, &str)]) -> Result<()> { + let mut env = env::vars().collect::>(); + env.extend(additional_env.iter().map(|(k, v)| (k.to_string(), v.to_string()))); + log::debug!("$ {cmdline}"); + let status = cmd!("bash", "-lc", cmdline).full_env(env).run()?; + if !status.status.success() { + let e = anyhow::anyhow!("command failed: {cmdline}"); + log::error!("{e}"); + return Err(e); + } + Ok(()) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + + match cli.cmd { + Some(CiCmd::Test) => { + run!("cargo test --all -- --skip unreal")?; + run!("bash tools/check-diff.sh")?; + run!("cargo run -p spacetimedb-codegen --example regen-csharp-moduledef && bash tools/check-diff.sh crates/bindings-csharp")?; + run!("(cd crates/bindings-csharp && dotnet test -warnaserror)")?; + } + + Some(CiCmd::Lint) => { + run!("cargo fmt --all -- --check")?; + run!("cargo clippy --all --tests --benches -- -D warnings")?; + run!("(cd crates/bindings-csharp && dotnet tool restore && dotnet csharpier --check .)")?; + run!( + "cd crates/bindings && cargo doc", + &[("RUSTDOCFLAGS", "--deny warnings")] + )?; + } + + Some(CiCmd::WasmBindings) => { + run!("cargo test -p spacetimedb-codegen")?; + run!("cargo update")?; + run!("cargo run -p spacetimedb-cli -- build --project-path modules/module-test")?; + } + + Some(CiCmd::Smoketests { mut args }) => { + let default_args = ["-x", "clear_database", "replication"]; + if args.is_empty() { + args = default_args.iter().map(ToString::to_string).collect(); + } + // Note: clear_database and replication only work in private + run!(&format!("python -m smoketests {}", args.join(" ")))?; + } + + Some(CiCmd::UpdateFlow { + target, + github_token_auth, + }) => { + let target = target.map(|t| format!("--target {t}")).unwrap_or_default(); + let github_token_auth_flag = if github_token_auth { + "--features github-token-auth " + } else { + "" + }; + + run!(&format!("echo 'checking update flow for target: {target}'"))?; + run!(&format!( + "cargo build {github_token_auth_flag}{target} -p spacetimedb-update" + ))?; + // NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`. + // My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it + // happens very frequently on the `macos-runner`, but we haven't seen it on any others). + run!(&format!( + r#" +ROOT_DIR="$(mktemp -d)" +cargo run {github_token_auth_flag}--target {target} -p spacetimedb-update -- self-install --root-dir="${{ROOT_DIR}}" --yes +"${{ROOT_DIR}}"/spacetime --root-dir="${{ROOT_DIR}}" help + "# + ))?; + } + + Some(CiCmd::UnrealTests) => { + run!("for p in \"$GITHUB_WORKSPACE\" \"${RUNNER_TEMP:-/__t}\" \"${RUNNER_TOOL_CACHE:-/__t}\"; do [ -d \"$p\" ] && setfacl -R -m u:ue4:rwX -m d:u:ue4:rwX \"$p\" || true; done")?; + + run!("export CARGO_HOME=\"${RUNNER_TOOL_CACHE:-/__t}/cargo\"")?; + run!("export RUSTUP_HOME=\"${RUNNER_TOOL_CACHE:-/__t}/rustup\"")?; + run!("mkdir -p \"$CARGO_HOME\" \"$RUSTUP_HOME\"")?; + + run!("chmod a+rx \"$UE_ROOT_PATH\" \"$UE_ROOT_PATH/Engine\" \"$UE_ROOT_PATH/Engine/Build\" \"$UE_ROOT_PATH/Engine/Build/BatchFiles/Linux\" || true")?; + run!("chmod a+rx \"$UE_ROOT_PATH/Engine/Build/BatchFiles/Linux/Build.sh\" || true")?; + + run!("sudo -E -H -u ue4 env HOME=/home/ue4 CARGO_HOME=\"$CARGO_HOME\" RUSTUP_HOME=\"$RUSTUP_HOME\" PATH=\"$CARGO_HOME/bin:$PATH\" bash -lc 'set -euxo pipefail; if ! command -v cargo >/dev/null 2>&1; then curl -sSf https://sh.rustup.rs | sh -s -- -y; fi; rustup show >/dev/null; git config --global --add safe.directory \"$GITHUB_WORKSPACE\" || true; cd \"$GITHUB_WORKSPACE/sdks/unreal\"; cargo --version; cargo test'")?; + } + + Some(CiCmd::CliDocs) => { + run!("pnpm install --recursive")?; + run!("cargo run --features markdown-docs -p spacetimedb-cli > docs/docs/cli-reference.md")?; + run!("pnpm format")?; + run!("git status")?; + run!( + r#" +if git diff --exit-code HEAD; then + echo "No docs changes detected" +else + echo "It looks like the CLI docs have changed:" + exit 1 +fi + "# + )?; + } + + Some(CiCmd::SelfDocs { check }) => { + let readme_content = ci_docs::generate_cli_docs(); + let path = Path::new(README_PATH); + + if check { + let existing = fs::read_to_string(path).unwrap_or_default(); + if existing != readme_content { + bail!("README.md is out of date. Please run `cargo ci self-docs` to update it."); + } else { + log::info!("README.md is up to date."); + } + } else { + fs::write(path, readme_content)?; + log::info!("Wrote CLI docs to {}", path.display()); + } + } + + None => run_all_clap_subcommands(&cli.skip)?, + } + + Ok(()) +}