From 77d77d7ce1ce524ee1444bdeda53c19e5e4e8645 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Tue, 6 Oct 2026 09:06:28 +0800 Subject: [PATCH 1/2] feat(music-data): take the deck model's WASM release packages as engines --replay-engine and --recommend-engine now take the ournotes-deck WASM release packages of the pinned model, as the downloaded ournotes--wasm-vVERSION.tar.gz or its directory. The package's build-info.json must describe that module's WASM package, name the pinned commit and list the SHA-256 of the web binding's JS and WASM files; the binding and build-info.json are copied byte for byte, so the published engine is the released one. The deck pin workflow follows ournotes-deck's releases: it moves the pin to the latest release once its replay and recommendation WASM packages are published. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deck.yml | 37 +++++++---- docs/music-data.md | 9 ++- docs/replay.md | 14 ++-- rust/Cargo.toml | 4 +- src/nnnotes/cli.py | 10 +-- src/nnnotes/replaydata.py | 79 +++++++++++++--------- tests/test_replaydata.py | 132 ++++++++++++++++++++----------------- 7 files changed, 163 insertions(+), 122 deletions(-) diff --git a/.github/workflows/deck.yml b/.github/workflows/deck.yml index 8b8a57c..c0a7a97 100644 --- a/.github/workflows/deck.yml +++ b/.github/workflows/deck.yml @@ -1,9 +1,11 @@ name: Deck model # nnnotes builds the deck model, the crate ournotes-sim of the ournotes-deck repository, at the commit rust/Cargo.toml -# pins, so that one nnnotes version always carries one deck model. When ournotes-deck's main moves past that commit, -# this opens a pull request that moves the pin and dispatches CI and the commit message check on its branch (a pull -# request opened with the workflow token starts no workflow by itself); their checks land on the pull request. +# pins, so that one nnnotes version always carries one deck model. The pin follows ournotes-deck's releases: when the +# latest release has published its replay and recommendation WASM packages (the engines of music-data --replay-engine +# and --recommend-engine) and its commit is past the pinned one, this opens a pull request that moves the pin and +# dispatches CI and the commit message check on its branch (a pull request opened with the workflow token starts no +# workflow by itself); their checks land on the pull request. on: schedule: - cron: "17 3 * * *" @@ -19,21 +21,29 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - name: Compare the pinned commit with ournotes-deck's main + - name: Compare the pinned commit with ournotes-deck's latest release id: deck env: GH_TOKEN: ${{ github.token }} run: | pinned=$(sed -nE 's/^ournotes-sim = .*rev = "([0-9a-f]{40})".*/\1/p' rust/Cargo.toml) [ -n "$pinned" ] || { echo "rust/Cargo.toml pins no ournotes-sim commit" >&2; exit 1; } - head=$(git ls-remote https://github.com/empty-sekai/ournotes-deck refs/heads/main | cut -f1) - # ahead: main contains the pinned commit and more; behind or diverged: leave the pin to a person (with no - # common history the comparison itself fails) + release=$(gh api repos/empty-sekai/ournotes-deck/releases/latest) + tag=$(jq -r .tag_name <<< "$release") + head=$(gh api "repos/empty-sekai/ournotes-deck/commits/$tag" --jq .sha) + published=true + for module in replay recommend; do + jq -e --arg name "ournotes-$module-wasm-$tag.tar.gz" 'any(.assets[]; .name == $name)' <<< "$release" \ + > /dev/null || published=false + done + # ahead: the release contains the pinned commit and more; behind or diverged: leave the pin to a person + # (with no common history the comparison itself fails) status=$(gh api "repos/empty-sekai/ournotes-deck/compare/$pinned...$head" --jq .status) - echo "pinned $pinned, main $head: $status" + echo "pinned $pinned, $tag $head: $status, WASM packages published: $published" echo "pinned=$pinned" >> "$GITHUB_OUTPUT" echo "head=$head" >> "$GITHUB_OUTPUT" - echo "move=$([ "$status" = ahead ] && echo true || echo false)" >> "$GITHUB_OUTPUT" + echo "tag=$tag" >> "$GITHUB_OUTPUT" + echo "move=$([ "$status" = ahead ] && [ "$published" = true ] && echo true || echo false)" >> "$GITHUB_OUTPUT" - name: Move the pin if: steps.deck.outputs.move == 'true' working-directory: rust @@ -45,11 +55,12 @@ jobs: if: steps.deck.outputs.move == 'true' with: branch: deck/update - commit-message: "build(deck): ournotes-deck ${{ steps.deck.outputs.head }}" - title: "build(deck): ournotes-deck ${{ steps.deck.outputs.head }}" + commit-message: "build(deck): ournotes-deck ${{ steps.deck.outputs.tag }}" + title: "build(deck): ournotes-deck ${{ steps.deck.outputs.tag }}" body: | - Moves the deck model to ournotes-deck's main, - https://github.com/empty-sekai/ournotes-deck/compare/${{ steps.deck.outputs.pinned }}...${{ steps.deck.outputs.head }} + Moves the deck model to ournotes-deck ${{ steps.deck.outputs.tag }} (${{ steps.deck.outputs.head }}), + https://github.com/empty-sekai/ournotes-deck/releases/tag/${{ steps.deck.outputs.tag }} + Changes: https://github.com/empty-sekai/ournotes-deck/compare/${{ steps.deck.outputs.pinned }}...${{ steps.deck.outputs.head }} Music data files made after this change name the new commit in `provenance.deck.commit`; their deck statistics may differ. diff --git a/docs/music-data.md b/docs/music-data.md index ef4caf9..7f30142 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -432,12 +432,15 @@ The deck model reads the same content under the format name `nnnotes.deck-data/1 ## Building -The deck model is the Rust crate ournotes-sim of the ournotes-deck repository, pinned by commit in `rust/Cargo.toml` +The deck model is the Rust crate ournotes-sim of the ournotes-deck repository, pinned to the commit of an ournotes-deck +release in `rust/Cargo.toml` (and `rust/Cargo.lock`) and built into the extension module `nnnotes._deck` with [maturin](https://www.maturin.rs/) (PyO3, the stable ABI of Python 3.11 and later: one wheel per platform). The release workflow builds the wheels; `pip install .` or `pip install -e .` in a checkout builds the module with the Rust toolchain. The same nnnotes version always carries the same deck model: the commit moves only through a pull request (`.github/workflows/deck.yml` -opens one when ournotes-deck's `main` moves), and `provenance.deck.commit` names it in every file. +opens one when ournotes-deck publishes a newer release with its WASM packages), and `provenance.deck.commit` names it in +every file. The replay and recommendation engines of `--replay-engine` and `--recommend-engine` are that release's WASM +packages. ## Versions @@ -454,4 +457,4 @@ their facts here) and the `nnnotes.deck-data/1` file of `nnnotes deck-data` (its Final aptitude exports require the standard-error target for both `score` and `scorePerfect`, each against its own paired no-skill baseline. Sampling extends the same published seed prefix through geometric batches, stopping as soon as both targets agree with the unchanged max(1% of increment, 0.1% of baseline) rule. The 65,536-seed cap is a failure guard, not a requirement to run every seed. An unmet cap aborts a normal export before writing artifacts. `--allow-unconverged-aptitude` is an explicit diagnostic option; it retains real SE values and unmet flags. -`--replay-dir OUT/replay --replay-engine WASM_PKG [--recommend-engine WASM_PKG]` writes normalized runtime inputs and pinned WASM assets (the replay engine and, optionally, the recommendation engine) as described in [replay.md](replay.md). The music data stays compact and carries the SHA-bound `replay.manifestUrl` pointer. No original chart/master blobs or native binary are included in this artifact bundle. +`--replay-dir OUT/replay --replay-engine PKG [--recommend-engine PKG]` writes normalized runtime inputs and the pinned model's WASM release packages (the replay engine and, optionally, the recommendation engine) as described in [replay.md](replay.md). The music data stays compact and carries the SHA-bound `replay.manifestUrl` pointer. No original chart/master blobs or native binary are included in this artifact bundle. diff --git a/docs/replay.md b/docs/replay.md index 867afc7..7a4c747 100644 --- a/docs/replay.md +++ b/docs/replay.md @@ -1,6 +1,6 @@ # Shared Rust/WASM replay inputs -`nnnotes music-data --replay-dir OUT/replay --replay-engine WASM_PKG [--recommend-engine WASM_PKG] -o OUT/music-data.json` writes the normalized input of the pinned deck model. The replay engine package must contain `ournotes_replay.js`, `ournotes_replay_bg.wasm` and `build.json` (`format:"ournotes.replay-engine/1"`); the optional recommendation engine package, the wasm-bindgen `--target web` build of the deck model's `wasm/recommend`, must contain `ournotes_recommend.js`, `ournotes_recommend_bg.wasm` and `build.json` (`format:"ournotes.recommend-engine/1"`). Each `build.json` records `commit`, `jsSha256` and `wasmSha256`; the commit and both file hashes must match the measured model and the package files. Actual ACB cue length is required. Original encrypted master files, chart blobs and native binaries are not copied. +`nnnotes music-data --replay-dir OUT/replay --replay-engine PKG [--recommend-engine PKG] -o OUT/music-data.json` writes the normalized input of the pinned deck model. Each engine is an ournotes-deck WASM release package of the pinned model, given as the downloaded archive or its extracted directory: `ournotes-replay-wasm-vVERSION.tar.gz` for `--replay-engine` and, optionally, `ournotes-recommend-wasm-vVERSION.tar.gz` for `--recommend-engine`. Its `build-info.json` must describe that module's WASM package (`kind:"wasm"`, `module`), name the pinned model's `commit`, and list in `files` the SHA-256 of the web binding's JS and WASM files, which must match the package files. The web binding (`web/ournotes__wasm.js` and `web/ournotes__wasm_bg.wasm`) and `build-info.json` are copied byte for byte. Actual ACB cue length is required. Original encrypted master files, chart blobs and native binaries are not copied. The music-data pointer is: @@ -18,15 +18,15 @@ The music-data pointer is: "engine": { "model": {"name":"ournotes-deck","version":"","source":"","commit":"","format":"ournotes-deck.chart-stats/2"}, "requestFormat":"ournotes.replay/1","class":"ReplaySession","methods":["describeChart","template","run"], - "js":{"url":"engine/ournotes_replay.js","sha256":"","bytes":0}, - "wasm":{"url":"engine/ournotes_replay_bg.wasm","sha256":"","bytes":0}, - "build":{"url":"engine/build.json","sha256":"","bytes":0} + "js":{"url":"engine/ournotes_replay_wasm.js","sha256":"","bytes":0}, + "wasm":{"url":"engine/ournotes_replay_wasm_bg.wasm","sha256":"","bytes":0}, + "build":{"url":"engine/build-info.json","sha256":"","bytes":0} }, "recommendEngine": { "model": {"name":"ournotes-deck","version":"","source":"","commit":"","format":"ournotes-deck.chart-stats/2"}, - "js":{"url":"recommend/ournotes_recommend.js","sha256":"","bytes":0}, - "wasm":{"url":"recommend/ournotes_recommend_bg.wasm","sha256":"","bytes":0}, - "build":{"url":"recommend/build.json","sha256":"","bytes":0} + "js":{"url":"recommend/ournotes_recommend_wasm.js","sha256":"","bytes":0}, + "wasm":{"url":"recommend/ournotes_recommend_wasm_bg.wasm","sha256":"","bytes":0}, + "build":{"url":"recommend/build-info.json","sha256":"","bytes":0} }, "unlistedScoreIds":[], "clock":"Explicit frames from ReplaySession.template; no Python/JS scoring or scheduling" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 2e45adc..58bfd60 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -14,8 +14,8 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.29", features = ["abi3-py311"] } -# the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned by commit; -# .github/workflows/deck.yml opens a pull request when ournotes-deck's main moves +# the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned to the commit of a release; +# .github/workflows/deck.yml opens a pull request when ournotes-deck publishes a newer release ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "ddf212b601731f9ab74245119d4408d2746a370c" } serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] } diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index 3bc659e..14cc82b 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -1002,10 +1002,12 @@ def target(m, what): help="do not read the BGM cue sheets (every song's bgm.length is null)") c.add_argument("--replay-dir", metavar="DIR", help="write canonical runtime DeckData, per-chart inputs and replay manifest under the output directory") - c.add_argument("--replay-engine", metavar="DIR", - help="copy pinned wasm-bindgen JS/WASM + build.json into --replay-dir") - c.add_argument("--recommend-engine", metavar="DIR", - help="copy the pinned wasm-bindgen recommendation JS/WASM + build.json into --replay-dir") + c.add_argument("--replay-engine", metavar="PKG", + help="the deck model's replay WASM release package (ournotes-replay-wasm-vVERSION.tar.gz or its " + "directory); its web JS/WASM and build-info.json are copied into --replay-dir") + c.add_argument("--recommend-engine", metavar="PKG", + help="the deck model's recommendation WASM release package (ournotes-recommend-wasm-vVERSION.tar.gz " + "or its directory); its web JS/WASM and build-info.json are copied into --replay-dir") c.add_argument("--jackets", metavar="DIR", help="also write every song's jacket as DIR/.webp (at most 320 px on the longer side)") _out(c, "output file (.json, or .json.gz for gzip)") diff --git a/src/nnnotes/replaydata.py b/src/nnnotes/replaydata.py index 1f9d864..d345b6b 100644 --- a/src/nnnotes/replaydata.py +++ b/src/nnnotes/replaydata.py @@ -3,6 +3,7 @@ import hashlib import json +import tarfile from pathlib import Path from . import deckdata @@ -10,44 +11,62 @@ FORMAT = "nnnotes.replay-manifest/1" CHART_FORMAT = "nnnotes.replay-chart/1" -JS = "ournotes_replay.js" -WASM = "ournotes_replay_bg.wasm" -RECOMMEND_JS = "ournotes_recommend.js" -RECOMMEND_WASM = "ournotes_recommend_bg.wasm" +BUILD_INFO = "build-info.json" def _json(value) -> bytes: return deckdata.encode(deckdata._value(value, "replay input")) -def _package(files: dict[str, bytes], source: Path, model: dict | None, name: str, build_format: str, prefix: str, - js: str, wasm: str) -> tuple[str, str, str]: - """Copy a wasm-bindgen package of the pinned deck model (`js`, `wasm`, build.json) into `files` under `prefix`; - build.json must name the model's commit and both file hashes. Returns the JS, WASM and build.json paths.""" - source = Path(source) +def _read_package(source: Path, paths: tuple[str, ...], name: str) -> dict[str, bytes]: + """Those of the files `paths` that a release package holds: a directory, or a .tar.gz archive holding the package + directory.""" + if source.is_dir(): + return {path: (source / path).read_bytes() for path in paths if (source / path).is_file()} try: - built = json.loads((source / "build.json").read_text(encoding="utf8")) - except (OSError, ValueError): - raise deckdata.DeckDataError(f"{name}: missing or malformed build.json") from None + with tarfile.open(source, "r:gz") as archive: + return {member.name.partition("/")[2]: archive.extractfile(member).read() + for member in archive.getmembers() if member.isfile() and member.name.partition("/")[2] in paths} + except (OSError, tarfile.TarError): + raise deckdata.DeckDataError( + f"{name}: {source} is neither a package directory nor a .tar.gz package") from None + + +def _package(files: dict[str, bytes], source: Path, model: dict | None, module: str, + prefix: str) -> tuple[str, str, str]: + """Copy the web binding of an ournotes-deck WASM release package of `module` (`replay` or `recommend`), the + `ournotes--wasm-v` .tar.gz or its directory, into `files` under `prefix`. Its build-info.json must + describe the WASM package of `module` built from the pinned model's commit, with the SHA-256 of both files. + Returns the JS, WASM and build-info.json paths.""" + name = f"{module} engine" + stem = f"web/ournotes_{module}_wasm" + js, wasm = f"{stem}.js", f"{stem}_bg.wasm" + package = _read_package(Path(source), (BUILD_INFO, js, wasm), name) if not model or not model.get("commit"): raise deckdata.DeckDataError(f"{name}: the pinned deck model identity is required") - if built.get("format") != build_format: - raise deckdata.DeckDataError(f"{name}: build.json format is {built.get('format')!r}, not {build_format}") - if built.get("commit") != model["commit"]: - raise deckdata.DeckDataError(f"{name}: build.json commit {built.get('commit')!r} differs from the pinned " + if BUILD_INFO not in package: + raise deckdata.DeckDataError(f"{name}: missing {BUILD_INFO} in {source}") + try: + info = json.loads(package[BUILD_INFO]) + except ValueError: + raise deckdata.DeckDataError(f"{name}: malformed {BUILD_INFO}") from None + if not isinstance(info, dict) or info.get("kind") != "wasm" or info.get("module") != module: + raise deckdata.DeckDataError(f"{name}: {BUILD_INFO} does not describe the {module} WASM package") + if info.get("commit") != model["commit"]: + raise deckdata.DeckDataError(f"{name}: {BUILD_INFO} commit {info.get('commit')!r} differs from the pinned " f"deck model {model['commit']}") - js_path, wasm_path, build_path = f"{prefix}/{js}", f"{prefix}/{wasm}", f"{prefix}/build.json" - for file, path in ((js, js_path), (wasm, wasm_path)): - if not (source / file).is_file(): - raise deckdata.DeckDataError(f"{name}: missing {file} in {source}") - files[path] = (source / file).read_bytes() - if not files[wasm_path].startswith(b"\x00asm\x01\x00\x00\x00"): + hashes = info.get("files") if isinstance(info.get("files"), dict) else {} + for path in (js, wasm): + if path not in package: + raise deckdata.DeckDataError(f"{name}: missing {path} in {source}") + if hashes.get(path) != hashlib.sha256(package[path]).hexdigest(): + raise deckdata.DeckDataError(f"{name}: {path} SHA-256 differs from {BUILD_INFO}") + if not package[wasm].startswith(b"\x00asm\x01\x00\x00\x00"): raise deckdata.DeckDataError(f"{name}: not a WASM v1 module") - if built.get("jsSha256") != hashlib.sha256(files[js_path]).hexdigest() or \ - built.get("wasmSha256") != hashlib.sha256(files[wasm_path]).hexdigest(): - raise deckdata.DeckDataError(f"{name}: JS/WASM SHA differs from build.json") - files[build_path] = _json(built) - return js_path, wasm_path, build_path + paths = tuple(f"{prefix}/{path.rsplit('/', 1)[-1]}" for path in (js, wasm, BUILD_INFO)) + for path, data in zip(paths, (package[js], package[wasm], package[BUILD_INFO])): + files[path] = data + return paths def bundle(music: dict, engine_dir: Path | None = None, @@ -93,15 +112,13 @@ def resource(path: str) -> dict: model = music["provenance"].get("deck") engine = None if engine_dir is not None: - js, wasm, build = _package(files, engine_dir, model, "replay engine", "ournotes.replay-engine/1", "engine", - JS, WASM) + js, wasm, build = _package(files, engine_dir, model, "replay", "engine") engine = {"model": model, "requestFormat": "ournotes.replay/1", "class": "ReplaySession", "methods": ["describeChart", "template", "run"], "js": resource(js), "wasm": resource(wasm), "build": resource(build)} recommend = None if recommend_dir is not None: - js, wasm, build = _package(files, recommend_dir, model, "recommend engine", "ournotes.recommend-engine/1", - "recommend", RECOMMEND_JS, RECOMMEND_WASM) + js, wasm, build = _package(files, recommend_dir, model, "recommend", "recommend") recommend = {"model": model, "js": resource(js), "wasm": resource(wasm), "build": resource(build)} manifest = {"format": FORMAT, "deckData": {"format": deckdata.DECK_FORMAT, **resource("deck-data.json")}, "charts": charts, "engine": engine, diff --git a/tests/test_replaydata.py b/tests/test_replaydata.py index b80b4ac..dd1b02d 100644 --- a/tests/test_replaydata.py +++ b/tests/test_replaydata.py @@ -1,6 +1,7 @@ """Runtime inputs retain normalized note order/identities and bind the engine to the pinned model.""" import hashlib import json +import tarfile import pytest @@ -36,23 +37,43 @@ def test_replay_requires_actual_audio_length_and_contains_no_guess(tmp_path): assert not (tmp_path / "music.json").exists() +WASM_MAGIC = b"\x00asm\x01\x00\x00\x00" + + +def release_package(directory, module, commit=FakeDeck.COMMIT, js=None): + """A synthetic ournotes-deck WASM release package of `module`: its web binding and build-info.json.""" + js = js if js is not None else f"export class {module.title()}Session {{}}".encode() + web = directory / "web" + web.mkdir(parents=True, exist_ok=True) + stem = f"ournotes_{module}_wasm" + (web / f"{stem}.js").write_bytes(js) + (web / f"{stem}_bg.wasm").write_bytes(WASM_MAGIC) + info = {"version": "0.0.1", "tag": "v0.0.1", "commit": commit, "kind": "wasm", "module": module, + "target": "wasm32-unknown-unknown", + "files": {f"web/{stem}.js": hashlib.sha256(js).hexdigest(), + f"web/{stem}_bg.wasm": hashlib.sha256(WASM_MAGIC).hexdigest()}} + (directory / replaydata.BUILD_INFO).write_text(json.dumps(info, indent=2) + "\n") + return info + + +def archive(directory): + """The release archive of a package directory: the directory itself at the archive root.""" + path = directory.parent / f"{directory.name}.tar.gz" + with tarfile.open(path, "w:gz") as output: + output.add(directory, arcname=directory.name) + return path + + def test_engine_manifest_rejects_stale_pin_and_changed_wasm(tmp_path): - engine = tmp_path / "engine" - engine.mkdir() - js, wasm = b"export class ReplaySession {}", b"\x00asm\x01\x00\x00\x00" - (engine / replaydata.JS).write_bytes(js) - (engine / replaydata.WASM).write_bytes(wasm) - built = {"format": "ournotes.replay-engine/1", "commit": "0" * 40, - "jsSha256": hashlib.sha256(js).hexdigest(), "wasmSha256": hashlib.sha256(wasm).hexdigest()} - (engine / "build.json").write_text(json.dumps(built)) + engine = tmp_path / "ournotes-replay-wasm-v0.0.1" + release_package(engine, "replay", commit="0" * 40) stale = tmp_path / "stale" stale.mkdir() - stale_pin = f"build.json commit '0{{40}}' differs from the pinned deck model {FakeDeck.COMMIT}" + stale_pin = f"build-info.json commit '0{{40}}' differs from the pinned deck model {FakeDeck.COMMIT}" with pytest.raises(musicdata.MusicDataError, match=stale_pin): export(stale, deck=FakeDeck(), replay_dir=stale / "replay", replay_engine=engine) assert not (stale / "music.json").exists() - built["commit"] = FakeDeck.COMMIT - (engine / "build.json").write_text(json.dumps(built)) + release_package(engine, "replay") valid = tmp_path / "valid" valid.mkdir() result = export(valid, deck=FakeDeck(), replay_dir=valid / "replay", replay_engine=engine) @@ -60,54 +81,34 @@ def test_engine_manifest_rejects_stale_pin_and_changed_wasm(tmp_path): old_bytes = old_manifest_path.read_bytes() manifest = json.loads(old_bytes) assert manifest["engine"]["model"]["commit"] == FakeDeck.COMMIT - new_js = js + b"\n// updated engine" - (engine / replaydata.JS).write_bytes(new_js) - built["jsSha256"] = hashlib.sha256(new_js).hexdigest() - (engine / "build.json").write_text(json.dumps(built)) + release_package(engine, "replay", js=b"export class ReplaySession {}\n// updated engine") result = musicdata.export(valid / "music.json", deckdata.master_files(valid / "m"), KEY, CHARTS.__getitem__, bgm, **PROV, deck=musicdata.Deck(module=FakeDeck()), replay_dir=valid / "replay", replay_engine=engine) assert valid / result["replay"]["manifestUrl"] != old_manifest_path assert old_manifest_path.read_bytes() == old_bytes # readers of the preceding document keep its resource tree - (engine / replaydata.WASM).write_bytes(wasm + b"changed") + (engine / "web" / "ournotes_replay_wasm_bg.wasm").write_bytes(WASM_MAGIC + b"changed") changed = tmp_path / "changed" changed.mkdir() - with pytest.raises(musicdata.MusicDataError, match="SHA differs"): + with pytest.raises(musicdata.MusicDataError, match="ournotes_replay_wasm_bg.wasm SHA-256 differs"): export(changed, deck=FakeDeck(), replay_dir=changed / "replay", replay_engine=engine) -def package(directory, js_name, wasm_name, build_format, commit, js=b"export class Session {}"): - """A synthetic wasm-bindgen package with its build.json.""" - directory.mkdir(parents=True, exist_ok=True) - wasm = b"\x00asm\x01\x00\x00\x00" - (directory / js_name).write_bytes(js) - (directory / wasm_name).write_bytes(wasm) - built = {"format": build_format, "commit": commit, "workingTreeDirty": False, - "jsSha256": hashlib.sha256(js).hexdigest(), "wasmSha256": hashlib.sha256(wasm).hexdigest()} - (directory / "build.json").write_text(json.dumps(built)) - return built - - -def replay_package(tmp_path): - return package(tmp_path / "replay-pkg", replaydata.JS, replaydata.WASM, "ournotes.replay-engine/1", FakeDeck.COMMIT) - - -def recommend_package(tmp_path, commit=FakeDeck.COMMIT, build_format="ournotes.recommend-engine/1"): - return package(tmp_path / "recommend-pkg", replaydata.RECOMMEND_JS, replaydata.RECOMMEND_WASM, build_format, commit, - b"export class RecommendationSession {}") - - -def test_recommend_engine_shares_the_model_of_the_deck_data(tmp_path): - replay_package(tmp_path) - built = recommend_package(tmp_path) +@pytest.mark.parametrize("packed", [False, True]) +def test_recommend_engine_shares_the_model_of_the_deck_data(tmp_path, packed): + replay = tmp_path / "ournotes-replay-wasm-v0.0.1" + recommend = tmp_path / "ournotes-recommend-wasm-v0.0.1" + release_package(replay, "replay") + info = release_package(recommend, "recommend") + replay_source, recommend_source = (archive(replay), archive(recommend)) if packed else (replay, recommend) plain = tmp_path / "plain" plain.mkdir() - result = export(plain, deck=FakeDeck(), replay_dir=plain / "replay", replay_engine=tmp_path / "replay-pkg") + result = export(plain, deck=FakeDeck(), replay_dir=plain / "replay", replay_engine=replay_source) assert "recommendEngine" not in json.loads((plain / result["replay"]["manifestUrl"]).read_bytes()) out = tmp_path / "out" out.mkdir() - result = export(out, deck=FakeDeck(), replay_dir=out / "replay", replay_engine=tmp_path / "replay-pkg", - recommend_engine=tmp_path / "recommend-pkg") + result = export(out, deck=FakeDeck(), replay_dir=out / "replay", replay_engine=replay_source, + recommend_engine=recommend_source) manifest_path = out / result["replay"]["manifestUrl"] manifest = json.loads(manifest_path.read_bytes()) entry = manifest["recommendEngine"] @@ -116,38 +117,45 @@ def test_recommend_engine_shares_the_model_of_the_deck_data(tmp_path): assert entry["model"] == manifest["engine"]["model"] == data["provenance"]["deck"] assert entry["model"]["commit"] == FakeDeck.COMMIT assert [entry[k]["url"] for k in ("js", "wasm", "build")] == [ - "recommend/ournotes_recommend.js", "recommend/ournotes_recommend_bg.wasm", "recommend/build.json"] - assert manifest["engine"]["build"]["url"] == "engine/build.json" - for key, name in (("js", replaydata.RECOMMEND_JS), ("wasm", replaydata.RECOMMEND_WASM)): + "recommend/ournotes_recommend_wasm.js", "recommend/ournotes_recommend_wasm_bg.wasm", "recommend/build-info.json"] + assert [manifest["engine"][k]["url"] for k in ("js", "wasm", "build")] == [ + "engine/ournotes_replay_wasm.js", "engine/ournotes_replay_wasm_bg.wasm", "engine/build-info.json"] + for key, name in (("js", "ournotes_recommend_wasm.js"), ("wasm", "ournotes_recommend_wasm_bg.wasm")): payload = (manifest_path.parent / entry[key]["url"]).read_bytes() - assert payload == (tmp_path / "recommend-pkg" / name).read_bytes() + assert payload == (recommend / "web" / name).read_bytes() assert (entry[key]["sha256"], entry[key]["bytes"]) == (hashlib.sha256(payload).hexdigest(), len(payload)) - assert (entry["js"]["sha256"], entry["wasm"]["sha256"]) == (built["jsSha256"], built["wasmSha256"]) + assert entry[key]["sha256"] == info["files"][f"web/{name}"] build = (manifest_path.parent / entry["build"]["url"]).read_bytes() - assert json.loads(build) == built + assert build == (recommend / replaydata.BUILD_INFO).read_bytes() assert (entry["build"]["sha256"], entry["build"]["bytes"]) == (hashlib.sha256(build).hexdigest(), len(build)) @pytest.mark.parametrize("change,message", [ - ("commit", "recommend engine: build.json commit '0{40}' differs from the pinned deck model"), - ("format", "recommend engine: build.json format is 'ournotes.replay-engine/1', not ournotes.recommend-engine/1"), - ("wasm", "recommend engine: JS/WASM SHA differs"), - ("missing", "recommend engine: missing ournotes_recommend.js"), + ("commit", "recommend engine: build-info.json commit '0{40}' differs from the pinned deck model"), + ("module", "recommend engine: build-info.json does not describe the recommend WASM package"), + ("wasm", "recommend engine: web/ournotes_recommend_wasm_bg.wasm SHA-256 differs from build-info.json"), + ("missing", "recommend engine: missing web/ournotes_recommend_wasm.js"), + ("not-a-package", "recommend engine: .* is neither a package directory nor a .tar.gz package"), ("no-replay-dir", "--recommend-engine needs --replay-dir"), ]) def test_recommend_engine_rejects_another_build(tmp_path, change, message): - replay_package(tmp_path) - recommend_package(tmp_path, commit="0" * 40 if change == "commit" else FakeDeck.COMMIT, - build_format="ournotes.replay-engine/1" if change == "format" else "ournotes.recommend-engine/1") + replay = tmp_path / "replay-pkg" + release_package(replay, "replay") pkg = tmp_path / "recommend-pkg" + release_package(pkg, "recommend", commit="0" * 40 if change == "commit" else FakeDeck.COMMIT) + if change == "module": + pkg = tmp_path / "other-pkg" + release_package(pkg, "replay") if change == "wasm": - (pkg / replaydata.RECOMMEND_WASM).write_bytes(b"\x00asm\x01\x00\x00\x00changed") + (pkg / "web" / "ournotes_recommend_wasm_bg.wasm").write_bytes(WASM_MAGIC + b"changed") if change == "missing": - (pkg / replaydata.RECOMMEND_JS).unlink() - replay = {} if change == "no-replay-dir" else {"replay_dir": tmp_path / "replay", - "replay_engine": tmp_path / "replay-pkg"} + (pkg / "web" / "ournotes_recommend_wasm.js").unlink() + if change == "not-a-package": + pkg = tmp_path / "recommend.tar.gz" + pkg.write_bytes(b"not an archive") + options = {} if change == "no-replay-dir" else {"replay_dir": tmp_path / "replay", "replay_engine": replay} with pytest.raises(musicdata.MusicDataError, match=message): - export(tmp_path, deck=FakeDeck(), recommend_engine=pkg, **replay) + export(tmp_path, deck=FakeDeck(), recommend_engine=pkg, **options) assert not (tmp_path / "music.json").exists() and not (tmp_path / "replay").exists() From 408358ac7ef77f1e5022e0f069463e9304ebe2f9 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Tue, 6 Oct 2026 09:06:52 +0800 Subject: [PATCH 2/2] build(deck): ournotes-deck v0.0.2 Moves the deck model to ournotes-deck v0.0.2 (76e96b617842bdcc031cce98e3dfeead27ddbb85), https://github.com/empty-sekai/ournotes-deck/releases/tag/v0.0.2 Music data files made after this change name the new commit in `provenance.deck.commit`; their deck statistics may differ. Co-Authored-By: Claude Opus 5.5 --- rust/Cargo.lock | 4 ++-- rust/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c294017..78763a6 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -95,8 +95,8 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "ournotes-sim" -version = "0.0.1" -source = "git+https://github.com/empty-sekai/ournotes-deck?rev=ddf212b601731f9ab74245119d4408d2746a370c#ddf212b601731f9ab74245119d4408d2746a370c" +version = "0.0.2" +source = "git+https://github.com/empty-sekai/ournotes-deck?rev=76e96b617842bdcc031cce98e3dfeead27ddbb85#76e96b617842bdcc031cce98e3dfeead27ddbb85" dependencies = [ "serde", "serde_json", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 58bfd60..44aca0f 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -16,7 +16,7 @@ crate-type = ["cdylib"] pyo3 = { version = "0.29", features = ["abi3-py311"] } # the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned to the commit of a release; # .github/workflows/deck.yml opens a pull request when ournotes-deck publishes a newer release -ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "ddf212b601731f9ab74245119d4408d2746a370c" } +ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "76e96b617842bdcc031cce98e3dfeead27ddbb85" } serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] } [profile.release]