feat(music-data): take the deck model's WASM release packages as engines

.github/workflows/deck.yml

@@ -1,9 +1,11 @@
 name: Deck model
 
 # nnnotes builds the deck model, the crate ournotes-sim of the ournotes-deck repository, at the commit rust/Cargo.toml
-# pins, so that one nnnotes version always carries one deck model. When ournotes-deck's main moves past that commit,
-# this opens a pull request that moves the pin and dispatches CI and the commit message check on its branch (a pull
-# request opened with the workflow token starts no workflow by itself); their checks land on the pull request.
+# pins, so that one nnnotes version always carries one deck model. The pin follows ournotes-deck's releases: when the
+# latest release has published its replay and recommendation WASM packages (the engines of music-data --replay-engine
+# and --recommend-engine) and its commit is past the pinned one, this opens a pull request that moves the pin and
+# dispatches CI and the commit message check on its branch (a pull request opened with the workflow token starts no
+# workflow by itself); their checks land on the pull request.
 on:
   schedule:
     - cron: "17 3 * * *"
@@ -19,21 +21,29 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
-      - name: Compare the pinned commit with ournotes-deck's main
+      - name: Compare the pinned commit with ournotes-deck's latest release
         id: deck
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           pinned=$(sed -nE 's/^ournotes-sim = .*rev = "([0-9a-f]{40})".*/\1/p' rust/Cargo.toml)
           [ -n "$pinned" ] || { echo "rust/Cargo.toml pins no ournotes-sim commit" >&2; exit 1; }
-          head=$(git ls-remote https://github.com/empty-sekai/ournotes-deck refs/heads/main | cut -f1)
-          # ahead: main contains the pinned commit and more; behind or diverged: leave the pin to a person (with no
-          # common history the comparison itself fails)
+          release=$(gh api repos/empty-sekai/ournotes-deck/releases/latest)
+          tag=$(jq -r .tag_name <<< "$release")
+          head=$(gh api "repos/empty-sekai/ournotes-deck/commits/$tag" --jq .sha)
+          published=true
+          for module in replay recommend; do
+            jq -e --arg name "ournotes-$module-wasm-$tag.tar.gz" 'any(.assets[]; .name == $name)' <<< "$release" \
+              > /dev/null || published=false
+          done
+          # ahead: the release contains the pinned commit and more; behind or diverged: leave the pin to a person
+          # (with no common history the comparison itself fails)
           status=$(gh api "repos/empty-sekai/ournotes-deck/compare/$pinned...$head" --jq .status)
-          echo "pinned $pinned, main $head: $status"
+          echo "pinned $pinned, $tag $head: $status, WASM packages published: $published"
           echo "pinned=$pinned" >> "$GITHUB_OUTPUT"
           echo "head=$head" >> "$GITHUB_OUTPUT"
-          echo "move=$([ "$status" = ahead ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
+          echo "tag=$tag" >> "$GITHUB_OUTPUT"
+          echo "move=$([ "$status" = ahead ] && [ "$published" = true ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
       - name: Move the pin
         if: steps.deck.outputs.move == 'true'
         working-directory: rust
@@ -45,11 +55,12 @@ jobs:
         if: steps.deck.outputs.move == 'true'
         with:
           branch: deck/update
-          commit-message: "build(deck): ournotes-deck ${{ steps.deck.outputs.head }}"
-          title: "build(deck): ournotes-deck ${{ steps.deck.outputs.head }}"
+          commit-message: "build(deck): ournotes-deck ${{ steps.deck.outputs.tag }}"
+          title: "build(deck): ournotes-deck ${{ steps.deck.outputs.tag }}"
           body: |
-            Moves the deck model to ournotes-deck's main,
-            https://github.com/empty-sekai/ournotes-deck/compare/${{ steps.deck.outputs.pinned }}...${{ steps.deck.outputs.head }}
+            Moves the deck model to ournotes-deck ${{ steps.deck.outputs.tag }} (${{ steps.deck.outputs.head }}),
+            https://github.com/empty-sekai/ournotes-deck/releases/tag/${{ steps.deck.outputs.tag }}
+            Changes: https://github.com/empty-sekai/ournotes-deck/compare/${{ steps.deck.outputs.pinned }}...${{ steps.deck.outputs.head }}
 
             Music data files made after this change name the new commit in `provenance.deck.commit`; their deck
             statistics may differ.

docs/music-data.md

@@ -432,12 +432,15 @@ The deck model reads the same content under the format name `nnnotes.deck-data/1
 
 ## Building
 
-The deck model is the Rust crate ournotes-sim of the ournotes-deck repository, pinned by commit in `rust/Cargo.toml`
+The deck model is the Rust crate ournotes-sim of the ournotes-deck repository, pinned to the commit of an ournotes-deck
+release in `rust/Cargo.toml`
 (and `rust/Cargo.lock`) and built into the extension module `nnnotes._deck` with [maturin](https://www.maturin.rs/)
 (PyO3, the stable ABI of Python 3.11 and later: one wheel per platform). The release workflow builds the wheels;
 `pip install .` or `pip install -e .` in a checkout builds the module with the Rust toolchain. The same nnnotes
 version always carries the same deck model: the commit moves only through a pull request (`.github/workflows/deck.yml`
-opens one when ournotes-deck's `main` moves), and `provenance.deck.commit` names it in every file.
+opens one when ournotes-deck publishes a newer release with its WASM packages), and `provenance.deck.commit` names it in
+every file. The replay and recommendation engines of `--replay-engine` and `--recommend-engine` are that release's WASM
+packages.
 
 ## Versions
 
@@ -454,4 +457,4 @@ their facts here) and the `nnnotes.deck-data/1` file of `nnnotes deck-data` (its
 
 Final aptitude exports require the standard-error target for both `score` and `scorePerfect`, each against its own paired no-skill baseline. Sampling extends the same published seed prefix through geometric batches, stopping as soon as both targets agree with the unchanged max(1% of increment, 0.1% of baseline) rule. The 65,536-seed cap is a failure guard, not a requirement to run every seed. An unmet cap aborts a normal export before writing artifacts. `--allow-unconverged-aptitude` is an explicit diagnostic option; it retains real SE values and unmet flags.
 
-`--replay-dir OUT/replay --replay-engine WASM_PKG [--recommend-engine WASM_PKG]` writes normalized runtime inputs and pinned WASM assets (the replay engine and, optionally, the recommendation engine) as described in [replay.md](replay.md). The music data stays compact and carries the SHA-bound `replay.manifestUrl` pointer. No original chart/master blobs or native binary are included in this artifact bundle.
+`--replay-dir OUT/replay --replay-engine PKG [--recommend-engine PKG]` writes normalized runtime inputs and the pinned model's WASM release packages (the replay engine and, optionally, the recommendation engine) as described in [replay.md](replay.md). The music data stays compact and carries the SHA-bound `replay.manifestUrl` pointer. No original chart/master blobs or native binary are included in this artifact bundle.

docs/replay.md

@@ -1,6 +1,6 @@
 # Shared Rust/WASM replay inputs
 
-`nnnotes music-data --replay-dir OUT/replay --replay-engine WASM_PKG [--recommend-engine WASM_PKG] -o OUT/music-data.json` writes the normalized input of the pinned deck model. The replay engine package must contain `ournotes_replay.js`, `ournotes_replay_bg.wasm` and `build.json` (`format:"ournotes.replay-engine/1"`); the optional recommendation engine package, the wasm-bindgen `--target web` build of the deck model's `wasm/recommend`, must contain `ournotes_recommend.js`, `ournotes_recommend_bg.wasm` and `build.json` (`format:"ournotes.recommend-engine/1"`). Each `build.json` records `commit`, `jsSha256` and `wasmSha256`; the commit and both file hashes must match the measured model and the package files. Actual ACB cue length is required. Original encrypted master files, chart blobs and native binaries are not copied.
+`nnnotes music-data --replay-dir OUT/replay --replay-engine PKG [--recommend-engine PKG] -o OUT/music-data.json` writes the normalized input of the pinned deck model. Each engine is an ournotes-deck WASM release package of the pinned model, given as the downloaded archive or its extracted directory: `ournotes-replay-wasm-vVERSION.tar.gz` for `--replay-engine` and, optionally, `ournotes-recommend-wasm-vVERSION.tar.gz` for `--recommend-engine`. Its `build-info.json` must describe that module's WASM package (`kind:"wasm"`, `module`), name the pinned model's `commit`, and list in `files` the SHA-256 of the web binding's JS and WASM files, which must match the package files. The web binding (`web/ournotes_<module>_wasm.js` and `web/ournotes_<module>_wasm_bg.wasm`) and `build-info.json` are copied byte for byte. Actual ACB cue length is required. Original encrypted master files, chart blobs and native binaries are not copied.
 
 The music-data pointer is:
 
@@ -18,15 +18,15 @@ The music-data pointer is:
   "engine": {
     "model": {"name":"ournotes-deck","version":"<version>","source":"<repository>","commit":"<pinned commit>","format":"ournotes-deck.chart-stats/2"},
     "requestFormat":"ournotes.replay/1","class":"ReplaySession","methods":["describeChart","template","run"],
-    "js":{"url":"engine/ournotes_replay.js","sha256":"<SHA>","bytes":0},
-    "wasm":{"url":"engine/ournotes_replay_bg.wasm","sha256":"<SHA>","bytes":0},
-    "build":{"url":"engine/build.json","sha256":"<SHA>","bytes":0}
+    "js":{"url":"engine/ournotes_replay_wasm.js","sha256":"<SHA>","bytes":0},
+    "wasm":{"url":"engine/ournotes_replay_wasm_bg.wasm","sha256":"<SHA>","bytes":0},
+    "build":{"url":"engine/build-info.json","sha256":"<SHA>","bytes":0}
   },
   "recommendEngine": {
     "model": {"name":"ournotes-deck","version":"<version>","source":"<repository>","commit":"<pinned commit>","format":"ournotes-deck.chart-stats/2"},
-    "js":{"url":"recommend/ournotes_recommend.js","sha256":"<SHA>","bytes":0},
-    "wasm":{"url":"recommend/ournotes_recommend_bg.wasm","sha256":"<SHA>","bytes":0},
-    "build":{"url":"recommend/build.json","sha256":"<SHA>","bytes":0}
+    "js":{"url":"recommend/ournotes_recommend_wasm.js","sha256":"<SHA>","bytes":0},
+    "wasm":{"url":"recommend/ournotes_recommend_wasm_bg.wasm","sha256":"<SHA>","bytes":0},
+    "build":{"url":"recommend/build-info.json","sha256":"<SHA>","bytes":0}
   },
   "unlistedScoreIds":[],
   "clock":"Explicit frames from ReplaySession.template; no Python/JS scoring or scheduling"

rust/Cargo.toml

@@ -14,9 +14,9 @@ crate-type = ["cdylib"]
 
 [dependencies]
 pyo3 = { version = "0.29", features = ["abi3-py311"] }
-# the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned by commit;
-# .github/workflows/deck.yml opens a pull request when ournotes-deck's main moves
-ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "ddf212b601731f9ab74245119d4408d2746a370c" }
+# the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned to the commit of a release;
+# .github/workflows/deck.yml opens a pull request when ournotes-deck publishes a newer release
+ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "76e96b617842bdcc031cce98e3dfeead27ddbb85" }
 serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] }
 
 [profile.release]

src/nnnotes/cli.py

@@ -1002,10 +1002,12 @@ def target(m, what):
                    help="do not read the BGM cue sheets (every song's bgm.length is null)")
     c.add_argument("--replay-dir", metavar="DIR",
                    help="write canonical runtime DeckData, per-chart inputs and replay manifest under the output directory")
-    c.add_argument("--replay-engine", metavar="DIR",
-                   help="copy pinned wasm-bindgen JS/WASM + build.json into --replay-dir")
-    c.add_argument("--recommend-engine", metavar="DIR",
-                   help="copy the pinned wasm-bindgen recommendation JS/WASM + build.json into --replay-dir")
+    c.add_argument("--replay-engine", metavar="PKG",
+                   help="the deck model's replay WASM release package (ournotes-replay-wasm-vVERSION.tar.gz or its "
+                        "directory); its web JS/WASM and build-info.json are copied into --replay-dir")
+    c.add_argument("--recommend-engine", metavar="PKG",
+                   help="the deck model's recommendation WASM release package (ournotes-recommend-wasm-vVERSION.tar.gz "
+                        "or its directory); its web JS/WASM and build-info.json are copied into --replay-dir")
     c.add_argument("--jackets", metavar="DIR",
                    help="also write every song's jacket as DIR/<jacket>.webp (at most 320 px on the longer side)")
     _out(c, "output file (.json, or .json.gz for gzip)")

src/nnnotes/replaydata.py

@@ -3,51 +3,70 @@
 
 import hashlib
 import json
+import tarfile
 from pathlib import Path
 
 from . import deckdata
 from .cache import write_atomic
 
 FORMAT = "nnnotes.replay-manifest/1"
 CHART_FORMAT = "nnnotes.replay-chart/1"
-JS = "ournotes_replay.js"
-WASM = "ournotes_replay_bg.wasm"
-RECOMMEND_JS = "ournotes_recommend.js"
-RECOMMEND_WASM = "ournotes_recommend_bg.wasm"
+BUILD_INFO = "build-info.json"
 
 
 def _json(value) -> bytes:
     return deckdata.encode(deckdata._value(value, "replay input"))
 
 
-def _package(files: dict[str, bytes], source: Path, model: dict | None, name: str, build_format: str, prefix: str,
-             js: str, wasm: str) -> tuple[str, str, str]:
-    """Copy a wasm-bindgen package of the pinned deck model (`js`, `wasm`, build.json) into `files` under `prefix`;
-    build.json must name the model's commit and both file hashes. Returns the JS, WASM and build.json paths."""
-    source = Path(source)
+def _read_package(source: Path, paths: tuple[str, ...], name: str) -> dict[str, bytes]:
+    """Those of the files `paths` that a release package holds: a directory, or a .tar.gz archive holding the package
+    directory."""
+    if source.is_dir():
+        return {path: (source / path).read_bytes() for path in paths if (source / path).is_file()}
     try:
-        built = json.loads((source / "build.json").read_text(encoding="utf8"))
-    except (OSError, ValueError):
-        raise deckdata.DeckDataError(f"{name}: missing or malformed build.json") from None
+        with tarfile.open(source, "r:gz") as archive:
+            return {member.name.partition("/")[2]: archive.extractfile(member).read()
+                    for member in archive.getmembers() if member.isfile() and member.name.partition("/")[2] in paths}
+    except (OSError, tarfile.TarError):
+        raise deckdata.DeckDataError(
+            f"{name}: {source} is neither a package directory nor a .tar.gz package") from None
+
+
+def _package(files: dict[str, bytes], source: Path, model: dict | None, module: str,
+             prefix: str) -> tuple[str, str, str]:
+    """Copy the web binding of an ournotes-deck WASM release package of `module` (`replay` or `recommend`), the
+    `ournotes-<module>-wasm-v<version>` .tar.gz or its directory, into `files` under `prefix`. Its build-info.json must
+    describe the WASM package of `module` built from the pinned model's commit, with the SHA-256 of both files.
+    Returns the JS, WASM and build-info.json paths."""
+    name = f"{module} engine"
+    stem = f"web/ournotes_{module}_wasm"
+    js, wasm = f"{stem}.js", f"{stem}_bg.wasm"
+    package = _read_package(Path(source), (BUILD_INFO, js, wasm), name)
     if not model or not model.get("commit"):
         raise deckdata.DeckDataError(f"{name}: the pinned deck model identity is required")
-    if built.get("format") != build_format:
-        raise deckdata.DeckDataError(f"{name}: build.json format is {built.get('format')!r}, not {build_format}")
-    if built.get("commit") != model["commit"]:
-        raise deckdata.DeckDataError(f"{name}: build.json commit {built.get('commit')!r} differs from the pinned "
+    if BUILD_INFO not in package:
+        raise deckdata.DeckDataError(f"{name}: missing {BUILD_INFO} in {source}")
+    try:
+        info = json.loads(package[BUILD_INFO])
+    except ValueError:
+        raise deckdata.DeckDataError(f"{name}: malformed {BUILD_INFO}") from None
+    if not isinstance(info, dict) or info.get("kind") != "wasm" or info.get("module") != module:
+        raise deckdata.DeckDataError(f"{name}: {BUILD_INFO} does not describe the {module} WASM package")
+    if info.get("commit") != model["commit"]:
+        raise deckdata.DeckDataError(f"{name}: {BUILD_INFO} commit {info.get('commit')!r} differs from the pinned "
                                      f"deck model {model['commit']}")
-    js_path, wasm_path, build_path = f"{prefix}/{js}", f"{prefix}/{wasm}", f"{prefix}/build.json"
-    for file, path in ((js, js_path), (wasm, wasm_path)):
-        if not (source / file).is_file():
-            raise deckdata.DeckDataError(f"{name}: missing {file} in {source}")
-        files[path] = (source / file).read_bytes()
-    if not files[wasm_path].startswith(b"\x00asm\x01\x00\x00\x00"):
+    hashes = info.get("files") if isinstance(info.get("files"), dict) else {}
+    for path in (js, wasm):
+        if path not in package:
+            raise deckdata.DeckDataError(f"{name}: missing {path} in {source}")
+        if hashes.get(path) != hashlib.sha256(package[path]).hexdigest():
+            raise deckdata.DeckDataError(f"{name}: {path} SHA-256 differs from {BUILD_INFO}")
+    if not package[wasm].startswith(b"\x00asm\x01\x00\x00\x00"):
         raise deckdata.DeckDataError(f"{name}: not a WASM v1 module")
-    if built.get("jsSha256") != hashlib.sha256(files[js_path]).hexdigest() or \
-            built.get("wasmSha256") != hashlib.sha256(files[wasm_path]).hexdigest():
-        raise deckdata.DeckDataError(f"{name}: JS/WASM SHA differs from build.json")
-    files[build_path] = _json(built)
-    return js_path, wasm_path, build_path
+    paths = tuple(f"{prefix}/{path.rsplit('/', 1)[-1]}" for path in (js, wasm, BUILD_INFO))
+    for path, data in zip(paths, (package[js], package[wasm], package[BUILD_INFO])):
+        files[path] = data
+    return paths
 
 
 def bundle(music: dict, engine_dir: Path | None = None,
@@ -93,15 +112,13 @@ def resource(path: str) -> dict:
     model = music["provenance"].get("deck")
     engine = None
     if engine_dir is not None:
-        js, wasm, build = _package(files, engine_dir, model, "replay engine", "ournotes.replay-engine/1", "engine",
-                                   JS, WASM)
+        js, wasm, build = _package(files, engine_dir, model, "replay", "engine")
         engine = {"model": model, "requestFormat": "ournotes.replay/1", "class": "ReplaySession",
                   "methods": ["describeChart", "template", "run"],
                   "js": resource(js), "wasm": resource(wasm), "build": resource(build)}
     recommend = None
     if recommend_dir is not None:
-        js, wasm, build = _package(files, recommend_dir, model, "recommend engine", "ournotes.recommend-engine/1",
-                                   "recommend", RECOMMEND_JS, RECOMMEND_WASM)
+        js, wasm, build = _package(files, recommend_dir, model, "recommend", "recommend")
         recommend = {"model": model, "js": resource(js), "wasm": resource(wasm), "build": resource(build)}
     manifest = {"format": FORMAT, "deckData": {"format": deckdata.DECK_FORMAT, **resource("deck-data.json")},
                 "charts": charts, "engine": engine,