diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f527498..ee15bc9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,6 +13,8 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v7 + with: + fetch-depth: 0 - uses: actions/setup-node@v7 with: @@ -23,7 +25,7 @@ jobs: run: npm ci - name: Validate model schemas - run: npm run validate:schema + run: npm run validate:schema -- --base ${{ github.event.pull_request.base.sha }} # The catalog library, the Helm validator, the catalog server and the site. - name: Test diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 7fb4858..8f0da02 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -25,6 +25,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + with: + fetch-depth: 0 - uses: actions/setup-node@v7 with: @@ -40,8 +42,6 @@ jobs: - name: validate run: npm run validate - # index.json is built here, from models/, and checked against the one live - # on the site: a published version must not change (server/data.js). - name: Build Pages site and catalog run: npm run build:site env: diff --git a/.gitignore b/.gitignore index fe00076..3e3ddeb 100644 --- a/.gitignore +++ b/.gitignore @@ -148,6 +148,7 @@ dist # End of https://www.toptal.com/developers/gitignore/api/node # Generated GitHub Pages site site/ +/models/*/*.history.json # Created by https://www.toptal.com/developers/gitignore/api/tags diff --git a/.husky/pre-commit b/.husky/pre-commit index 9a53bb8..784c8d7 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1 +1 @@ -npm run validate:schema +npm run validate:schema -- --base HEAD diff --git a/README.md b/README.md index 4178330..75a9edf 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,49 @@ tuning: A report must be a complete HTML document whose embedded JSON parses. `validate:schema` fails on a missing, empty or invalid one; the site skips linking it. The images it records must be the two variants' own (registry, repository and tag), which `validate:images` checks. +## Schema v2 + +One file per variant, edited in place ([#30](https://github.com/modelsphere/model-catalog/issues/30)). + +``` +models// + metadata.yaml apiVersion: catalog.swiss/v2 schema/metadata.v2.schema.json + .yaml one variant schema/variant.schema.json +``` + +```yaml +# models/kimi-k3/sglang-tp8-pp4-6000d.yaml +apiVersion: catalog.swiss/v2 +kind: Variant +model: kimi-k3 +id: sglang-tp8-pp4-6000d +version: 1.0.5 # semver, raised on every change +releaseDate: 2026-10-12 +servedName: kimi-k3 +engine: sglang +chart: { name: sglang, version: ">=0.7.0" } +requires: { gpus: 8, nodes: 4, topology: lws, gpuProduct: [NVIDIA-RTX-6000D] } +values: { ... } +``` + +```yaml +# models/kimi-k3/metadata.yaml +apiVersion: catalog.swiss/v2 +name: kimi-k3 +source: { hf: moonshotai/Kimi-K3 } +defaultVariant: sglang-tp8-b300-optimized # replaces a variant's default: true +tuning: + - version: 1.0.4 # legacy: the v1 version measured, kept for index.json + baseline: sglang-tp8-b300-baseline@1.0.5 # @ measured + optimized: sglang-tp8-b300-optimized@1.0.5 + uplift: 57.6 + report: kimi-k3-1.0.4-b300-report.html +``` + +- A deploy pins commit, path and sha256; `history.json` maps each version to its commit. +- Published: `index-v2.json` (`schema/index-v2.schema.json`) and `models//.history.json`. +- Migrate a model: `node hack/migrate-v2.js [--dry-run]`. + ## Development | command | does | @@ -121,7 +164,7 @@ linking it. The images it records must be the two variants' own (registry, repos | `npm run validate` | `validate:schema`, `validate:helm` and `validate:images` | | `npm run dev:site` | the site at , live from `models/` | | `npm run build:site` | the site and `index.json` into `site/` (`PUBLISHED_INDEX=none` offline); `npm run preview:site` serves it | -| `npm run serve` | the catalog server: `index.json`, the version files, reports and `catalog.json`, live from `models/` | +| `npm run serve` | the catalog server: `index.json`, `index-v2.json`, the version and variant files, reports and `catalog.json`, live from `models/` | `validate:helm` and `validate:images` check every version file; pass files, or `--base REF` for what a branch changes, to narrow them. diff --git a/catalog.yaml b/catalog.yaml index 7c820f4..598d92b 100644 --- a/catalog.yaml +++ b/catalog.yaml @@ -9,3 +9,6 @@ apiVersion: catalog.swiss/v1 # itself: a version file is immutable once published, and would have to name # a page that does not exist until the site is deployed. site: https://modelsphere.github.io/model-catalog/ + +# Where old v2 revisions are fetched from, raw at a commit. +repo: https://github.com/modelsphere/model-catalog diff --git a/hack/lib/catalog.js b/hack/lib/catalog.js index cc78c7e..dba80e6 100644 --- a/hack/lib/catalog.js +++ b/hack/lib/catalog.js @@ -5,11 +5,16 @@ const crypto = require("crypto"); const fs = require("fs"); const path = require("path"); +const { isDeepStrictEqual } = require("util"); const yaml = require("yaml"); +const { fileHistory } = require("./history"); const root = path.resolve(__dirname, "../.."); const modelsDir = path.join(root, "models"); +const V1 = "catalog.swiss/v1"; +const V2 = "catalog.swiss/v2"; + function readYaml(p) { return yaml.parse(fs.readFileSync(p, "utf8")); } @@ -22,14 +27,47 @@ function modelNames(dir = modelsDir) { .sort(); } -// A model's version files (-.yaml), as repo-relative paths. -// metadata.yaml has no "-", which is what tells the two apart. -function versionFiles(name, dir = modelsDir) { +// "version" (v1), "variant" (v2) or null, by content, never by file name. +function kindOf(doc) { + if (!doc || typeof doc !== "object") return null; + if (doc.apiVersion === V1 && doc.kind === undefined) return "version"; + if (doc.apiVersion === V2 && doc.kind === "Variant") return "variant"; + return null; +} + +// A model's YAML files but metadata.yaml, each with its kind. +function modelFiles(name, dir = modelsDir) { return fs - .readdirSync(path.join(dir, name)) - .filter((f) => f.endsWith(".yaml") && f.includes("-")) + .readdirSync(path.join(dir, name), { withFileTypes: true }) + .filter((e) => e.isFile() && e.name.endsWith(".yaml") && e.name !== "metadata.yaml") + .map((e) => e.name) .sort() - .map((f) => `models/${name}/${f}`); + .map((f) => { + const rel = `models/${name}/${f}`; + try { + return { rel, kind: kindOf(yaml.parse(fs.readFileSync(path.join(dir, name, f), "utf8"))) }; + } catch (error) { + return { rel, kind: null, error }; + } + }); +} + +// A model's v1 version files (-.yaml), as repo-relative paths. +function versionFiles(name, dir = modelsDir) { + return modelFiles(name, dir) + .filter((f) => f.kind === "version") + .map((f) => f.rel); +} + +// A model's v2 variant files (.yaml). +function variantFiles(name, dir = modelsDir) { + return modelFiles(name, dir) + .filter((f) => f.kind === "variant") + .map((f) => f.rel); +} + +function schemaOf(meta) { + return meta?.apiVersion === V2 ? "v2" : "v1"; } function digest(buf) { @@ -86,7 +124,7 @@ function prune(value) { // index.json: the published surface. Each version carries a sha256 of its // file -- the lock a deploy records -- and metadata.yaml is inlined, since a -// consumer never fetches it. +// consumer never fetches it. A v2 model keeps only its tuning with a v1 version. function buildIndex() { const models = []; for (const name of modelNames()) { @@ -123,7 +161,7 @@ function buildIndex() { family: meta.family, tags: meta.tags, deprecated: meta.deprecated, - tuning: meta.tuning, + tuning: schemaOf(meta) === "v1" ? meta.tuning : legacyTuning(meta.tuning), source: { hf: source.hf, revision: source.revision, sizeGiB: source.sizeGiB }, latest: latestOf(versions.map((v) => v.version)), versions: versions.reverse().map(({ version, path: p, digest: d, variants }) => ({ @@ -137,23 +175,257 @@ function buildIndex() { } models.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); const site = readYaml(path.join(root, "catalog.yaml"))?.site; - return prune({ apiVersion: "catalog.swiss/v1", site, count: models.length, models }); + return prune({ apiVersion: V1, site, count: models.length, models }); +} + +function legacyTuning(tuning) { + const out = (Array.isArray(tuning) ? tuning : []) + .filter((t) => t?.version != null) + .map((t) => ({ ...t, baseline: String(t.baseline).split("@")[0], optimized: String(t.optimized).split("@")[0] })); + return out.length ? out : undefined; } function serializeIndex(index) { return JSON.stringify(index, null, 2) + "\n"; } +// variants[].link is only schema-checked as a URI, which admits javascript:. +function safeLink(link) { + return typeof link === "string" && /^https?:\/\//i.test(link) ? link : undefined; +} + +const historyPath = (rel) => rel.replace(/\.yaml$/, ".history.json"); + +// Reports the site may link; validate:schema fails on the others. +function linkableReports(base, name, warn) { + const { reportProblem } = require("./report"); + const dir = path.join(base, "models", name); + return fs + .readdirSync(dir) + .filter((f) => f.endsWith(".html")) + .sort() + .filter((f) => { + const problem = reportProblem(path.join(dir, f)); + if (problem) warn(`models/${name}/${f}: ${problem} -- not linked`); + return !problem; + }); +} + +// Every v2 variant's revisions, oldest first, by path, deleted ones included. +// An uncommitted edit is a revision with no commit. +function variantRevisions(files, base) { + const out = new Map(); + for (const [rel, changes] of files) { + const revisions = changes + .map((c) => ({ c, doc: c.bytes && parseOrNull(c.bytes) })) + .filter(({ doc }) => kindOf(doc) === "variant") + .map(({ c, doc }) => ({ doc, commit: c.commit, subject: c.subject, digest: digest(c.bytes) })); + if (revisions.length) out.set(rel, { revisions, gone: changes.at(-1).deleted ? changes.at(-1) : null }); + } + for (const name of modelNames(path.join(base, "models"))) { + for (const rel of variantFiles(name, path.join(base, "models"))) { + const body = fs.readFileSync(path.join(base, rel)); + const h = out.get(rel) ?? { revisions: [], gone: null }; + out.set(rel, h); + if (!h.gone && h.revisions.at(-1)?.digest === digest(body)) continue; + h.revisions.push({ doc: yaml.parse(body.toString("utf8")), digest: digest(body), subject: "(uncommitted)" }); + h.gone = null; + } + } + return out; +} + +function historyDoc(rel, { revisions, gone }) { + const [, model, file] = rel.split("/"); + return prune({ + model, + id: file.replace(/\.yaml$/, ""), + path: rel, + revisions: revisions + .map((r) => ({ version: r.doc.version, releaseDate: r.doc.releaseDate, commit: r.commit, digest: r.digest, subject: r.subject })) + .reverse(), + deleted: gone && { commit: gone.commit, subject: gone.subject }, + }); +} + +function historiesOf({ root: base = root } = {}) { + return new Map([...variantRevisions(fileHistory(base).files, base)].map(([rel, h]) => [rel, historyDoc(rel, h)])); +} + +// Each change to a variant file raises its version, and changes more than that. +function revisionProblems({ root: base = root } = {}) { + const strip = ({ version, releaseDate, ...rest }) => rest; + const out = []; + for (const [rel, { revisions }] of variantRevisions(fileHistory(base).files, base)) { + revisions.forEach((r, i) => { + const prev = revisions[i - 1]; + if (!prev) return; + const at = r.commit ? r.commit.slice(0, 7) : "working copy"; + if (!(compareVersions(String(r.doc.version), String(prev.doc.version)) > 0)) { + out.push([rel, `${at}: changed without raising version above ${prev.doc.version}`]); + } else if (isDeepStrictEqual(strip(r.doc), strip(prev.doc))) { + out.push([rel, `${at}: version ${r.doc.version} changes nothing else`]); + } + }); + } + return out; +} + +function parseOrNull(bytes) { + try { + return yaml.parse(bytes.toString("utf8")); + } catch { + return null; + } +} + +function rawBase(settings) { + if (settings.raw) return settings.raw; + const m = /^https:\/\/github\.com\/([^/]+\/[^/]+?)\/?$/.exec(settings.repo ?? ""); + return m ? `https://raw.githubusercontent.com/${m[1]}/` : undefined; +} + +// index-v2.json: v2 models and their variant files only; and the histories. +function buildIndexV2({ root: base = root, warn = console.warn } = {}) { + const dir = path.join(base, "models"); + const settings = readYaml(path.join(base, "catalog.yaml")) ?? {}; + const { files, last } = fileHistory(base); + const histories = new Map([...variantRevisions(files, base)].map(([rel, h]) => [rel, historyDoc(rel, h)])); + const models = []; + for (const name of modelNames(dir)) { + const metaPath = path.join(dir, name, "metadata.yaml"); + const meta = fs.existsSync(metaPath) ? readYaml(metaPath) : null; + if (schemaOf(meta) !== "v2") continue; + const variants = v2Variants(base, name, histories); + if (!variants.length) continue; + const reports = linkableReports(base, name, warn); + // The legacy version is index.json's. + const tuning = (Array.isArray(meta.tuning) ? meta.tuning : []).map(({ version, ...t }) => { + if (!t.report || reports.includes(t.report)) return t; + if (!fs.existsSync(path.join(dir, name, t.report))) warn(`models/${name}/${t.report}: missing -- not linked`); + return { ...t, report: undefined }; + }); + models.push( + prune({ + name, + displayName: meta.displayName, + description: meta.description, + family: meta.family, + series: meta.series, + tags: meta.tags, + deprecated: meta.deprecated, + source: meta.source && { hf: meta.source.hf, revision: meta.source.revision, sizeGiB: meta.source.sizeGiB }, + quantization: meta.quantization, + defaultVariant: meta.defaultVariant, + reports, + tuning, + variants, + }) + ); + } + const index = prune({ + apiVersion: V2, + site: settings.site, + repo: settings.repo, + raw: rawBase(settings), + commit: last, + count: models.length, + models, + }); + return { index, histories }; +} + +function summary(v) { + return { + engine: v.engine, + description: v.description, + link: safeLink(v.link), + chart: v.chart, + image: v.image, + requires: v.requires, + }; +} + +function v2Variants(base, name, histories) { + return variantFiles(name, path.join(base, "models")).map((rel) => { + const body = fs.readFileSync(path.join(base, rel)); + const doc = yaml.parse(body.toString("utf8")); + const top = histories.get(rel).revisions[0]; + return { + id: doc.id, + version: doc.version, + releaseDate: doc.releaseDate, + commit: top.commit, + path: rel, + digest: digest(body), + history: historyPath(rel), + servedName: doc.servedName, + ...summary(doc), + deprecated: doc.deprecated, + replaces: doc.replaces, + }; + }); +} + +// Validators for schema/*.schema.json, by file name; they $ref each other. +function schemaValidator() { + const Ajv2020 = require("ajv/dist/2020"); + const addFormats = require("ajv-formats"); + const ajv = new Ajv2020({ allErrors: true, strict: true, strictTypes: false, verbose: true }); + addFormats(ajv); + const dir = path.join(root, "schema"); + for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".schema.json"))) { + ajv.addSchema(JSON.parse(fs.readFileSync(path.join(dir, f), "utf8")), f); + } + return (file) => { + const validate = ajv.getSchema(file); + if (!validate) throw new Error(`schema/${file}: no such schema`); + return validate; + }; +} + +// The index matches its schema and each variant its newest revision. +function checkIndexV2({ index, histories }) { + const validate = schemaValidator()("index-v2.schema.json"); + const problems = []; + if (!validate(index)) { + problems.push(...validate.errors.map((e) => `index-v2.json${e.instancePath}: ${e.message}`)); + } + for (const m of index.models ?? []) { + for (const v of m.variants ?? []) { + if (!v.history) continue; + const top = histories.get(v.path)?.revisions[0]; + if (!top || top.version !== v.version || top.commit !== v.commit || top.digest !== v.digest) { + problems.push(`${v.history}: its newest revision is not ${m.name}/${v.id} as index-v2.json lists it`); + } + } + } + return problems; +} + module.exports = { root, modelsDir, + V1, + V2, readYaml, modelNames, + kindOf, + modelFiles, versionFiles, + variantFiles, + schemaOf, digest, compareVersions, isPrerelease, latestOf, buildIndex, serializeIndex, + legacyTuning, + historyPath, + historiesOf, + revisionProblems, + buildIndexV2, + schemaValidator, + checkIndexV2, }; diff --git a/hack/lib/catalog.test.js b/hack/lib/catalog.test.js index 5a0aa3a..b1ed06c 100644 --- a/hack/lib/catalog.test.js +++ b/hack/lib/catalog.test.js @@ -33,3 +33,125 @@ test("the index locks every version to its file's bytes", () => { } } }); + +const os = require("node:os"); +const { execFileSync } = require("node:child_process"); +const YAML = require("yaml"); +const { buildIndexV2, checkIndexV2, modelFiles, revisionProblems, versionFiles, variantFiles } = require("./catalog"); + +// A git repo; each step writes files (null deletes) and commits at a UTC time. +function fixture(steps) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "catalog-")); + const git = (args, at) => + execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.com", ...args], { + cwd: dir, + env: { ...process.env, GIT_AUTHOR_DATE: at, GIT_COMMITTER_DATE: at }, + stdio: "pipe", + }); + git(["init", "-q", "-b", "master"]); + const write = (files) => { + for (const [rel, body] of Object.entries(files)) { + const file = path.join(dir, rel); + if (body === null) fs.rmSync(file); + else { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, typeof body === "string" ? body : YAML.stringify(body)); + } + } + }; + const commit = (at, subject, files) => { + write(files); + git(["add", "-A"], at); + git(["commit", "-q", "-m", subject], at); + }; + for (const step of steps) commit(...step); + return { dir, write, commit, head: () => git(["rev-parse", "HEAD"]).toString().trim(), sha: (n) => git(["rev-parse", `HEAD~${n}`]).toString().trim() }; +} + +const variant = (id, version = "1.0.0", gpus = 8) => ({ + apiVersion: "catalog.swiss/v2", + kind: "Variant", + model: "m2", + id, + version, + releaseDate: "2026-10-01", + engine: "sglang", + chart: { name: "sglang", version: ">=0.7.0" }, + requires: { gpus }, +}); + +const catalogFiles = { + "catalog.yaml": { apiVersion: "catalog.swiss/v1", site: "https://example.com/c/", repo: "https://github.com/o/c" }, + "models/m1/metadata.yaml": { apiVersion: "catalog.swiss/v1", name: "m1", source: { hf: "o/m1" } }, + "models/m1/m1-1.0.0.yaml": { + apiVersion: "catalog.swiss/v1", + name: "m1", + version: "1.0.0", + variants: [{ id: "a", engine: "vllm", chart: { name: "vllm", version: "0.1.0" }, requires: { gpus: 1 } }, { id: "b", default: true, engine: "vllm", chart: { name: "vllm", version: "0.1.0" }, requires: { gpus: 2 } }], + }, + "models/m2/metadata.yaml": { apiVersion: "catalog.swiss/v2", name: "m2", source: { hf: "o/m2" }, defaultVariant: "sglang-tp8" }, + "models/m2/sglang-tp8.yaml": variant("sglang-tp8"), + "models/m2/sglang-tp4.yaml": variant("sglang-tp4", "1.0.0", 4), +}; + +test("a file is a v1 version or a v2 variant by its content, never its name", () => { + const { dir } = fixture([["2026-10-01T10:00:00Z", "init", catalogFiles]]); + const models = path.join(dir, "models"); + assert.deepEqual(versionFiles("m1", models), ["models/m1/m1-1.0.0.yaml"]); + assert.deepEqual(variantFiles("m2", models), ["models/m2/sglang-tp4.yaml", "models/m2/sglang-tp8.yaml"]); + fs.writeFileSync(path.join(models, "m2/odd-1.0.0.yaml"), "apiVersion: catalog.swiss/v3\n"); + fs.writeFileSync(path.join(models, "m2/broken.yaml"), "a: [\n"); + const odd = modelFiles("m2", models).filter((f) => !f.kind); + assert.deepEqual(odd.map((f) => [f.rel, !!f.error]), [["models/m2/broken.yaml", true], ["models/m2/odd-1.0.0.yaml", false]]); +}); + +test("index-v2 lists only v2 models, each variant file at its latest version", () => { + const repo = fixture([ + ["2026-10-01T10:00:00Z", "init", catalogFiles], + ["2026-10-02T08:00:00Z", "tp8: one", { "models/m2/sglang-tp8.yaml": variant("sglang-tp8", "1.0.1", 7) }], + ["2026-10-02T09:00:00Z", "tp8: two", { "models/m2/sglang-tp8.yaml": variant("sglang-tp8", "1.1.0", 6) }], + ["2026-10-03T09:00:00Z", "drop tp4", { "models/m2/sglang-tp4.yaml": null }], + ]); + const built = buildIndexV2({ root: repo.dir, warn: () => {} }); + const { index, histories } = built; + assert.deepEqual(checkIndexV2(built), []); + assert.equal(index.commit, repo.head()); + assert.equal(index.raw, "https://raw.githubusercontent.com/o/c/"); + assert.deepEqual(index.models.map((m) => m.name), ["m2"]); + const [m2] = index.models; + assert.deepEqual(m2.variants.map((v) => [v.id, v.version, v.commit, v.history]), [["sglang-tp8", "1.1.0", repo.sha(1), "models/m2/sglang-tp8.history.json"]]); + const tp8 = histories.get("models/m2/sglang-tp8.yaml"); + assert.deepEqual(tp8.revisions.map((r) => [r.version, r.subject]), [["1.1.0", "tp8: two"], ["1.0.1", "tp8: one"], ["1.0.0", "init"]]); + assert.equal(tp8.revisions[0].digest, m2.variants[0].digest); + const tp4 = histories.get("models/m2/sglang-tp4.yaml"); + assert.deepEqual([tp4.revisions.map((r) => r.version), tp4.deleted.subject], [["1.0.0"], "drop tp4"]); + assert.deepEqual(revisionProblems({ root: repo.dir }), []); +}); + +test("a change must raise the version, and a raised version must change something", () => { + const repo = fixture([["2026-10-01T10:00:00Z", "init", catalogFiles]]); + repo.write({ "models/m2/sglang-tp8.yaml": variant("sglang-tp8", "1.0.0", 7) }); + assert.deepEqual(revisionProblems({ root: repo.dir }), [["models/m2/sglang-tp8.yaml", "working copy: changed without raising version above 1.0.0"]]); + repo.write({ "models/m2/sglang-tp8.yaml": variant("sglang-tp8", "1.0.1") }); + assert.deepEqual(revisionProblems({ root: repo.dir }), [["models/m2/sglang-tp8.yaml", "working copy: version 1.0.1 changes nothing else"]]); + repo.write({ "models/m2/sglang-tp8.yaml": variant("sglang-tp8", "1.0.1", 7) }); + const { index } = buildIndexV2({ root: repo.dir, warn: () => {} }); + assert.deepEqual([index.models[0].variants[1].version, index.models[0].variants[1].commit], ["1.0.1", undefined]); + assert.deepEqual(revisionProblems({ root: repo.dir }), []); +}); + +test("a shallow clone is refused: its versions would be wrong", () => { + const repo = fixture([ + ["2026-10-01T10:00:00Z", "init", catalogFiles], + ["2026-10-02T10:00:00Z", "edit", { "models/m2/sglang-tp8.yaml": variant("sglang-tp8", "1.0.1", 7) }], + ]); + const clone = fs.mkdtempSync(path.join(os.tmpdir(), "catalog-shallow-")); + execFileSync("git", ["clone", "-q", "--depth=1", `file://${repo.dir}`, clone]); + assert.throws(() => buildIndexV2({ root: clone, warn: () => {} }), /shallow clone/); +}); + +test("the catalog's own index-v2.json passes its checks", () => { + const built = buildIndexV2({ warn: () => {} }); + assert.deepEqual(checkIndexV2(built), []); + assert.ok(built.index.models.every((m) => m.variants.every((v) => v.history))); +}); diff --git a/hack/lib/helm-validation.js b/hack/lib/helm-validation.js index 7594d3c..9de84fd 100644 --- a/hack/lib/helm-validation.js +++ b/hack/lib/helm-validation.js @@ -4,7 +4,7 @@ const fs = require("node:fs"); const path = require("node:path"); const {spawnSync} = require("node:child_process"); const YAML = require("yaml"); -const {modelNames, versionFiles} = require("./catalog"); +const {modelNames, versionFiles, variantFiles, kindOf} = require("./catalog"); function affectsValidator(file) { return file.startsWith("schema/crds/") || @@ -31,14 +31,16 @@ function git(root, args) { // Keep both sides of renames for validator-change detection. NUL delimiters // preserve filenames with whitespace; deleted versions are not selected. // `added` lists the new versions, held to the chart minimum: a published version -// keeps the chart it was published with. Explicit files are checked as new. +// keeps the chart it was published with. A changed v2 variant counts as new. +// Explicit files are checked as new. function selectFiles(root, {all = false, base = "origin/HEAD", files = []} = {}) { const models = path.join(root, "models"); - const versions = modelNames(models).flatMap((name) => versionFiles(name, models)); - if (all) return {files: versions, added: [], reason: "every version file"}; + const variants = modelNames(models).flatMap((name) => variantFiles(name, models)); + const versions = [...modelNames(models).flatMap((name) => versionFiles(name, models)), ...variants].sort(); + if (all) return {files: versions, added: [], reason: "every version and variant file"}; if (files.length) { const selected = [...new Set(files.map((file) => path.relative(root, path.resolve(root, file)).split(path.sep).join("/")))].sort(); - if (selected.some((file) => !versions.includes(file))) throw new Error("explicit files must be existing model version YAMLs under models/"); + if (selected.some((file) => !versions.includes(file))) throw new Error("explicit files must be existing model version or variant YAMLs under models/"); return {files: selected, added: selected, reason: "explicit files"}; } const mergeBase = git(root, ["merge-base", base, "HEAD"]).trim(); @@ -49,7 +51,7 @@ function selectFiles(root, {all = false, base = "origin/HEAD", files = []} = {}) const status = tokens[i++]; changed.push(tokens[i++]); if (/^[RC]/.test(status)) changed.push(tokens[i++]); - if (/^[ARC]/.test(status)) added.push(changed.at(-1)); + if (/^[ARC]/.test(status) || (status === "M" && variants.includes(changed.at(-1)))) added.push(changed.at(-1)); } const among = (list) => versions.filter((file) => list.includes(file)); if (changed.some(affectsValidator)) { @@ -85,7 +87,9 @@ function failureLines(result) { // validate:schema owns a version file's shape; this only refuses to pass a file // with nothing to render. function parseVersions(file) { - const variants = YAML.parse(fs.readFileSync(file, "utf8"))?.variants; + const doc = YAML.parse(fs.readFileSync(file, "utf8")); + if (kindOf(doc) === "variant") return [doc]; + const variants = doc?.variants; if (!Array.isArray(variants) || !variants.length) throw new Error("variants: expected a nonempty array"); return variants; } diff --git a/hack/lib/helm-validation.test.js b/hack/lib/helm-validation.test.js index e9c8e6e..15ed0e9 100644 --- a/hack/lib/helm-validation.test.js +++ b/hack/lib/helm-validation.test.js @@ -9,6 +9,10 @@ const {execFileSync} = require("node:child_process"); const {selectFiles, kubeconformArgs, failureLines} = require("./helm-validation"); const config = require("../../schema/crds/config.json"); +const VERSION = "apiVersion: catalog.swiss/v1\n"; +const MODIFIED = "apiVersion: catalog.swiss/v1\nversion: modified\n"; +const VARIANT = "apiVersion: catalog.swiss/v2\nkind: Variant\n"; + function repository(t) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "helm-selection-")); t.after(() => fs.rmSync(dir, {recursive: true, force: true})); @@ -18,13 +22,13 @@ function repository(t) { git("config", "user.email", "validator@example.invalid"); // The fixture's commits are throwaway: never ask the developer's gpg. git("config", "commit.gpgsign", "false"); - function write(file, body = "test\n") { + function write(file, body = file.endsWith(".yaml") ? VERSION : "test\n") { fs.mkdirSync(path.dirname(path.join(dir, file)), {recursive: true}); fs.writeFileSync(path.join(dir, file), body); } function commit() { git("add", "."); git("commit", "-m", "fixture"); } write("models/example/example-1.0.0.yaml"); - write("models/example/metadata.yaml"); + write("models/example/metadata.yaml", "test\n"); write("hack/validate-helm.js"); commit(); git("branch", "base"); @@ -42,7 +46,7 @@ test("add, modify and rename destinations are selected; deletions are excluded", r.write("models/example/example-1.1.0.yaml"); r.commit(); r.git("branch", "before"); r.git("mv", "models/example/example-1.0.0.yaml", "models/example/renamed version-1.0.0.yaml"); - r.write("models/example/example-1.1.0.yaml", "modified\n"); + r.write("models/example/example-1.1.0.yaml", MODIFIED); r.write("models/example/example-1.2.0.yaml"); r.commit(); assert.deepEqual(selectFiles(r.dir, {base: "before"}).files, [ "models/example/example-1.1.0.yaml", "models/example/example-1.2.0.yaml", "models/example/renamed version-1.0.0.yaml"]); @@ -74,7 +78,7 @@ test("validator, dependency and schema changes, including deletions, trigger all test("only added versions are held to the chart minimum, also in a full scan", (t) => { const r = repository(t); - r.write("models/example/example-1.0.0.yaml", "modified\n"); + r.write("models/example/example-1.0.0.yaml", MODIFIED); r.write("models/example/example-1.1.0.yaml"); r.write("hack/validate-helm.js", "changed\n"); r.commit(); const selection = selectFiles(r.dir, {base: "base"}); @@ -84,6 +88,20 @@ test("only added versions are held to the chart minimum, also in a full scan", ( assert.deepEqual(selectFiles(r.dir, {files: ["models/example/example-1.0.0.yaml"]}).added, ["models/example/example-1.0.0.yaml"]); }); +test("a v2 variant file is selected by its content, and any change to one is held to the chart minimum", (t) => { + const r = repository(t); + r.write("models/example/sglang-tp8.yaml", VARIANT); + r.write("models/example/vllm-tp8.yaml", VARIANT); r.commit(); + r.git("branch", "before"); + r.write("models/example/sglang-tp8.yaml", `${VARIANT}id: sglang-tp8\n`); + r.write("models/example/sglang-tp4.yaml", VARIANT); r.commit(); + const selection = selectFiles(r.dir, {base: "before"}); + assert.deepEqual(selection.files, ["models/example/sglang-tp4.yaml", "models/example/sglang-tp8.yaml"]); + assert.deepEqual(selection.added, ["models/example/sglang-tp4.yaml", "models/example/sglang-tp8.yaml"]); + assert.deepEqual(selectFiles(r.dir, {all: true}).files, [ + "models/example/example-1.0.0.yaml", "models/example/sglang-tp4.yaml", "models/example/sglang-tp8.yaml", "models/example/vllm-tp8.yaml"]); +}); + test("unknown base fails rather than reporting no changes", (t) => { const r = repository(t); assert.throws(() => selectFiles(r.dir, {base: "missing-ref"})); diff --git a/hack/lib/history.js b/hack/lib/history.js new file mode 100644 index 0000000..b201c20 --- /dev/null +++ b/hack/lib/history.js @@ -0,0 +1,72 @@ +"use strict"; +// The git history v2's date versions and history.json are derived from. + +const { spawnSync } = require("child_process"); + +function git(root, args, input) { + const r = spawnSync("git", args, { cwd: root, input, maxBuffer: 1 << 30 }); + if (r.error) throw new Error(`git: ${r.error.message} -- date versions are derived from git history`); + if (r.status !== 0) throw new Error(`git ${args[0]}: ${r.stderr.toString().trim()}`); + return r.stdout; +} + +function blobs(root, specs) { + if (specs.length === 0) return []; + const out = git(root, ["cat-file", "--batch"], specs.join("\n") + "\n"); + const bodies = []; + let at = 0; + for (const spec of specs) { + const nl = out.indexOf(10, at); + const header = out.toString("utf8", at, nl); + const size = Number(header.split(" ")[2]); + if (!/ blob \d+$/.test(header)) throw new Error(`git cat-file ${spec}: ${header}`); + bodies.push(out.subarray(nl + 1, nl + 1 + size)); + at = nl + 1 + size + 1; + } + return bodies; +} + +// Changes to models//*.yaml on HEAD's first-parent chain, oldest first: +// path -> [{commit, subject, deleted, bytes}]. last is the last commit touching +// models/ or catalog.yaml. +function fileHistory(root) { + if (git(root, ["rev-parse", "--is-shallow-repository"]).toString().trim() === "true") { + throw new Error("a shallow clone: date versions need the full history (actions/checkout with fetch-depth: 0, or git fetch --unshallow)"); + } + const files = new Map(); + if (spawnSync("git", ["rev-parse", "--verify", "--quiet", "HEAD"], { cwd: root }).status !== 0) return { files, last: null }; + const log = git(root, [ + "log", + "--reverse", + "--first-parent", + "--diff-merges=first-parent", + "--no-renames", + "--name-status", + "-z", + "--format=%x1e%H%x1f%s", + "--", + "models", + "catalog.yaml", + ]).toString("utf8"); + let last = null; + const pending = []; + for (const record of log.split("\x1e").slice(1)) { + const end = record.indexOf("\0"); + const [commit, subject] = record.slice(0, end).split("\x1f"); + last = commit; + const fields = record.slice(end + 1).replace(/^\n/, "").split("\0"); + for (let i = 0; i + 1 < fields.length; i += 2) { + const [status, file] = [fields[i], fields[i + 1]]; + if (!/^models\/[^/]+\/[^/]+\.yaml$/.test(file) || file.endsWith("/metadata.yaml")) continue; + const change = { commit, subject, deleted: status === "D", bytes: null }; + if (!files.has(file)) files.set(file, []); + files.get(file).push(change); + if (!change.deleted) pending.push([change, `${commit}:${file}`]); + } + } + const bodies = blobs(root, pending.map(([, spec]) => spec)); + pending.forEach(([change], i) => (change.bytes = bodies[i])); + return { files, last }; +} + +module.exports = { fileHistory }; diff --git a/hack/lib/images.js b/hack/lib/images.js index 849ced8..6d47951 100644 --- a/hack/lib/images.js +++ b/hack/lib/images.js @@ -11,7 +11,7 @@ const path = require("path"); const { promisify } = require("util"); const ipaddr = require("ipaddr.js"); const { command } = require("./helm-validation"); -const { root, modelNames, versionFiles } = require("./catalog"); +const { root, modelNames, versionFiles, variantFiles } = require("./catalog"); const { jsonBlocks } = require("./report"); // A repository the way containerd reads it: no registry host (or docker.io) @@ -185,7 +185,7 @@ function changedVersionFiles(root, base) { const changed = git(["diff", "--name-only", "-z", "--diff-filter=AMR", mergeBase, "HEAD"]).split("\0").filter(Boolean); const touched = new Set(changed.filter((f) => /^models\/[^/]+\/(metadata\.yaml|[^/]+\.html)$/.test(f)).map((f) => f.split("/")[1])); const models = path.join(root, "models"); - const versions = modelNames(models).flatMap((name) => versionFiles(name, models)); + const versions = modelNames(models).flatMap((name) => [...versionFiles(name, models), ...variantFiles(name, models)]); return versions.filter((file) => changed.includes(file) || touched.has(file.split("/")[1])); } diff --git a/hack/migrate-v2.js b/hack/migrate-v2.js new file mode 100644 index 0000000..dee00b3 --- /dev/null +++ b/hack/migrate-v2.js @@ -0,0 +1,123 @@ +#!/usr/bin/env node +// Move a model to schema v2, from its latest v1 version file. v1 files stay. +// +// node hack/migrate-v2.js ... [--dry-run] +"use strict" + +const fs = require("fs") +const os = require("os") +const path = require("path") +const {parseArgs, isDeepStrictEqual} = require("util") +const {spawnSync} = require("child_process") +const YAML = require("yaml") +const {root, modelsDir, readYaml, versionFiles, variantFiles, compareVersions, latestOf, schemaOf, V2} = require("./lib/catalog") + +const STRINGIFY = {lineWidth: 0} + +function migrate(name, {releaseDate = new Date().toISOString().slice(0, 10)} = {}) { + const dir = path.join(modelsDir, name) + const metaPath = path.join(dir, "metadata.yaml") + if (!fs.existsSync(metaPath)) throw new Error(`models/${name}/: no such model`) + const meta = YAML.parseDocument(fs.readFileSync(metaPath, "utf8")) + if (schemaOf(meta.toJS()) === "v2") throw new Error(`${name}: already on v2`) + if (variantFiles(name).length) throw new Error(`${name}: already has v2 variant files`) + + const docs = new Map(versionFiles(name).map((rel) => [rel, readYaml(path.join(root, rel))])) + if (docs.size === 0) throw new Error(`${name}: no version files`) + const byVersion = new Map([...docs.values()].map((d) => [String(d.version), d])) + const latest = latestOf([...byVersion.keys()].sort(compareVersions)) + const from = [...docs].find(([, d]) => String(d.version) === latest)[0] + const text = fs.readFileSync(path.join(root, from), "utf8") + const version = YAML.parseDocument(text) + const variants = version.get("variants", true).items + const servedName = version.get("servedName") + + const heading = version.contents.items[0]?.key.commentBefore + const out = new Map() + const taken = new Set(fs.readdirSync(dir)) + for (const item of variants) { + const id = item.get("id") + if (id === "metadata") throw new Error(`${name}: variant id metadata is the model's own file; rename the variant first`) + if (taken.has(`${id}.yaml`)) throw new Error(`${name}: models/${name}/${id}.yaml exists already`) + const doc = new YAML.Document({apiVersion: V2, kind: "Variant", model: name}) + doc.contents.items[0].key.commentBefore = [heading, item.commentBefore].filter(Boolean).join("\n") || undefined + // Moved nodes keep their comments and quoting. + const own = item.items.filter((p) => p.key.value !== "default") + doc.contents.items.push(own.find((p) => p.key.value === "id")) + doc.set("version", latest) + doc.set("releaseDate", releaseDate) + if (servedName != null) doc.set("servedName", servedName) + doc.contents.items.push(...own.filter((p) => p.key.value !== "id")) + const body = doc.toString(STRINGIFY) + const {apiVersion, kind, model, version: v, releaseDate: r, servedName: served, ...rest} = YAML.parse(body) + const {default: _, ...was} = item.toJSON() + if (!isDeepStrictEqual(rest, was) || served !== (servedName ?? undefined)) throw new Error(`${name}: ${id} would not migrate unchanged`) + out.set(`models/${name}/${id}.yaml`, body) + } + + // A tuning entry carries over only if its variants are unchanged since measured. + const plain = (v) => byVersion.get(latest).variants.find((x) => x.id === v) + meta.set("apiVersion", V2) + const def = byVersion.get(latest).variants.find((v) => v.default === true) + if (def) { + const at = meta.contents.items.findIndex((p) => ["tuning", "source"].includes(p.key.value)) + const pair = meta.createPair("defaultVariant", def.id) + if (at < 0) meta.contents.items.push(pair) + else meta.contents.items.splice(at, 0, pair) + } + const tuning = meta.get("tuning", true) + for (const [i, entry] of (tuning?.items ?? []).entries()) { + const measured = byVersion.get(String(entry.get("version"))) + for (const key of ["baseline", "optimized"]) { + const id = entry.get(key) + const was = measured?.variants.find((v) => v.id === id) + if (!was || !isDeepStrictEqual(was, plain(id))) { + throw new Error( + `${name}: tuning[${i}] measured ${id} in ${entry.get("version")}, and ${latest} ${plain(id) ? "changed" : "dropped"} it: ` + + "v2 has no revision of what was measured. Re-measure the latest, or remove the entry, then migrate" + ) + } + entry.set(key, `${id}@${latest}`) + } + } + out.set(`models/${name}/metadata.yaml`, meta.toString(STRINGIFY)) + return {from, files: out} +} + +function main() { + const {values, positionals} = parseArgs({allowPositionals: true, options: {"dry-run": {type: "boolean", default: false}}}) + if (positionals.length === 0) throw new Error("usage: node hack/migrate-v2.js [...] [--dry-run]") + const plans = positionals.map((name) => ({name, ...migrate(name)})) + if (values["dry-run"]) { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "migrate-v2-")) + for (const {files} of plans) { + for (const [rel, body] of files) { + const now = path.join(tmp, rel) + fs.mkdirSync(path.dirname(now), {recursive: true}) + fs.writeFileSync(now, body) + const was = path.join(root, rel) + const r = spawnSync("git", ["diff", "--no-index", "--", fs.existsSync(was) ? was : "/dev/null", now], {encoding: "utf8"}) + process.stdout.write(r.stdout.replaceAll(`${tmp.slice(1)}/`, "").replaceAll(`${root.slice(1)}/`, "")) + } + } + fs.rmSync(tmp, {recursive: true}) + } else { + for (const {files} of plans) for (const [rel, body] of files) fs.writeFileSync(path.join(root, rel), body) + } + for (const {name, from, files} of plans) { + console.error(`${name}: ${files.size - 1} variant files from ${path.basename(from)}${values["dry-run"] ? " (dry run: nothing written)" : ""}`) + } + const lineage = plans.map(({name, from}) => `${name}@${readYaml(path.join(root, from)).version}`).join(", ") + if (!values["dry-run"]) console.error(`commit as one change, e.g.: git commit -m "migrate ${plans.map((p) => p.name).join(", ")} to schema v2 (from ${lineage})"`) +} + +if (require.main === module) { + try { + main() + } catch (err) { + console.error(err.message) + process.exit(1) + } +} + +module.exports = {migrate} diff --git a/hack/migrate-v2.test.js b/hack/migrate-v2.test.js new file mode 100644 index 0000000..253b3f5 --- /dev/null +++ b/hack/migrate-v2.test.js @@ -0,0 +1,30 @@ +"use strict"; + +const assert = require("node:assert/strict"); +const { test } = require("node:test"); +const YAML = require("yaml"); +const { modelNames, readYaml, root, schemaOf, schemaValidator } = require("./lib/catalog"); +const { migrate } = require("./migrate-v2"); + +test("every v1 model migrates to files the v2 schemas accept, its variants unchanged", () => { + const schema = schemaValidator(); + const v1 = modelNames().filter((name) => schemaOf(readYaml(`${root}/models/${name}/metadata.yaml`)) === "v1"); + assert.ok(v1.length > 0); + for (const name of v1) { + const { files } = migrate(name); + for (const [rel, body] of files) { + const doc = YAML.parse(body); + const validate = schema(rel.endsWith("/metadata.yaml") ? "metadata.v2.schema.json" : "variant.schema.json"); + assert.ok(validate(doc), `${rel}: ${JSON.stringify(validate.errors)}`); + if (!rel.endsWith("/metadata.yaml")) assert.equal(rel, `models/${name}/${doc.id}.yaml`); + } + const latest = YAML.parse([...files.values()][0]).version; + const was = (readYaml(`${root}/models/${name}/metadata.yaml`).tuning ?? []).map((t) => ({ ...t, baseline: `${t.baseline}@${latest}`, optimized: `${t.optimized}@${latest}` })); + assert.deepEqual(YAML.parse(files.get(`models/${name}/metadata.yaml`)).tuning ?? [], was, `${name}: tuning`); + } +}); + +test("a model already on v2 is not migrated again", () => { + const v2 = modelNames().find((name) => schemaOf(readYaml(`${root}/models/${name}/metadata.yaml`)) === "v2"); + if (v2) assert.throws(() => migrate(v2), /already on v2/); +}); diff --git a/hack/validate-helm.js b/hack/validate-helm.js index 630b552..fa30515 100644 --- a/hack/validate-helm.js +++ b/hack/validate-helm.js @@ -98,10 +98,10 @@ function main(opts, {repositoryRoot = root, runCommand = command} = {}) { const added = new Set(selection.added); Object.assign(summary, selection, {runDirectory: path.relative(root, runDir)}); const count = selection.files.length; - log(`Selected ${count} version file${count === 1 ? "" : "s"}: ${selection.reason}` + + log(`Selected ${count} version or variant file${count === 1 ? "" : "s"}: ${selection.reason}` + `${selection.mergeBase ? ` (${selection.mergeBase})` : ""}`); if (!count) { - log("No model version configuration changes (无版本配置变更)."); + log("No model version or variant configuration changes (无版本配置变更)."); return true; } log(); @@ -223,7 +223,7 @@ function main(opts, {repositoryRoot = root, runCommand = command} = {}) { if (added.has(file)) throw new Error(`chart.version: ${below}`); // A published version keeps its chart; rewriting it to pass would // change the digest every existing deploy recorded. - record.warning = `published version declares ${below}`; + record.warning = `as published, declares ${below}`; if (!pkg.archive) { record.skipped = true; continue; diff --git a/hack/validate-helm.test.js b/hack/validate-helm.test.js index a0c541f..9909a14 100644 --- a/hack/validate-helm.test.js +++ b/hack/validate-helm.test.js @@ -15,7 +15,7 @@ function fixture(t, failure, {chartName = "sglang", declaredVersion = "0.8.0", l t.after(() => fs.rmSync(root, {recursive: true, force: true})); const file = "models/example/example-1.0.0.yaml"; fs.mkdirSync(path.join(root, "models/example"), {recursive: true}); - fs.writeFileSync(path.join(root, file), YAML.stringify({variants: [ + fs.writeFileSync(path.join(root, file), YAML.stringify({apiVersion: "catalog.swiss/v1", variants: [ {id: "default", default: true, chart: {name: chartName, version: declaredVersion}}, {id: "other", chart: {name: chartName, version: `>=${config.minimumChartVersions[chartName]}`}, values: {extraArgs: ["--other"]}} ]})); diff --git a/hack/validate-images.js b/hack/validate-images.js index 9fe8de5..89b9cbd 100644 --- a/hack/validate-images.js +++ b/hack/validate-images.js @@ -15,7 +15,8 @@ const fs = require("node:fs"); const path = require("node:path"); const { parseArgs } = require("node:util"); const YAML = require("yaml"); -const { root, modelNames, versionFiles } = require("./lib/catalog"); +const { spawnSync } = require("node:child_process"); +const { root, modelNames, versionFiles, variantFiles, kindOf, schemaOf, historiesOf } = require("./lib/catalog"); const { checkImage, reportMismatches, changedVersionFiles } = require("./lib/images"); function parse(args) { @@ -28,26 +29,27 @@ function parse(args) { const files = positionals.map((file) => path.relative(root, path.resolve(file)).split(path.sep).join("/")); if (base && files.length) throw new Error("choose --base REF or explicit files, not both"); if (base) return { files: changedVersionFiles(root, base), matchDigest }; - const versions = modelNames().flatMap((name) => versionFiles(name)); + const versions = modelNames().flatMap((name) => [...versionFiles(name), ...variantFiles(name)]); if (!files.length) return { files: versions, matchDigest }; const unknown = files.filter((f) => !versions.includes(f)); - if (unknown.length) throw new Error(`not a model version file: ${unknown.join(", ")}`); + if (unknown.length) throw new Error(`not a model version or variant file: ${unknown.join(", ")}`); return { files, matchDigest }; } async function main() { const { files, matchDigest } = parse(process.argv.slice(2)); if (files.length === 0) { - console.log("no version files to check"); + console.log("no version or variant files to check"); return 0; } const docs = new Map(files.map((file) => [file, YAML.parse(fs.readFileSync(path.join(root, file), "utf8"))])); + const variantsOf = (doc) => (kindOf(doc) === "variant" ? [doc] : (doc?.variants ?? [])); // Each distinct image once, however many variants share it. const uses = new Map(); for (const [file, doc] of docs) { - for (const v of doc.variants ?? []) { + for (const v of variantsOf(doc)) { if (!v?.image?.repository) continue; const key = JSON.stringify([v.image.repository, String(v.image.tag ?? "latest"), v.image.digest ?? ""]); if (!uses.has(key)) uses.set(key, []); @@ -79,13 +81,26 @@ async function main() { // Tuned pairs of the selected versions. A missing report or variant is // validate:schema's to report. let reports = 0; + let histories; + const measured = (name, ref) => { + const [id, version] = String(ref ?? "").split("@"); + const rel = `models/${name}/${id}.yaml`; + const now = docs.get(rel) ?? (fs.existsSync(path.join(root, rel)) ? YAML.parse(fs.readFileSync(path.join(root, rel), "utf8")) : undefined); + if (String(now?.version) === version) return now; + histories ??= historiesOf(); + const commit = histories.get(rel)?.revisions.find((r) => r.version === version)?.commit; + const r = commit && spawnSync("git", ["show", `${commit}:${rel}`], { cwd: root, encoding: "utf8" }); + return r?.status === 0 ? YAML.parse(r.stdout) : undefined; + }; for (const name of new Set([...docs.keys()].map((file) => file.split("/")[1]))) { const meta = `models/${name}/metadata.yaml`; - const tuning = YAML.parse(fs.readFileSync(path.join(root, meta), "utf8"))?.tuning ?? []; - tuning.forEach((t, i) => { + const metadata = YAML.parse(fs.readFileSync(path.join(root, meta), "utf8")); + const v2 = schemaOf(metadata) === "v2"; + (metadata?.tuning ?? []).forEach((t, i) => { const doc = [...docs].find(([file, d]) => file.split("/")[1] === name && String(d?.version) === String(t?.version))?.[1]; - const variant = (id) => doc?.variants?.find((v) => v?.id === id); - const [b, o] = [variant(t?.baseline), variant(t?.optimized)]; + const variant = (key) => (v2 ? measured(name, t?.[key]) : doc?.variants?.find((v) => v?.id === t?.[key])); + if (v2 && ![t?.baseline, t?.optimized].some((ref) => docs.has(`models/${name}/${String(ref ?? "").split("@")[0]}.yaml`))) return; + const [b, o] = [variant("baseline"), variant("optimized")]; const html = path.join(root, "models", name, String(t?.report ?? "")); if (!t?.report || !b?.image || !o?.image || !fs.existsSync(html)) return; reports++; @@ -104,7 +119,7 @@ async function main() { const count = (level) => [...results.values()].filter((r) => r.level === level).length; const errors = count("error"); const bad = [...byFile.keys()].filter((f) => f.endsWith("metadata.yaml")).reduce((n, f) => n + byFile.get(f).length, 0); - console.log(`${plural(keys.length, "image")} in ${plural(files.length, "version file")}: ${count("ok")} ok, ${plural(count("warning"), "warning")}, ${plural(errors, "error")}`); + console.log(`${plural(keys.length, "image")} in ${plural(files.length, "file")}: ${count("ok")} ok, ${plural(count("warning"), "warning")}, ${plural(errors, "error")}`); console.log(`${reports} perf report${reports === 1 ? "" : "s"} checked against their variants' images, ${bad} mismatch${bad === 1 ? "" : "es"}`); return errors || bad ? 1 : 0; } diff --git a/hack/validate-schema.js b/hack/validate-schema.js index d6f647e..28d2db6 100644 --- a/hack/validate-schema.js +++ b/hack/validate-schema.js @@ -1,37 +1,16 @@ #!/usr/bin/env node -// Schema-check this catalog. +// Schema-check this catalog. --base REF also checks what changed since REF. "use strict" const fs = require("fs") const path = require("path") -// "2020" is the JSON Schema draft these files declare (2020-12), not the -// package's age. require("ajv") speaks draft-07; this entry point is Ajv 8. -const Ajv2020 = require("ajv/dist/2020") -const addFormats = require("ajv-formats") +const {parseArgs} = require("util") +const {spawnSync} = require("child_process") const YAML = require("yaml") -const {root, modelsDir, modelNames, versionFiles} = require("./lib/catalog") +const {root, modelsDir, modelNames, modelFiles, versionFiles, variantFiles, schemaOf, schemaValidator, historiesOf, revisionProblems} = require("./lib/catalog") const {reportProblem} = require("./lib/report") -const metadataFiles = modelNames() - .map((name) => path.join(modelsDir, name, "metadata.yaml")) - .filter((file) => fs.existsSync(file)) -const versionFilePaths = modelNames().flatMap((name) => versionFiles(name).map((rel) => path.join(root, rel))) - -const checks = [ - { - schema: "schema/catalog.schema.json", - files: fs.existsSync(path.join(root, "catalog.yaml")) ? [path.join(root, "catalog.yaml")] : [], - missing: "catalog.yaml: missing -- it names where the site is published", - }, - { - schema: "schema/metadata.schema.json", - files: metadataFiles, - }, - { - schema: "schema/version.schema.json", - files: versionFilePaths, - }, -] +const {values: {base}} = parseArgs({options: {base: {type: "string"}}}) function pointer(instancePath) { if (!instancePath) return "" @@ -91,161 +70,210 @@ function report(rel, lines) { for (const line of lines) console.error(` ${line}`) } -// strictTypes rejects draft 2020-12 that the spec allows: a union type, and -// keywords such as minimum without a sibling type. Keep the other strict checks. -// verbose puts the failing value on each error. -const ajv = new Ajv2020({allErrors: true, strict: true, strictTypes: false, verbose: true}) -addFormats(ajv) - let failed = false -for (const check of checks) { - if (check.files.length === 0) { - console.error(check.missing ?? `${check.schema}: no files`) - failed = true - continue +function fail(rel, lines) { + failed = true + report(rel, lines) +} + +function parse(file) { + return YAML.parse(fs.readFileSync(file, "utf8")) +} + +const parsed = new Map() +function doc(file) { + if (!parsed.has(file)) { + try { + parsed.set(file, parse(file)) + } catch { + parsed.set(file, undefined) + } } - let validate + return parsed.get(file) +} + +let schemaFor +try { + schemaFor = schemaValidator() +} catch (err) { + console.error(`schema/: ${err.message}`) + process.exit(1) +} + +function schemaCheck(file, schema) { + const rel = path.relative(root, file) + let data try { - validate = ajv.compile(JSON.parse(fs.readFileSync(path.join(root, check.schema), "utf8"))) + data = parse(file) } catch (err) { - console.error(`${check.schema}: ${err.message}`) - process.exit(1) + const at = err.linePos?.[0] + const where = at ? `line ${at.line}:${at.col}: ` : "" + fail(rel, `${where}${err.message}`.split("\n")) + return } - for (const file of check.files) { - const rel = path.relative(root, file) - let data - try { - data = YAML.parse(fs.readFileSync(file, "utf8")) - } catch (err) { - const at = err.linePos?.[0] - const where = at ? `line ${at.line}:${at.col}: ` : "" - report(rel, `${where}${err.message}`.split("\n")) - failed = true - continue + const validate = schemaFor(schema) + if (validate(data)) return + fail(rel, meaningful(validate.errors).map((err) => { + const at = pointer(err.instancePath) + const text = detail(err) + return at ? `${at}: ${text}` : text + })) +} + +// Files are v1 or v2 by their content, never their name. +const catalogFile = path.join(root, "catalog.yaml") +if (fs.existsSync(catalogFile)) schemaCheck(catalogFile, "catalog.schema.json") +else fail("catalog.yaml", ["missing -- it names where the site is published"]) + +if (modelNames().length === 0) fail("models/", ["no models"]) +for (const name of modelNames()) { + const dir = path.join(modelsDir, name) + const meta = path.join(dir, "metadata.yaml") + if (!fs.existsSync(meta)) { + fail(path.relative(root, dir), ["missing metadata.yaml"]) + continue + } + schemaCheck(meta, schemaOf(doc(meta)) === "v2" ? "metadata.v2.schema.json" : "metadata.schema.json") + for (const {rel, kind, error} of modelFiles(name)) { + if (kind === "version") schemaCheck(path.join(root, rel), "version.schema.json") + else if (kind === "variant") schemaCheck(path.join(root, rel), "variant.schema.json") + else if (error) schemaCheck(path.join(root, rel), "version.schema.json") // reports the parse error + else { + fail(rel, [ + "neither a v1 version file (apiVersion: catalog.swiss/v1) nor a v2 variant (apiVersion: catalog.swiss/v2, kind: Variant)", + ]) } - if (validate(data)) continue - failed = true - report(rel, meaningful(validate.errors).map((err) => { - const at = pointer(err.instancePath) - const text = detail(err) - return at ? `${at}: ${text}` : text - })) + } + for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".history.json"))) { + fail(`models/${name}/${f}`, ["generated by the Pages build from git, never committed: delete it"]) } } // What the schema cannot see: a variant id is unique within its version. A // deploy, a tuning entry and validate:helm all name a variant by it. +const versionFilePaths = modelNames().flatMap((name) => versionFiles(name).map((rel) => path.join(root, rel))) for (const file of versionFilePaths) { - let variants - try { - variants = YAML.parse(fs.readFileSync(file, "utf8"))?.variants - } catch { - continue // a parse error is already reported above - } + const variants = doc(file)?.variants if (!Array.isArray(variants)) continue const ids = variants.map((v) => v?.id) const twice = [...new Set(ids.filter((id, i) => ids.indexOf(id) !== i))] if (twice.length > 0) { - failed = true - report(path.relative(root, file), twice.map((id) => `variants: id ${JSON.stringify(id)} is used twice`)) + fail(path.relative(root, file), twice.map((id) => `variants: id ${JSON.stringify(id)} is used twice`)) } } +// An LWS group that disagrees with requires.nodes hangs at rendezvous. +function variantProblems(v, at) { + const lines = [] + const lws = v?.values?.lws + if (lws?.enabled === true && lws.size !== v?.requires?.nodes) { + lines.push(`${at}values.lws.size ${lws.size} is not requires.nodes ${v?.requires?.nodes}`) + } + if (v?.chart?.name && v?.engine && v.chart.name !== v.engine) { + lines.push(`${at}chart.name ${v.chart.name} is not its engine, ${v.engine}`) + } + return lines +} + // What the schema cannot see, per model: it lives in models//, named // after metadata.yaml's name -- the index gives a perf report as a bare file -// name, and consumers fetch it from models// -- it has a version -// to publish, each version file declares the model's name and is named -// -.yaml, at most one variant is the default, and an LWS group -// is as many pods as its variant requires nodes -- a group that disagrees -// hangs at rendezvous instead of failing. +// name, and consumers fetch it from models// -- and it has +// something to serve. for (const name of modelNames()) { const dir = path.join(modelsDir, name) - let model - try { - model = YAML.parse(fs.readFileSync(path.join(dir, "metadata.yaml"), "utf8"))?.name - } catch { - // reported by the schema check above - } + const meta = doc(path.join(dir, "metadata.yaml")) + const model = meta?.name if (typeof model === "string" && model !== name) { - failed = true - report(path.relative(root, dir), [ + fail(path.relative(root, dir), [ `must be named models/${model}/, after its model (metadata.yaml's name): consumers fetch its reports from models//`, ]) } - const files = versionFiles(name) - if (files.length === 0) { - failed = true - report(path.relative(root, dir), ["no version files: publish one, or remove the directory"]) + const v2 = schemaOf(meta) === "v2" + const versions = versionFiles(name) + const variants = variantFiles(name) + if (!v2 && variants.length > 0) { + fail(path.relative(root, dir), [ + `${variants.map((rel) => path.basename(rel)).join(", ")}: v2 variants in a v1 model; ` + + `convert it first (node hack/migrate-v2.js ${name}), which makes metadata.yaml v2`, + ]) + } + if (v2 ? variants.length === 0 : versions.length === 0) { + fail(path.relative(root, dir), [v2 ? "no variant files: add one, or remove the directory" : "no version files: publish one, or remove the directory"]) continue } - for (const rel of files) { - let doc - try { - doc = YAML.parse(fs.readFileSync(path.join(root, rel), "utf8")) - } catch { - continue // reported by the schema check above - } + for (const rel of versions) { + const d = doc(path.join(root, rel)) + if (!d) continue // reported by the schema check above const lines = [] - if (model && doc?.name !== model) { - lines.push(`name: ${JSON.stringify(doc?.name)} is not its model's, ${JSON.stringify(model)} in metadata.yaml`) + if (model && d.name !== model) { + lines.push(`name: ${JSON.stringify(d.name)} is not its model's, ${JSON.stringify(model)} in metadata.yaml`) } - const want = `${doc?.name}-${doc?.version}.yaml` + const want = `${d.name}-${d.version}.yaml` if (path.basename(rel) !== want) lines.push(`must be named ${want}, after the name and version it declares`) - const variants = Array.isArray(doc?.variants) ? doc.variants : [] - const defaults = variants.filter((v) => v?.default === true).map((v) => v.id) + const vs = Array.isArray(d.variants) ? d.variants : [] + const defaults = vs.filter((v) => v?.default === true).map((v) => v.id) if (defaults.length > 1) lines.push(`variants: ${defaults.join(", ")} are all marked default; at most one may be`) - for (const v of variants) { - const lws = v?.values?.lws - if (lws?.enabled === true && lws.size !== v?.requires?.nodes) { - lines.push(`variants: ${v.id}: values.lws.size ${lws.size} is not requires.nodes ${v?.requires?.nodes}`) - } - } - if (lines.length > 0) { - failed = true - report(rel, lines) - } + for (const v of vs) lines.push(...variantProblems(v, `variants: ${v?.id}: `)) + if (lines.length > 0) fail(rel, lines) + } + if (!v2) continue + const stems = new Set(versions.map((rel) => path.basename(rel, ".yaml"))) + const ids = new Set() + for (const rel of variants) { + const d = doc(path.join(root, rel)) + if (!d) continue + const lines = [] + if (model && d.model !== model) lines.push(`model: ${JSON.stringify(d.model)} is not its model's, ${JSON.stringify(model)} in metadata.yaml`) + if (d.id === "metadata") lines.push("id: metadata is the model's own file; choose another id") + else if (stems.has(d.id)) lines.push(`id: ${d.id} is a v1 version file's name; choose another id`) + else if (path.basename(rel) !== `${d.id}.yaml`) lines.push(`must be named ${d.id}.yaml, after the id it declares`) + lines.push(...variantProblems(d, "")) + if (d.replaces !== undefined && d.replaces === d.id) lines.push("replaces: names the variant itself") + if (lines.length > 0) fail(rel, lines) + ids.add(d.id) + } + if (meta?.defaultVariant !== undefined && !ids.has(meta.defaultVariant)) { + fail(`models/${name}/metadata.yaml`, [`defaultVariant: ${JSON.stringify(meta.defaultVariant)} is not a variant of this model`]) } } +// Built from git only when needed. +let histories +function history(rel) { + histories ??= historiesOf() + return histories.get(rel) +} + +// A variant id with an "optimized" segment (the naming convention, as in +// sglang-tp8-h100-optimized) must be recorded as the optimized side of some +// tuning entry, or the sites show the model as untuned. +const namedOptimized = (id) => typeof id === "string" && /(^|-)optimi[sz]ed(-|$)/.test(id) + +function workloadProblems(t, at) { + const names = Array.isArray(t.workloads) ? t.workloads.map((w) => w?.name) : [] + return [...new Set(names.filter((n, j) => names.indexOf(n) !== j))].map((name) => `${at}.workloads: ${JSON.stringify(name)} is listed twice`) +} + // What the schema cannot see: a tuning pair must name a published version of // its model, two different variants of that version, and a report that exists; // and a variant named as tuned must have its result recorded. swiss does not // check any of it -- tuning is display only, and a mistake in it must not stop // a catalog loading for deploys -- so it is caught here instead. -for (const meta of metadataFiles) { - let tuning - try { - tuning = YAML.parse(fs.readFileSync(meta, "utf8"))?.tuning - } catch { - continue // a parse error is already reported above - } - // Not a list is a schema error, reported above; a model with none still - // gets the recorded-result check below. - if (!Array.isArray(tuning)) tuning = [] - - const dir = path.dirname(meta) +function v1Tuning(dir, tuning) { const published = new Map() for (const rel of versionFiles(path.basename(dir))) { - try { - const doc = YAML.parse(fs.readFileSync(path.join(root, rel), "utf8")) - if (doc?.version) published.set(String(doc.version), new Set((doc.variants ?? []).map((v) => v?.id))) - } catch { - // reported by the schema check above - } + const d = doc(path.join(root, rel)) + if (d?.version) published.set(String(d.version), new Set((d.variants ?? []).map((v) => v?.id))) } const lines = [] - - // A variant whose id names it optimized (the naming convention: an - // "optimized" segment, as in sglang-tp8-h100-optimized) must be recorded as - // the optimized side of some tuning entry, or the sites show the model as - // untuned. Any version's entry will do: a tuned variant carried into a later - // version unchanged is shown with the number it was measured with. + // Any version's entry will do: a tuned variant carried into a later version + // unchanged is shown with the number it was measured with. const recorded = new Set(tuning.map((t) => t?.optimized)) const missing = new Map() for (const [version, ids] of published) { for (const id of ids) { - if (typeof id !== "string" || !/(^|-)optimi[sz]ed(-|$)/.test(id) || recorded.has(id)) continue + if (!namedOptimized(id) || recorded.has(id)) continue missing.set(id, [...(missing.get(id) ?? []), version]) } } @@ -275,15 +303,83 @@ for (const meta of metadataFiles) { if (typeof t.report === "string" && !fs.existsSync(path.join(dir, t.report))) { lines.push(`${at}.report: no file ${JSON.stringify(t.report)} beside metadata.yaml`) } - const names = Array.isArray(t.workloads) ? t.workloads.map((w) => w?.name) : [] - for (const name of new Set(names.filter((n, j) => names.indexOf(n) !== j))) { - lines.push(`${at}.workloads: ${JSON.stringify(name)} is listed twice`) - } + lines.push(...workloadProblems(t, at)) }) - if (lines.length > 0) { - failed = true - report(path.relative(root, meta), lines) + return lines +} + +// v2: is the current variant, @ a revision in its history. +function v2Tuning(dir, tuning) { + const name = path.basename(dir) + const current = new Map(variantFiles(name).map((rel) => doc(path.join(root, rel))).filter(Boolean).map((d) => [d.id, String(d.version)])) + const lines = [] + const recorded = new Set(tuning.map((t) => String(t?.optimized ?? "").split("@")[0])) + for (const id of current.keys()) { + if (namedOptimized(id) && !recorded.has(id)) { + lines.push(`variant ${JSON.stringify(id)} is named optimized but has no tuning entry; add one to tuning with optimized: ${id}, its baseline, and the uplift`) + } } + const legacy = new Map(versionFiles(name).map((rel) => doc(path.join(root, rel))).filter(Boolean) + .map((d) => [String(d.version), new Set((d.variants ?? []).map((v) => v?.id))])) + tuning.forEach((t, i) => { + if (!t || typeof t !== "object") return + const at = `tuning[${i}]` + if (t.version !== undefined) { + const ids = legacy.get(String(t.version)) + if (!ids) lines.push(`${at}.version: ${JSON.stringify(t.version)} is not a v1 version of this model`) + else for (const key of ["baseline", "optimized"]) { + const id = String(t[key] ?? "").split("@")[0] + if (!ids.has(id)) lines.push(`${at}.${key}: version ${t.version} has no variant ${JSON.stringify(id)}`) + } + } + for (const key of ["baseline", "optimized"]) { + if (typeof t[key] !== "string") continue + const [id, version] = t[key].split("@") + if (!version) { + lines.push(`${at}.${key}: name the version measured, ${id}@${current.get(id) ?? ""}`) + continue + } + if (current.get(id) === version) continue + let h + try { + h = history(`models/${name}/${id}.yaml`) + } catch (err) { + lines.push(`${at}.${key}: cannot check ${t[key]}: ${err.message}`) + continue + } + if (!h) lines.push(`${at}.${key}: ${JSON.stringify(id)} has never been a variant of this model`) + else if (!h.revisions.some((r) => r.version === version)) { + lines.push(`${at}.${key}: ${id} has no version ${version}; it has ${h.revisions.map((r) => r.version).join(", ")}`) + } + } + if (t.baseline !== undefined && t.baseline === t.optimized) { + lines.push(`${at}: baseline and optimized are the same variant`) + } + if (typeof t.report === "string" && !fs.existsSync(path.join(dir, t.report))) { + lines.push(`${at}.report: no file ${JSON.stringify(t.report)} beside metadata.yaml`) + } + lines.push(...workloadProblems(t, at)) + }) + return lines +} + +for (const name of modelNames()) { + const meta = path.join(modelsDir, name, "metadata.yaml") + if (!fs.existsSync(meta)) continue + const d = doc(meta) + if (!d) continue // a parse error is already reported above + // Not a list is a schema error, reported above; a model with none still + // gets the recorded-result check. + const tuning = Array.isArray(d.tuning) ? d.tuning : [] + const lines = schemaOf(d) === "v2" ? v2Tuning(path.dirname(meta), tuning) : v1Tuning(path.dirname(meta), tuning) + if (lines.length > 0) fail(path.relative(root, meta), lines) +} + +// A variant file's version goes up with every change, and only with one. +try { + for (const [rel, line] of revisionProblems()) fail(rel, [line]) +} catch (err) { + fail("models/", [`cannot check variant versions: ${err.message}`]) } // The site links every HTML file beside a model's metadata and skips one it @@ -295,11 +391,60 @@ for (const name of modelNames()) { for (const file of fs.readdirSync(dir).filter((f) => f.endsWith(".html")).sort()) { const problem = file === "index.html" ? "index.html is the model's page on the site; name the report something else" : reportProblem(path.join(dir, file)) - if (problem) { - failed = true - report(path.relative(root, path.join(dir, file)), [problem]) + if (problem) fail(path.relative(root, path.join(dir, file)), [problem]) + } +} + +function git(args) { + const r = spawnSync("git", args, {cwd: root, encoding: "utf8", maxBuffer: 1 << 28}) + if (r.status !== 0) throw new Error(`git ${args.join(" ")}: ${(r.stderr || r.error?.message || "").trim()}`) + return r.stdout +} + +// --base: checks on what changed since the merge-base. +function baseChecks() { + const mergeBase = git(["merge-base", base, "HEAD"]).trim() + const at = (rel) => { + const r = spawnSync("git", ["show", `${mergeBase}:${rel}`], {cwd: root, encoding: "utf8", maxBuffer: 1 << 28}) + return r.status === 0 ? r.stdout : null + } + const parseAt = (rel) => { + try { + return YAML.parse(at(rel) ?? "") + } catch { + return undefined } } + const status = new Map() + const tokens = git(["diff", "--name-status", "--no-renames", "-z", mergeBase, "--", "models"]).split("\0") + for (let i = 0; i + 1 < tokens.length; i += 2) status.set(tokens[i + 1], tokens[i]) + for (const rel of git(["ls-files", "--others", "--exclude-standard", "-z", "--", "models"]).split("\0").filter(Boolean)) status.set(rel, "A") + + for (const name of modelNames()) { + const metaRel = `models/${name}/metadata.yaml` + const meta = doc(path.join(root, metaRel)) + if (schemaOf(meta) !== "v2") continue + for (const rel of versionFiles(name)) { + if (status.get(rel) === "A") { + fail(rel, [`${name} is on v2 (its metadata.yaml): change a variant file in place rather than publishing a new version file`]) + } + } + for (const [rel, s] of status) { + if (s !== "D" || !rel.startsWith(`models/${name}/`) || !rel.endsWith(".yaml")) continue + const was = parseAt(rel) + if (was?.kind === "Variant" && !was.deprecated) { + fail(rel, ["deleted without being deprecated first: mark it deprecated: and publish that, then delete it in a later change"]) + } + } + } +} + +if (base) { + try { + baseChecks() + } catch (err) { + fail(`--base ${base}`, [err.message]) + } } process.exit(failed ? 1 : 0) diff --git a/package.json b/package.json index 2aef394..641da11 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "validate:schema": "node hack/validate-schema.js", "validate:helm": "node hack/validate-helm.js", "validate:images": "node hack/validate-images.js", - "test": "node --test 'hack/lib/*.test.js' hack/validate-helm.test.js 'server/*.test.js' && npm run test:web", + "test": "node --test 'hack/lib/*.test.js' hack/validate-helm.test.js hack/migrate-v2.test.js 'server/*.test.js' && npm run test:web", "test:helm:integration": "node --test hack/validate-helm.integration.test.js", "build:site": "vite build --config web/vite.config.mts --logLevel warn", "serve": "node server", diff --git a/schema/catalog.schema.json b/schema/catalog.schema.json index d38fd26..def830e 100644 --- a/schema/catalog.schema.json +++ b/schema/catalog.schema.json @@ -17,6 +17,18 @@ "type": "string", "format": "uri", "pattern": "^https?://.+/$" + }, + "repo": { + "description": "The catalog repository on GitHub.", + "type": "string", + "format": "uri", + "pattern": "^https://.+[^/]$" + }, + "raw": { + "description": "Where files are fetched raw at a commit, as /. Defaults to raw.githubusercontent.com for a GitHub repo.", + "type": "string", + "format": "uri", + "pattern": "^https?://.+/$" } } } diff --git a/schema/index-v2.schema.json b/schema/index-v2.schema.json new file mode 100644 index 0000000..ce8e590 --- /dev/null +++ b/schema/index-v2.schema.json @@ -0,0 +1,252 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://swiss.catalog/schema/index-v2.schema.json", + "title": "Swiss catalog index (v2)", + "description": "index-v2.json: the v2 models (metadata.yaml catalog.swiss/v2) and their variant files, generated by the Pages build from models/, catalog.yaml and git history. v1 models stay in index.json.", + "type": "object", + "additionalProperties": false, + "required": [ + "apiVersion", + "count", + "models" + ], + "properties": { + "apiVersion": { + "const": "catalog.swiss/v2" + }, + "site": { + "$ref": "catalog.schema.json#/properties/site" + }, + "repo": { + "$ref": "catalog.schema.json#/properties/repo" + }, + "raw": { + "description": "Old revisions are fetched at /.", + "$ref": "catalog.schema.json#/properties/raw" + }, + "commit": { + "description": "The last commit that changed models/ or catalog.yaml.", + "$ref": "#/$defs/commit" + }, + "count": { + "type": "integer", + "minimum": 0 + }, + "models": { + "type": "array", + "items": { + "$ref": "#/$defs/model" + } + } + }, + "$defs": { + "commit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "digest": { + "description": "sha256 of the file's bytes.", + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "path": { + "type": "string", + "pattern": "^models/[a-z0-9][a-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$" + }, + "model": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "reports", + "tuning", + "variants" + ], + "properties": { + "name": { + "$ref": "metadata.schema.json#/properties/name" + }, + "displayName": { + "type": "string" + }, + "description": { + "type": "string" + }, + "family": { + "type": "string" + }, + "series": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "deprecated": { + "type": [ + "boolean", + "string" + ] + }, + "source": { + "type": "object", + "additionalProperties": false, + "properties": { + "hf": { + "type": "string" + }, + "revision": { + "type": "string" + }, + "sizeGiB": { + "type": "number" + } + } + }, + "quantization": { + "type": "string" + }, + "defaultVariant": { + "description": "The first variant when absent.", + "type": "string" + }, + "reports": { + "description": "Linkable reports at models//.", + "type": "array", + "items": { + "type": "string" + } + }, + "tuning": { + "description": "As in metadata.yaml, without the legacy version; report absent when not linkable.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "baseline", + "optimized" + ], + "properties": { + "baseline": { + "type": "string" + }, + "optimized": { + "type": "string" + }, + "uplift": { + "type": "number" + }, + "workloads": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "uplift" + ], + "properties": { + "name": { + "type": "string" + }, + "uplift": { + "type": "number" + } + } + } + }, + "report": { + "type": "string" + } + } + } + }, + "variants": { + "description": "One per variant file, by id.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/variant" + } + } + } + }, + "variant": { + "description": "A variant without its values.", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "version", + "path", + "digest", + "engine", + "chart", + "requires", + "history", + "releaseDate" + ], + "properties": { + "id": { + "$ref": "version.schema.json#/$defs/variant/properties/id" + }, + "version": { + "description": "The variant file's version.", + "type": "string" + }, + "releaseDate": { + "type": "string", + "format": "date" + }, + "commit": { + "description": "The last commit that changed the file; with path and digest, what a deploy pins. Absent when uncommitted.", + "$ref": "#/$defs/commit" + }, + "path": { + "$ref": "#/$defs/path" + }, + "digest": { + "$ref": "#/$defs/digest" + }, + "history": { + "description": "Its history.json.", + "$ref": "#/$defs/path" + }, + "servedName": { + "type": "string" + }, + "engine": { + "type": "string" + }, + "description": { + "type": "string" + }, + "link": { + "type": "string", + "pattern": "^https?://" + }, + "chart": { + "$ref": "version.schema.json#/$defs/variant/properties/chart" + }, + "image": { + "$ref": "version.schema.json#/$defs/variant/properties/image" + }, + "requires": { + "$ref": "version.schema.json#/$defs/variant/properties/requires" + }, + "deprecated": { + "type": [ + "boolean", + "string" + ] + }, + "replaces": { + "type": "string" + } + } + } + } +} diff --git a/schema/metadata.v2.schema.json b/schema/metadata.v2.schema.json new file mode 100644 index 0000000..21f7de2 --- /dev/null +++ b/schema/metadata.v2.schema.json @@ -0,0 +1,104 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://swiss.catalog/schema/metadata.v2.schema.json", + "title": "Swiss catalog model metadata (v2)", + "description": "models//metadata.yaml for a v2 model: what the model is, its default variant and tuning results. Edited in place.", + "type": "object", + "additionalProperties": false, + "required": [ + "apiVersion", + "name", + "source" + ], + "properties": { + "apiVersion": { + "const": "catalog.swiss/v2" + }, + "name": { + "$ref": "metadata.schema.json#/properties/name" + }, + "displayName": { + "$ref": "metadata.schema.json#/properties/displayName" + }, + "description": { + "$ref": "metadata.schema.json#/properties/description" + }, + "homepage": { + "$ref": "metadata.schema.json#/properties/homepage" + }, + "license": { + "$ref": "metadata.schema.json#/properties/license" + }, + "family": { + "$ref": "metadata.schema.json#/properties/family" + }, + "series": { + "description": "Groups related models, e.g. deepseek-v4-flash and deepseek-v4-flash-0731.", + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]*$", + "maxLength": 63 + }, + "quantization": { + "description": "Weight format, e.g. fp8, nvfp4.", + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$", + "maxLength": 32 + }, + "tags": { + "$ref": "metadata.schema.json#/properties/tags" + }, + "deprecated": { + "$ref": "metadata.schema.json#/properties/deprecated" + }, + "defaultVariant": { + "description": "The variant a UI preselects; the first by id when unset.", + "$ref": "version.schema.json#/$defs/variant/properties/id" + }, + "tuning": { + "description": "Measured tuning results. Display only.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "baseline", + "optimized", + "report" + ], + "properties": { + "version": { + "description": "Legacy: the v1 version measured, for index.json.", + "$ref": "metadata.schema.json#/properties/tuning/items/properties/version" + }, + "baseline": { + "description": "The variant measured as the baseline.", + "$ref": "#/$defs/measured" + }, + "optimized": { + "description": "The tuned variant.", + "$ref": "#/$defs/measured" + }, + "uplift": { + "$ref": "metadata.schema.json#/properties/tuning/items/properties/uplift" + }, + "workloads": { + "$ref": "metadata.schema.json#/properties/tuning/items/properties/workloads" + }, + "report": { + "$ref": "metadata.schema.json#/properties/tuning/items/properties/report" + } + } + } + }, + "source": { + "$ref": "metadata.schema.json#/properties/source" + } + }, + "$defs": { + "measured": { + "description": "@: the variant version measured.", + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*@\\d+\\.\\d+\\.\\d+(-[0-9A-Za-z.-]+)?$" + } + } +} diff --git a/schema/variant.schema.json b/schema/variant.schema.json new file mode 100644 index 0000000..903d881 --- /dev/null +++ b/schema/variant.schema.json @@ -0,0 +1,81 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://swiss.catalog/schema/variant.schema.json", + "title": "Swiss catalog variant (v2)", + "description": "models//.yaml: one variant of a model, edited in place, its version raised on every change. Site-specific and per-deploy settings are rejected, as in v1.", + "type": "object", + "additionalProperties": false, + "required": [ + "apiVersion", + "kind", + "model", + "id", + "version", + "releaseDate", + "engine", + "chart", + "requires" + ], + "properties": { + "apiVersion": { + "const": "catalog.swiss/v2" + }, + "kind": { + "const": "Variant" + }, + "model": { + "description": "The model: its metadata.yaml name and directory.", + "$ref": "metadata.schema.json#/properties/name" + }, + "id": { + "description": "Unique within the model, and the file name. Not metadata.", + "$ref": "version.schema.json#/$defs/variant/properties/id" + }, + "version": { + "description": "Semver, raised on every change to this file.", + "type": "string", + "pattern": "^\\d+\\.\\d+\\.\\d+(-[0-9A-Za-z.-]+)?$" + }, + "releaseDate": { + "description": "When this version was released.", + "type": "string", + "format": "date" + }, + "servedName": { + "description": "--served-model-name, when it differs from the model name.", + "$ref": "version.schema.json#/properties/servedName" + }, + "description": { + "$ref": "version.schema.json#/$defs/variant/properties/description" + }, + "link": { + "$ref": "version.schema.json#/$defs/variant/properties/link" + }, + "engine": { + "$ref": "version.schema.json#/$defs/variant/properties/engine" + }, + "chart": { + "$ref": "version.schema.json#/$defs/variant/properties/chart" + }, + "image": { + "$ref": "version.schema.json#/$defs/variant/properties/image" + }, + "requires": { + "$ref": "version.schema.json#/$defs/variant/properties/requires" + }, + "values": { + "$ref": "version.schema.json#/$defs/variant/properties/values" + }, + "deprecated": { + "description": "Deprecate before deleting the file. A string is the reason.", + "type": [ + "boolean", + "string" + ] + }, + "replaces": { + "description": "The id of a variant this one renames or supersedes.", + "$ref": "version.schema.json#/$defs/variant/properties/id" + } + } +} diff --git a/server/data.js b/server/data.js index 62475cf..e3ee262 100644 --- a/server/data.js +++ b/server/data.js @@ -12,7 +12,7 @@ const fs = require("fs"); const path = require("path"); const { execFileSync } = require("child_process"); const { parseDocument, LineCounter } = require("yaml"); -const { root, modelsDir, readYaml, modelNames, versionFiles, compareVersions, latestOf, buildIndex, serializeIndex } = require("../hack/lib/catalog"); +const { root, modelsDir, readYaml, modelNames, versionFiles, compareVersions, latestOf, buildIndex, serializeIndex, buildIndexV2, checkIndexV2, historyPath, schemaOf, legacyTuning } = require("../hack/lib/catalog"); const { reportProblem } = require("../hack/lib/report"); // The GitHub repository the version files live in: the one being built in CI, @@ -176,7 +176,7 @@ function loadCatalog() { })); // A tuning entry whose report is skipped keeps its numbers; only the link goes. const linked = new Set(reports.map((r) => r.file)); - const tuning = (Array.isArray(meta.tuning) ? meta.tuning : []).map((t) => { + const tuning = ((schemaOf(meta) === "v2" ? legacyTuning(meta.tuning) : meta.tuning) ?? []).map((t) => { if (!t || !t.report || linked.has(t.report)) return t; if (!htmlFiles.includes(t.report)) linkable(`models/${name}/${t.report}`); return { ...t, report: undefined }; @@ -382,6 +382,31 @@ function publishCatalog(index, outDir) { return { models: index.models.length, files }; } +// index-v2.json and its histories; throws when they fail their checks. +function checkedIndexV2() { + const built = buildIndexV2(); + const problems = checkIndexV2(built); + if (problems.length) throw new Error(problems.join("\n")); + return built; +} + +// index-v2.json, the v2 variant files and every history.json. +function publishV2(outDir) { + const { index, histories } = checkedIndexV2(); + fs.writeFileSync(path.join(outDir, "index-v2.json"), serializeIndex(index)); + let files = 0; + for (const v of index.models.flatMap((m) => m.variants)) { + fs.mkdirSync(path.dirname(path.join(outDir, v.path)), { recursive: true }); + fs.copyFileSync(path.join(root, v.path), path.join(outDir, v.path)); + files++; + } + for (const [rel, h] of histories) { + fs.mkdirSync(path.dirname(path.join(outDir, rel)), { recursive: true }); + fs.writeFileSync(path.join(outDir, historyPath(rel)), JSON.stringify(h, null, 2) + "\n"); + } + return { models: index.count, files, histories: histories.size }; +} + // Everything beside the pages, as a build writes it into outDir: catalog.json, // the perf reports, and the catalog (index.json and its version files), from // checkedIndex. Returns a line saying what it wrote. @@ -390,6 +415,7 @@ function writeSiteData(outDir, { index, rewrites }) { fs.writeFileSync(path.join(outDir, "catalog.json"), JSON.stringify(catalog, null, 2) + "\n"); copyReports(catalog, outDir); const published = publishCatalog(index, outDir); + const v2 = publishV2(outDir); // GitHub Pages serves the tree as it is. fs.writeFileSync(path.join(outDir, ".nojekyll"), ""); const variants = catalog.models.reduce((n, m) => n + m.versions.find((v) => v.version === m.latest).variants.length, 0); @@ -398,8 +424,9 @@ function writeSiteData(outDir, { index, rewrites }) { `${catalog.models.filter((m) => m.comparison).length} optimized-vs-baseline, ` + `${catalog.models.reduce((n, m) => n + m.reports.length, 0)} perf HTML; ` + `catalog: ${published.models} models, ${published.files} version files` + - (rewrites ? `, ${rewrites} rewritten in place (REWRITES=allow)` : "") + (rewrites ? `, ${rewrites} rewritten in place (REWRITES=allow)` : "") + + `; index-v2: ${v2.models} models, ${v2.files} variant files, ${v2.histories} histories` ); } -module.exports = { loadCatalog, checkedIndex, rewritesOf, writeSiteData, buildIndex, serializeIndex }; +module.exports = { loadCatalog, checkedIndex, checkedIndexV2, rewritesOf, writeSiteData, buildIndex, serializeIndex }; diff --git a/server/data.test.js b/server/data.test.js index 74db6d2..cc9e0f7 100644 --- a/server/data.test.js +++ b/server/data.test.js @@ -28,3 +28,13 @@ test("a published version is matched by model and version: moved unchanged is a test("nothing published yet: nothing to check", () => { assert.deepEqual(rewritesOf(null, index([["m", [v("1.0.0", "models/m/m-1.0.0.yaml", "sha256:a")]]])), { rewrites: [], moves: [] }); }); + +test("catalog.json pairs a v2 model's pinned tuning refs with its variants", () => { + const { loadCatalog } = require("./data"); + const { readYaml, root, schemaOf } = require("../hack/lib/catalog"); + const tuned = loadCatalog().models.filter((m) => schemaOf(readYaml(`${root}/models/${m.dir}/metadata.yaml`)) === "v2" && m.tuning.length); + for (const m of tuned) { + assert.ok(m.comparison, `${m.name}: no comparison`); + assert.ok(m.versions[0].variants.some((v) => v.isOptimized), `${m.name}: no optimized variant`); + } +}); diff --git a/server/index.js b/server/index.js index cd99d9a..59e31d8 100644 --- a/server/index.js +++ b/server/index.js @@ -6,6 +6,8 @@ // GET /models//.yaml the version files it names // GET /models//.html perf reports // GET /catalog.json what the web app renders from +// GET /index-v2.json v2 models and their variant files +// GET /models//.history.json // // npm run serve # http://127.0.0.1:8000 // npm run serve -- 9000 @@ -17,8 +19,9 @@ const fs = require("fs"); const http = require("http"); const path = require("path"); -const { root, modelsDir } = require("../hack/lib/catalog"); -const { loadCatalog, buildIndex, serializeIndex } = require("./data"); +const { execFileSync } = require("child_process"); +const { root, modelsDir, historyPath } = require("../hack/lib/catalog"); +const { loadCatalog, checkedIndexV2, buildIndex, serializeIndex } = require("./data"); const TYPES = { ".json": "application/json", ".yaml": "text/yaml; charset=utf-8", ".html": "text/html; charset=utf-8" }; @@ -27,7 +30,18 @@ const TYPES = { ".json": "application/json", ".yaml": "text/yaml; charset=utf-8" // index.json is cheap, so it is always built fresh. function createServer() { let catalog = null; - const watcher = fs.watch(modelsDir, { recursive: true }, () => (catalog = null)); + let v2 = null; + const watcher = fs.watch(modelsDir, { recursive: true }, () => (catalog = v2 = null)); + // Rebuilt after a commit too: versions come from git. + const head = () => execFileSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }).trim(); + const indexV2 = () => { + const at = head(); + if (v2?.at !== at) { + const { index, histories } = checkedIndexV2(); + v2 = { at, index: serializeIndex(index), histories: new Map([...histories].map(([rel, h]) => [historyPath(rel), JSON.stringify(h, null, 2) + "\n"])) }; + } + return v2; + }; const server = http.createServer((req, res) => { const send = (status, type, body) => { @@ -49,6 +63,11 @@ function createServer() { try { if (url === "/index.json") return send(200, TYPES[".json"], serializeIndex(buildIndex())); if (url === "/catalog.json") return send(200, TYPES[".json"], JSON.stringify((catalog ??= loadCatalog()))); + if (url === "/index-v2.json") return send(200, TYPES[".json"], indexV2().index); + if (/^\/models\/[^/]+\/[^/]+\.history\.json$/.test(url)) { + const body = indexV2().histories.get(url.slice(1)); + return body ? send(200, TYPES[".json"], body) : send(404, "text/plain", "not in this catalog\n"); + } const file = path.join(root, url); if (/^\/models\/[^/]+\/[^/]+\.(yaml|html)$/.test(url) && file.startsWith(modelsDir + path.sep) && fs.existsSync(file)) { return send(200, TYPES[path.extname(file)], fs.readFileSync(file)); diff --git a/server/index.test.js b/server/index.test.js index 3025e0c..fac3103 100644 --- a/server/index.test.js +++ b/server/index.test.js @@ -44,6 +44,15 @@ test("catalog.json and the perf reports it links are served", async () => { assert.match(res.headers.get("content-type"), /^text\/html/); }); +test("index-v2.json lists only v2 models, and each variant's history is served", async () => { + const index = await (await fetch(`${base}/index-v2.json`)).json(); + assert.equal(index.apiVersion, "catalog.swiss/v2"); + for (const v of index.models.flatMap((m) => m.variants)) { + const h = await (await fetch(`${base}/${v.history}`)).json(); + assert.deepEqual([h.path, h.revisions[0].digest], [v.path, v.digest]); + } +}); + test("nothing outside models/ is served, and only reads are allowed", async () => { const name = modelNames()[0]; for (const p of ["/package.json", "/models/../package.json", "/models/%2e%2e/catalog.yaml", `/models/${name}/../../catalog.yaml`, "/models/nope/x.yaml"]) {