diff --git a/README.md b/README.md index 58f5d03..af9794e 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ npx @dcl/opscli pointer-consistency --cid "bafkrei..." - `--pointer` Scene pointer coordinate (e.g. `"0,0"`) - `--cid` Entity ID to resolve the pointer from (alternative to `--pointer`) - `--env` Environment: `org` (default), `today`, or `zone` +- `--registry` Which registry to read: `unity` (default) or `abgen` — see [asset bundle registries](#asset-bundle-registries) ### Check world asset bundle status @@ -25,10 +26,43 @@ Check asset bundle conversion status for all scenes in a world. Fetches scenes f ``` npx @dcl/opscli world-ab-status --world "dalkia.dcl.eth" npx @dcl/opscli world-ab-status --world "dalkia.dcl.eth" --env zone +npx @dcl/opscli world-ab-status --world "dalkia.dcl.eth" --registry abgen ``` - `--world` World name (e.g. `"dalkia.dcl.eth"`) - `--env` Environment: `org` (default), `today`, or `zone` +- `--registry` Which registry to read: `unity` (default) or `abgen` — see [asset bundle registries](#asset-bundle-registries) + +### Check the conversion queue + +Report how many jobs are pending per platform queue, and optionally where given entities sit in them. + +``` +npx @dcl/opscli ab-queue-status +npx @dcl/opscli ab-queue-status --env zone --registry abgen +npx @dcl/opscli ab-queue-status --cid "bafkrei..." --cid "bafkrei..." +``` + +- `--env` Environment: `org` (default), `today`, or `zone` +- `--registry` Which registry's queue to read: `unity` (default) or `abgen` +- `--cid` Entity ID to locate in each queue; repeatable. Positions are 1-indexed — the next entity to convert is `#1` + +### Asset bundle registries + +The Unity converters and the abgen generator publish to two separate registries: + +| Registry | Base URL | Environments | +|---|---|---| +| `unity` (default) | `https://asset-bundle-registry.decentraland.` | `org`, `today`, `zone` | +| `abgen` | `https://asset-bundle-registry-abgen.decentraland.` | `org`, `zone` | + +`--registry abgen` on `today` is rejected: abgen has no queue there and the hostname does not resolve. + +The two count bundle versions in **separate series** — abgen is in the `v1000`s while the Unity +converters are in the `v40`s — so an abgen `v1003` is not "newer" than a Unity `v49`. + +Both registries still return a `webgl` platform key, empty or years stale. It is decommissioned and +the status commands do not report it. ### Schedule asset bundle conversion diff --git a/src/bin.ts b/src/bin.ts index 8f595ee..19affb0 100644 --- a/src/bin.ts +++ b/src/bin.ts @@ -15,6 +15,7 @@ import checkAbConversion from './commands/check-ab-conversion' import circleci from './commands/circleci' import concatSnapshots from './commands/concat-snapshots' import worldAbStatus from './commands/world-ab-status' +import abQueueStatus from './commands/ab-queue-status' export class CliError extends Error {} @@ -33,7 +34,8 @@ const commands = { 'check-ab-conversion': checkAbConversion, circleci: circleci, 'concat-snapshots': concatSnapshots, - 'world-ab-status': worldAbStatus + 'world-ab-status': worldAbStatus, + 'ab-queue-status': abQueueStatus } async function main() { diff --git a/src/commands/ab-queue-status.ts b/src/commands/ab-queue-status.ts new file mode 100644 index 0000000..9a0bb56 --- /dev/null +++ b/src/commands/ab-queue-status.ts @@ -0,0 +1,47 @@ +import arg from 'arg' +import { fetch } from 'undici' +import { CliError } from '../bin' +import { REPORTED_PLATFORMS, parseRegistry, registryUrlFor } from '../helpers/ab-registry' + +// `PendingJobs`, each an ordered list of entity IDs waiting for that +// converter queue. +type QueueStatus = Record + +export default async function () { + const args = arg({ + '--env': String, + '--registry': String, + '--cid': [String] + }) + + const env = args['--env'] || 'org' + const registry = parseRegistry(args['--registry'], env) + const cids = args['--cid'] || [] + + const response = await fetch(`${registryUrlFor(registry, env)}/queues/status`) + + if (!response.ok) { + throw new CliError(`! Registry returned ${response.status} for /queues/status`) + } + + const queues = (await response.json()) as QueueStatus + + console.log(`> Queue status [${registry}] (${env})`) + + for (const platform of REPORTED_PLATFORMS) { + const jobs = queues[`${platform}PendingJobs`] + + if (!jobs) { + console.log(` ${platform.padEnd(10)} no queue reported`) + continue + } + + console.log(` ${platform.padEnd(10)} ${jobs.length} pending job(s)`) + + // Positions are 1-indexed: the next entity to be converted is #1. + for (const cid of cids) { + const index = jobs.indexOf(cid) + console.log(` ${cid}: ${index === -1 ? 'not in queue' : `position #${index + 1} of ${jobs.length}`}`) + } + } +} diff --git a/src/commands/pointer-consistency.ts b/src/commands/pointer-consistency.ts index 0e31a74..632d79f 100644 --- a/src/commands/pointer-consistency.ts +++ b/src/commands/pointer-consistency.ts @@ -2,21 +2,26 @@ import arg from 'arg' import { fetch } from 'undici' import { ago } from '../helpers/ago' import { daoCatalysts, fetchEntityByPointer } from '../helpers/catalysts' +import { + REPORTED_PLATFORMS, + RegistryKind, + formatVersion, + parseRegistry, + registryUrlFor, + statusLabel +} from '../helpers/ab-registry' function getEnvConfig(env: string) { - const registryUrl = `https://asset-bundle-registry.decentraland.${env}` - if (env === 'today') { return { catalystUrls: ['https://peer-testing.decentraland.org'], - registryUrl, peerUrl: 'https://peer-testing.decentraland.org' } } if (env === 'zone') { - return { catalystUrls: ['https://peer.decentraland.zone'], registryUrl, peerUrl: 'https://peer.decentraland.zone' } + return { catalystUrls: ['https://peer.decentraland.zone'], peerUrl: 'https://peer.decentraland.zone' } } - return { catalystUrls: null, registryUrl, peerUrl: 'https://peer.decentraland.org' } // null = fetch from DAO + return { catalystUrls: null, peerUrl: 'https://peer.decentraland.org' } // null = fetch from DAO } async function resolvePointerFromCid(peerUrl: string, cid: string): Promise { @@ -41,24 +46,8 @@ async function resolvePointerFromCid(peerUrl: string, cid: string): Promise Asset Bundle Registry (pointer: ${entry.pointers.join(', ')}):`) + console.log(`> Asset Bundle Registry [${kind}] (pointer: ${entry.pointers.join(', ')}):`) console.log(` Entity ID match: ${hashMatch ? '✅' : '❌'} registry=${entry.id}`) console.log(` Global status: ${statusLabel(entry.status)}`) console.log(' Platform status:') - const platforms = ['windows', 'mac'] - for (const platform of platforms) { + for (const platform of REPORTED_PLATFORMS) { const assetStatus = entry.bundles?.assets?.[platform] || 'unknown' - const version = entry.versions?.assets?.[platform] - const versionStr = version ? `v${version.version} (${version.buildDate})` : 'N/A' + const versionStr = formatVersion(entry.versions?.assets?.[platform]) console.log(` ${platform.padEnd(10)} ${statusLabel(assetStatus).padEnd(4)} version: ${versionStr}`) } @@ -115,11 +102,13 @@ export default async function () { const args = arg({ '--pointer': String, '--cid': String, - '--env': String + '--env': String, + '--registry': String }) const env = args['--env'] || 'org' - const { catalystUrls, registryUrl, peerUrl } = getEnvConfig(env) + const registry = parseRegistry(args['--registry'], env) + const { catalystUrls, peerUrl } = getEnvConfig(env) let pointer = args['--pointer'] || (args['--cid'] ? await resolvePointerFromCid(peerUrl, args['--cid']) : null) if (!pointer) { @@ -175,6 +164,6 @@ export default async function () { current.timestamp > latest.timestamp ? current : latest ) console.log(`> Most recent deployment entity ID: ${mostRecent.entityId}`) - await checkAssetBundleStatus(registryUrl, [pointer], mostRecent.entityId) + await checkAssetBundleStatus(registry, registryUrlFor(registry, env), [pointer], mostRecent.entityId) } } diff --git a/src/commands/world-ab-status.ts b/src/commands/world-ab-status.ts index 48ab883..fc76000 100644 --- a/src/commands/world-ab-status.ts +++ b/src/commands/world-ab-status.ts @@ -1,22 +1,6 @@ import arg from 'arg' import { fetch } from 'undici' - -function statusLabel(status: string): string { - switch (status) { - case 'complete': - return '✅ complete' - case 'pending': - return '⏳ pending' - case 'failed': - return '❌ failed' - case 'obsolete': - return '🗑️ obsolete' - case 'fallback': - return '🔄 fallback' - default: - return `⚠️ ${status}` - } -} +import { REPORTED_PLATFORMS, formatVersion, parseRegistry, registryUrlFor, statusLabel } from '../helpers/ab-registry' type Scene = { entityId: string @@ -37,7 +21,8 @@ type RegistryEntity = { export default async function () { const args = arg({ '--world': String, - '--env': String + '--env': String, + '--registry': String }) const world = args['--world'] @@ -46,8 +31,9 @@ export default async function () { } const env = args['--env'] || 'org' + const registry = parseRegistry(args['--registry'], env) const worldsUrl = `https://worlds-content-server.decentraland.${env}` - const registryUrl = `https://asset-bundle-registry.decentraland.${env}` + const registryUrl = registryUrlFor(registry, env) // Step 1: Discover scenes console.log(`> Fetching scenes for world: ${world} (${env})`) @@ -66,24 +52,19 @@ export default async function () { console.log(`> Found ${scenes.length} scene(s)`) - // Collect all pointers and build a map of entityId per pointer + // Collect every parcel of every scene: the registry is queried for all of them at once const allPointers: string[] = [] - const entityIdByPointer = new Map() for (const scene of scenes) { const name = scene.entity.metadata?.display?.title || 'Untitled' const base = scene.entity.metadata?.scene?.base || scene.parcels[0] console.log(`\n Scene: ${name} (base: ${base}, parcels: ${scene.parcels.length})`) console.log(` Entity ID: ${scene.entityId}`) - - for (const pointer of scene.parcels) { - allPointers.push(pointer) - entityIdByPointer.set(pointer, scene.entityId) - } + allPointers.push(...scene.parcels) } // Step 2: Check Asset Bundle Registry - console.log(`\n> Querying asset bundle registry...`) + console.log(`\n> Querying asset bundle registry [${registry}]...`) const registryResponse = await fetch(`${registryUrl}/entities/active?world_name=${encodeURIComponent(world)}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -103,11 +84,6 @@ export default async function () { } // Match registry entities back to scenes - const registryByEntityId = new Map() - for (const entry of registryEntities) { - registryByEntityId.set(entry.id, entry) - } - console.log('') for (const scene of scenes) { const name = scene.entity.metadata?.display?.title || 'Untitled' @@ -129,11 +105,9 @@ export default async function () { console.log(` Entity ID match: ${idMatch ? '✅' : '🚨 STALE — registry has different entity'}`) console.log(` Status: ${statusLabel(registryEntry.status)}`) - const platforms = ['windows', 'mac'] console.log(' Versions:') - for (const platform of platforms) { - const version = registryEntry.versions?.assets?.[platform] - const versionStr = version ? `v${version.version} (${version.buildDate})` : 'N/A' + for (const platform of REPORTED_PLATFORMS) { + const versionStr = formatVersion(registryEntry.versions?.assets?.[platform]) console.log(` ${platform.padEnd(10)} ${versionStr}`) } } diff --git a/src/helpers/ab-registry.ts b/src/helpers/ab-registry.ts new file mode 100644 index 0000000..069b6fa --- /dev/null +++ b/src/helpers/ab-registry.ts @@ -0,0 +1,64 @@ +import { CliError } from '../bin' + +// Which asset bundle registry to read. The abgen generator (the Rust pipeline +// behind `--platform abgen`) publishes to its own registry, not the one the +// Unity converters write to. +export type RegistryKind = 'unity' | 'abgen' + +// abgen only runs where the ab-admin has an abgen queue configured; `today` has +// none, and the registry hostname does not even resolve there. +const ABGEN_ENVS = ['org', 'zone'] + +// webgl is decommissioned, but both registries still return the key (empty, or a +// version years stale), so the status commands filter on this list instead of +// reporting whatever the payload happens to carry. +export const REPORTED_PLATFORMS = ['windows', 'mac'] + +function isRegistryKind(value: string): value is RegistryKind { + return value === 'unity' || value === 'abgen' +} + +export function parseRegistry(value: string | undefined, env: string): RegistryKind { + const kind = value ?? 'unity' + + if (!isRegistryKind(kind)) { + throw new CliError(`! Unknown --registry "${value}". Valid values are "unity" (default) and "abgen".`) + } + + if (kind === 'abgen' && !ABGEN_ENVS.includes(env)) { + throw new CliError( + `! There is no abgen registry on "${env}" — abgen runs on ${ABGEN_ENVS.join(' and ')} only. ` + + `Re-run without --registry abgen to read the Unity registry.` + ) + } + + return kind +} + +export function registryUrlFor(kind: RegistryKind, env: string): string { + const host = kind === 'abgen' ? 'asset-bundle-registry-abgen' : 'asset-bundle-registry' + return `https://${host}.decentraland.${env}` +} + +// The registry already stores the version prefixed ("v49"), so it is printed as +// it comes; an entry with an empty version is a platform that never built. +export function formatVersion(version: { version: string; buildDate: string } | undefined): string { + return version?.version ? `${version.version} (${version.buildDate})` : 'N/A' +} + +export function statusLabel(status: string): string { + switch (status) { + case 'complete': + return '✅ complete' + case 'pending': + return '⏳ pending' + case 'failed': + return '❌ failed' + case 'obsolete': + return '🗑️ obsolete' + case 'fallback': + return '🔄 fallback' + default: + return `⚠️ ${status}` + } +}