diff --git a/apps/dokploy/__test__/nomad/nomad-builder.test.ts b/apps/dokploy/__test__/nomad/nomad-builder.test.ts index 5d98fdecc..391d2d358 100644 --- a/apps/dokploy/__test__/nomad/nomad-builder.test.ts +++ b/apps/dokploy/__test__/nomad/nomad-builder.test.ts @@ -2,7 +2,9 @@ import { applyServiceScalingOverrides, generateNomadJobSpec, getBuildNomadCommand, + getBuildNomadPackCommand, type NomadServiceSpec, + packRenderDir, } from "@nomploy/server/utils/builders/nomad"; import { describe, expect, it } from "vitest"; @@ -701,3 +703,35 @@ describe("nomad builder — shared-mode group autoscaling", () => { expect(hcl).not.toContain("scaling {"); }); }); + +describe("nomad builder — Nomad Pack render command", () => { + const packCompose = { + appName: "mypack", + serverId: null, + composeFile: "", + nomadPack: "grafana", + nomadPackRegistry: null, + nomadPackRef: "abc1234", + // biome-ignore lint/suspicious/noExplicitAny: test mock of NomadComposeNested + } as any; + + it("renders the pack to a dir (single TS registration), not `nomad-pack run`", () => { + const cmd = getBuildNomadPackCommand(packCompose); + // New behavior: render to a dir; the Nomad registration is done in TS + // afterwards (registerRenderedPackJobs) so a canary roll is one clean + // transition instead of two registrations. + expect(cmd).toContain("nomad-pack render grafana"); + expect(cmd).toContain(`--to-dir "${packRenderDir(packCompose)}"`); + expect(cmd).toContain("--auto-approve"); + expect(cmd).toContain("Nomad Pack rendered"); + // Must NOT run (register) the pack from the shell any more. + expect(cmd).not.toContain("nomad-pack run"); + expect(cmd).not.toContain('--name "mypack"'); + }); + + it("packRenderDir sits under the pack's code projectPath", () => { + expect(packRenderDir(packCompose).endsWith("mypack/code/rendered")).toBe( + true, + ); + }); +}); diff --git a/packages/server/src/services/compose.ts b/packages/server/src/services/compose.ts index de2053954..2a94ace97 100644 --- a/packages/server/src/services/compose.ts +++ b/packages/server/src/services/compose.ts @@ -7,12 +7,18 @@ import { cleanAppName, compose, } from "@nomploy/server/db/schema"; -import { monitorNomadRollout } from "@nomploy/server/setup/deploy-monitor"; +import { + monitorNomadRollout, + waitForRollout, +} from "@nomploy/server/setup/deploy-monitor"; import { syncIntentionsForOrg } from "@nomploy/server/setup/nomad-connect"; +import type { PackMetaCtx } from "@nomploy/server/setup/pack-nomad"; import { getBuildComposeCommand } from "@nomploy/server/utils/builders/compose"; import { getBuildNomadCommand, getBuildNomadPackCommand, + type NomadComposeNested, + packRenderDir, } from "@nomploy/server/utils/builders/nomad"; import { randomizeSpecificationFile } from "@nomploy/server/utils/docker/compose"; import { @@ -52,6 +58,59 @@ import { validUniqueServerAppName } from "./project"; export type Compose = typeof compose.$inferSelect; +/** + * Nomad-Pack single registration. The deploy shell has already RENDERED the + * pack's job files (nomad-pack render --to-dir); here we do the ONE Nomad + * registration in TS — ingress tags + canary_tags/shutdown_delay and the panel's + * scaling overrides folded into that single register (buildPackJobPatch), re- + * stamping the pack.* meta so `nomad-pack destroy` still finds the jobs. Then we + * wait for the rollout to go healthy (reusing the detached-path poller). This + * replaces `nomad-pack run` + a post-deploy re-patch, which was TWO registrations + * per deploy and broke canary (each register is its own canary transition → + * consulCatalog 404 window). Throws on a definitive rollout failure so the deploy + * is marked failed; warns-and-continues on timeout. + */ +const registerPackDeployment = async ( + entity: NomadComposeNested, + logPath: string, +): Promise => { + const { registerRenderedPackJobs } = await import( + "@nomploy/server/setup/pack-nomad" + ); + const { buildPackJobPatch } = await import( + "@nomploy/server/setup/pack-domains" + ); + const meta: PackMetaCtx = { + deploymentName: entity.appName, + packName: (entity.nomadPack || "").trim(), + registryName: entity.nomadPackRegistry ? "nomploy-custom" : "default", + version: (entity.nomadPackRef || "").trim() || "latest", + }; + const patch = await buildPackJobPatch(entity); + const ids = await registerRenderedPackJobs( + packRenderDir(entity), + meta, + patch, + ); + const outcome = await waitForRollout(ids, 150_000); + const line = + outcome === "failed" + ? "Nomad Pack rollout FAILED ❌" + : outcome === "ok" + ? "Nomad Pack rollout healthy ✅" + : "Nomad Pack rollout not confirmed within the window (still in progress) — check the dashboard"; + const logCmd = `echo "${encodeBase64(`${line}\n`)}" | base64 -d >> ${logPath}`; + try { + if (entity.serverId) await execAsyncRemote(entity.serverId, logCmd); + else await execAsync(logCmd); + } catch { + // best-effort log line + } + if (outcome === "failed") { + throw new Error("Nomad Pack rollout did not become healthy"); + } +}; + export const createCompose = async ( input: z.infer, ) => { @@ -316,21 +375,14 @@ export const deployCompose = async ({ await execAsync(commandWithLog); } + // Nomad-Pack: the shell only RENDERED the pack; do the single Nomad + // registration (ingress + scaling folded in) and wait for health in TS. + if (compose.composeType === "nomad-pack") { + await registerPackDeployment(entity, deployment.logPath); + } + const finalizeSuccess = async () => { await updateDeploymentStatus(deployment.deploymentId, "done"); - // Nomad-Pack: patch the just-deployed job's service(s) to provider=consul - // + Traefik tags so nomploy's consulCatalog routes the pack's domains - // pool-wide (packs default to Nomad-native registration, invisible to - // consulCatalog). See setup/pack-domains. - if (compose.composeType === "nomad-pack") { - const { applyPackJobPatches } = await import( - "@nomploy/server/setup/pack-domains" - ); - // Domains + scaling (count/resources/autoscaling) in one re-registration. - await applyPackJobPatches(compose).catch((e) => - console.error("pack job patches failed:", e), - ); - } // Phase B: refresh Connect intentions so this project's mesh services // (isolated) get their allow-rules; no-op for non-isolated orgs. if (compose.environment.project.isolated) { @@ -487,21 +539,14 @@ export const rebuildCompose = async ({ await execAsync(commandWithLog); } + // Nomad-Pack: the shell only RENDERED the pack; do the single Nomad + // registration (ingress + scaling folded in) and wait for health in TS. + if (compose.composeType === "nomad-pack") { + await registerPackDeployment(compose, deployment.logPath); + } + const finalizeSuccess = async () => { await updateDeploymentStatus(deployment.deploymentId, "done"); - // Nomad-Pack: patch the just-deployed job's service(s) to provider=consul - // + Traefik tags so nomploy's consulCatalog routes the pack's domains - // pool-wide (packs default to Nomad-native registration, invisible to - // consulCatalog). See setup/pack-domains. - if (compose.composeType === "nomad-pack") { - const { applyPackJobPatches } = await import( - "@nomploy/server/setup/pack-domains" - ); - // Domains + scaling (count/resources/autoscaling) in one re-registration. - await applyPackJobPatches(compose).catch((e) => - console.error("pack job patches failed:", e), - ); - } // Phase B: refresh Connect intentions so this project's mesh services // (isolated) get their allow-rules; no-op for non-isolated orgs. if (compose.environment.project.isolated) { diff --git a/packages/server/src/setup/deploy-monitor.ts b/packages/server/src/setup/deploy-monitor.ts index aad3f1df9..daa9e6c0d 100644 --- a/packages/server/src/setup/deploy-monitor.ts +++ b/packages/server/src/setup/deploy-monitor.ts @@ -92,6 +92,29 @@ const jobState = async (jid: string): Promise => { } }; +/** + * Synchronously poll the given Nomad job id(s) until they reach a terminal + * rollout state: "ok" once every job is healthy, "failed" on a definitive + * failure of any job, or "pending" if the deadline passes while still rolling. + * Reuses the same deployment/allocation signal as monitorNomadRollout + * ({@link jobState}). Used by the Nomad-Pack deploy path, which registers in TS + * (single registration) and then waits here before finalizing. + */ +export const waitForRollout = async ( + ids: string[], + deadlineMs: number = DEFAULT_DEADLINE_MS, +): Promise => { + if (!ids.length) return "ok"; + const deadline = Date.now() + deadlineMs; + while (Date.now() < deadline) { + const states = await Promise.all(ids.map(jobState)); + if (states.some((s) => s === "failed")) return "failed"; + if (states.every((s) => s === "ok")) return "ok"; + await sleep(POLL_INTERVAL_MS); + } + return "pending"; +}; + export interface RolloutMonitorOptions { appName: string; mode: "job" | "pack"; @@ -140,18 +163,7 @@ export const monitorNomadRollout = (opts: RolloutMonitorOptions): void => { ); outcome = "ok"; } else { - while (Date.now() < deadline) { - const states = await Promise.all(ids.map(jobState)); - if (states.some((s) => s === "failed")) { - outcome = "failed"; - break; - } - if (states.every((s) => s === "ok")) { - outcome = "ok"; - break; - } - await sleep(POLL_INTERVAL_MS); - } + outcome = await waitForRollout(ids, deadline - Date.now()); } } catch (err) { // Unexpected monitor error: don't fail an otherwise-registered deploy. diff --git a/packages/server/src/setup/pack-domains.ts b/packages/server/src/setup/pack-domains.ts index 92a24038a..a82858adf 100644 --- a/packages/server/src/setup/pack-domains.ts +++ b/packages/server/src/setup/pack-domains.ts @@ -133,6 +133,13 @@ export const applyDomainsToJob = ( // the roll. canary_tags keeps the old alloc serving until the canary // is healthy and promoted, then Nomad swaps in the real tags. s.CanaryTags = s.Tags.length > 0 ? ["traefik.enable=false"] : []; + // Graceful drain: on promotion the old alloc deregisters from Consul + // then waits before stopping, so consulCatalog/Traefik sees the new + // alloc before the old one disappears (overlap, not a gap). Same as the + // panel job's shutdown_delay. 5s, in nanoseconds; don't override one the + // pack author already set. + if (s.Tags.length > 0 && !s.ShutdownDelay) + s.ShutdownDelay = 5_000_000_000; changed = true; } } @@ -184,3 +191,27 @@ export const applyPackJobPatches = async ( return d || s; }); }; + +/** + * Build the mutator applied to a pack's job(s) at their SINGLE registration + * (registerRenderedPackJobs): the same domains (Traefik tags + canary_tags + + * shutdown_delay) and scaling (count/resources/autoscaling) overrides that + * {@link applyPackJobPatches} used to apply as a second re-registration. Folding + * them into the one register is what keeps a canary roll a single clean + * transition (no 404 window). Resolves domains + the default cert resolver once, + * up front, so the returned patch is synchronous. + */ +export const buildPackJobPatch = async ( + compose: Pick & + PackScalingCompose & { domains?: Domain[] }, +): Promise<(job: NomadJob) => void> => { + const domains = applyDefaultCertResolver( + compose.domains ?? [], + await getDefaultCertResolver().catch(() => "letsencrypt"), + ); + const { applyScalingToJob } = await import("./pack-scaling"); + return (job) => { + applyDomainsToJob(job, compose.appName, domains); + applyScalingToJob(job, compose); + }; +}; diff --git a/packages/server/src/setup/pack-nomad.ts b/packages/server/src/setup/pack-nomad.ts index 3c48bf35a..7e148bac8 100644 --- a/packages/server/src/setup/pack-nomad.ts +++ b/packages/server/src/setup/pack-nomad.ts @@ -4,6 +4,9 @@ // pack-domains (Traefik tags) and pack-scaling (count/resources/autoscaling) // patch the SAME deployed jobs, so the fetch/post/find helpers live here. +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; + // Nomad reads/writes always go to the control plane (not a resource's serverId). const NOMAD = (process.env.NOMAD_ADDRESS || "http://127.0.0.1:4646").replace( /\/$/, @@ -57,6 +60,21 @@ export const nomadDelete = async (path: string): Promise => { } }; +/** Parse a job's HCL into canonical JSON via Nomad's /v1/jobs/parse. */ +export const nomadParseHCL = async (hcl: string): Promise => { + const res = await fetch(`${NOMAD}/v1/jobs/parse`, { + method: "POST", + headers: { "Content-Type": "application/json", ...nomadHeaders() }, + body: JSON.stringify({ JobHCL: hcl, Canonicalize: true }), + }); + if (!res.ok) { + throw new Error( + `Nomad POST ${res.status} on /jobs/parse: ${await res.text().catch(() => "")}`, + ); + } + return res.json() as Promise; +}; + // Free-form Nomad job JSON we read and patch in place. export type NomadJob = any; @@ -106,3 +124,70 @@ export const patchPackJobs = async ( } } }; + +/** The pack.* meta `nomad-pack run` stamps — re-stamped on a direct register so + * `nomad-pack destroy` and getPackJobIds still find the jobs. */ +export interface PackMetaCtx { + deploymentName: string; // pack.deployment_name (== appName) + packName: string; // pack.name + registryName: string; // pack.registry (e.g. "nomploy-custom" / "default") + version: string; // pack.version (the pinned ref, or "latest") +} + +/** Recursively collect *.nomad files under a render output dir. */ +const findNomadFiles = async (dir: string): Promise => { + const out: string[] = []; + // Infer the Dirent[] element type from the call (annotating with + // Awaited> picks the Buffer overload → TS error). + const entries = await readdir(dir, { withFileTypes: true }).catch(() => null); + if (!entries) return out; + for (const e of entries) { + const p = join(dir, e.name); + if (e.isDirectory()) out.push(...(await findNomadFiles(p))); + else if (e.name.endsWith(".nomad")) out.push(p); + } + return out; +}; + +/** + * Deploy a Nomad-Pack in a SINGLE registration: read the pack's rendered job + * files (written by `nomad-pack render --to-dir`), parse each to JSON, re-stamp + * the pack.* meta `nomad-pack run` would add (so `nomad-pack destroy` and + * getPackJobIds still find them), run `patch` on the job (ingress tags, scaling, + * canary_tags, shutdown_delay …), and register it. Returns the job IDs. + * + * Replaces `nomad-pack run` + a post-deploy re-patch, which was TWO registrations + * per deploy — fatal to canary: each registration is its own canary transition, + * and consulCatalog drops a service with no passing instance during the churn → + * a 404 window. One registration = one clean canary. + */ +export const registerRenderedPackJobs = async ( + renderDir: string, + meta: PackMetaCtx, + patch: (job: NomadJob) => void, +): Promise => { + const files = await findNomadFiles(renderDir); + const ids: string[] = []; + for (const file of files) { + const hcl = await readFile(file, "utf8"); + if (!hcl.trim()) continue; + const parsed = await nomadParseHCL(hcl); + if (!parsed || !parsed.ID) continue; + parsed.Meta = { + ...(parsed.Meta ?? {}), + "pack.deployment_name": meta.deploymentName, + "pack.name": meta.packName, + "pack.job": parsed.ID, + "pack.registry": meta.registryName, + "pack.version": meta.version, + }; + try { + patch(parsed); + } catch (e) { + console.error(`registerRenderedPackJobs: patch ${parsed.ID} failed:`, e); + } + await nomadPost("/jobs", { Job: parsed }); + ids.push(parsed.ID); + } + return ids; +}; diff --git a/packages/server/src/utils/builders/nomad.ts b/packages/server/src/utils/builders/nomad.ts index c3b0546cf..931331f19 100644 --- a/packages/server/src/utils/builders/nomad.ts +++ b/packages/server/src/utils/builders/nomad.ts @@ -429,11 +429,25 @@ ${isNativeHcl ? healthCheckSnippet(appName, "job") : ""}` `; }; +/** + * The directory `nomad-pack render --to-dir` writes the pack's rendered job + * files into — the same `//code` projectPath the deploy + * command uses, plus `rendered`. {@link registerRenderedPackJobs} reads these + * back to do the SINGLE Nomad registration (no `nomad-pack run`, so no second + * re-register and no canary churn). + */ +export const packRenderDir = (compose: NomadComposeNested): string => { + const { COMPOSE_PATH } = paths(!!compose.serverId); + return join(COMPOSE_PATH, compose.appName, "code", "rendered"); +}; + /** * Build the deploy command for a Nomad Pack (composeType = "nomad-pack"). * Writes the pack variables (composeFile, HCL) to a var-file, optionally - * registers a custom pack registry, and runs `nomad-pack run`. The deployment is - * named by appName so it maps back to nomploy for status/logs. + * registers a custom pack registry, and RENDERS the pack's job files to a dir + * (via `nomad-pack render --to-dir`). The actual Nomad registration — a single + * one, with ingress/scaling patches folded in — happens in TS afterwards + * (registerRenderedPackJobs), so canary rolls see one clean transition. */ export const getBuildNomadPackCommand = ( compose: NomadComposeNested, @@ -442,6 +456,7 @@ export const getBuildNomadPackCommand = ( const { appName, composeFile, nomadPack, nomadPackRegistry, nomadPackRef } = compose; const projectPath = join(COMPOSE_PATH, appName, "code"); + const renderDir = packRenderDir(compose); const varFile = join(projectPath, `${appName}.vars.hcl`); if (!nomadPack || !nomadPack.trim()) { @@ -485,9 +500,9 @@ set -e { command -v nomad-pack >/dev/null 2>&1 || { echo "Error: nomad-pack is not installed on this host. Nomad Pack deploys run on the control plane — deploy this compose without a specific server, or install nomad-pack on the target."; exit 1; } mkdir -p "${projectPath}" -${writeVars}${addRegistry} nomad-pack run ${nomadPack}${registryFlag}${refFlag}${varFlag} --name "${appName}" 2>&1 - echo "Nomad Pack deployed" -${healthCheckSnippet(appName, "pack")}} || { +${writeVars}${addRegistry} rm -rf "${renderDir}" && nomad-pack render ${nomadPack}${registryFlag}${refFlag}${varFlag} --to-dir "${renderDir}" --auto-approve 2>&1 + echo "Nomad Pack rendered" +} || { echo "Error: Nomad Pack deployment failed" exit 1 }