packs: single-registration deploy for zero-downtime canary rolls

apps/dokploy/__test__/nomad/nomad-builder.test.ts

@@ -2,7 +2,9 @@ import {
 	applyServiceScalingOverrides,
 	generateNomadJobSpec,
 	getBuildNomadCommand,
+	getBuildNomadPackCommand,
 	type NomadServiceSpec,
+	packRenderDir,
 } from "@nomploy/server/utils/builders/nomad";
 import { describe, expect, it } from "vitest";
 
@@ -701,3 +703,35 @@ describe("nomad builder — shared-mode group autoscaling", () => {
 		expect(hcl).not.toContain("scaling {");
 	});
 });
+
+describe("nomad builder — Nomad Pack render command", () => {
+	const packCompose = {
+		appName: "mypack",
+		serverId: null,
+		composeFile: "",
+		nomadPack: "grafana",
+		nomadPackRegistry: null,
+		nomadPackRef: "abc1234",
+		// biome-ignore lint/suspicious/noExplicitAny: test mock of NomadComposeNested
+	} as any;
+
+	it("renders the pack to a dir (single TS registration), not `nomad-pack run`", () => {
+		const cmd = getBuildNomadPackCommand(packCompose);
+		// New behavior: render to a dir; the Nomad registration is done in TS
+		// afterwards (registerRenderedPackJobs) so a canary roll is one clean
+		// transition instead of two registrations.
+		expect(cmd).toContain("nomad-pack render grafana");
+		expect(cmd).toContain(`--to-dir "${packRenderDir(packCompose)}"`);
+		expect(cmd).toContain("--auto-approve");
+		expect(cmd).toContain("Nomad Pack rendered");
+		// Must NOT run (register) the pack from the shell any more.
+		expect(cmd).not.toContain("nomad-pack run");
+		expect(cmd).not.toContain('--name "mypack"');
+	});
+
+	it("packRenderDir sits under the pack's code projectPath", () => {
+		expect(packRenderDir(packCompose).endsWith("mypack/code/rendered")).toBe(
+			true,
+		);
+	});
+});

packages/server/src/services/compose.ts

@@ -7,12 +7,18 @@ import {
 	cleanAppName,
 	compose,
 } from "@nomploy/server/db/schema";
-import { monitorNomadRollout } from "@nomploy/server/setup/deploy-monitor";
+import {
+	monitorNomadRollout,
+	waitForRollout,
+} from "@nomploy/server/setup/deploy-monitor";
 import { syncIntentionsForOrg } from "@nomploy/server/setup/nomad-connect";
+import type { PackMetaCtx } from "@nomploy/server/setup/pack-nomad";
 import { getBuildComposeCommand } from "@nomploy/server/utils/builders/compose";
 import {
 	getBuildNomadCommand,
 	getBuildNomadPackCommand,
+	type NomadComposeNested,
+	packRenderDir,
 } from "@nomploy/server/utils/builders/nomad";
 import { randomizeSpecificationFile } from "@nomploy/server/utils/docker/compose";
 import {
@@ -52,6 +58,59 @@ import { validUniqueServerAppName } from "./project";
 
 export type Compose = typeof compose.$inferSelect;
 
+/**
+ * Nomad-Pack single registration. The deploy shell has already RENDERED the
+ * pack's job files (nomad-pack render --to-dir); here we do the ONE Nomad
+ * registration in TS — ingress tags + canary_tags/shutdown_delay and the panel's
+ * scaling overrides folded into that single register (buildPackJobPatch), re-
+ * stamping the pack.* meta so `nomad-pack destroy` still finds the jobs. Then we
+ * wait for the rollout to go healthy (reusing the detached-path poller). This
+ * replaces `nomad-pack run` + a post-deploy re-patch, which was TWO registrations
+ * per deploy and broke canary (each register is its own canary transition →
+ * consulCatalog 404 window). Throws on a definitive rollout failure so the deploy
+ * is marked failed; warns-and-continues on timeout.
+ */
+const registerPackDeployment = async (
+	entity: NomadComposeNested,
+	logPath: string,
+): Promise<void> => {
+	const { registerRenderedPackJobs } = await import(
+		"@nomploy/server/setup/pack-nomad"
+	);
+	const { buildPackJobPatch } = await import(
+		"@nomploy/server/setup/pack-domains"
+	);
+	const meta: PackMetaCtx = {
+		deploymentName: entity.appName,
+		packName: (entity.nomadPack || "").trim(),
+		registryName: entity.nomadPackRegistry ? "nomploy-custom" : "default",
+		version: (entity.nomadPackRef || "").trim() || "latest",
+	};
+	const patch = await buildPackJobPatch(entity);
+	const ids = await registerRenderedPackJobs(
+		packRenderDir(entity),
+		meta,
+		patch,
+	);
+	const outcome = await waitForRollout(ids, 150_000);
+	const line =
+		outcome === "failed"
+			? "Nomad Pack rollout FAILED ❌"
+			: outcome === "ok"
+				? "Nomad Pack rollout healthy ✅"
+				: "Nomad Pack rollout not confirmed within the window (still in progress) — check the dashboard";
+	const logCmd = `echo "${encodeBase64(`${line}\n`)}" | base64 -d >> ${logPath}`;
+	try {
+		if (entity.serverId) await execAsyncRemote(entity.serverId, logCmd);
+		else await execAsync(logCmd);
+	} catch {
+		// best-effort log line
+	}
+	if (outcome === "failed") {
+		throw new Error("Nomad Pack rollout did not become healthy");
+	}
+};
+
 export const createCompose = async (
 	input: z.infer<typeof apiCreateCompose>,
 ) => {
@@ -316,21 +375,14 @@ export const deployCompose = async ({
 			await execAsync(commandWithLog);
 		}
 
+		// Nomad-Pack: the shell only RENDERED the pack; do the single Nomad
+		// registration (ingress + scaling folded in) and wait for health in TS.
+		if (compose.composeType === "nomad-pack") {
+			await registerPackDeployment(entity, deployment.logPath);
+		}
+
 		const finalizeSuccess = async () => {
 			await updateDeploymentStatus(deployment.deploymentId, "done");
-			// Nomad-Pack: patch the just-deployed job's service(s) to provider=consul
-			// + Traefik tags so nomploy's consulCatalog routes the pack's domains
-			// pool-wide (packs default to Nomad-native registration, invisible to
-			// consulCatalog). See setup/pack-domains.
-			if (compose.composeType === "nomad-pack") {
-				const { applyPackJobPatches } = await import(
-					"@nomploy/server/setup/pack-domains"
-				);
-				// Domains + scaling (count/resources/autoscaling) in one re-registration.
-				await applyPackJobPatches(compose).catch((e) =>
-					console.error("pack job patches failed:", e),
-				);
-			}
 			// Phase B: refresh Connect intentions so this project's mesh services
 			// (isolated) get their allow-rules; no-op for non-isolated orgs.
 			if (compose.environment.project.isolated) {
@@ -487,21 +539,14 @@ export const rebuildCompose = async ({
 			await execAsync(commandWithLog);
 		}
 
+		// Nomad-Pack: the shell only RENDERED the pack; do the single Nomad
+		// registration (ingress + scaling folded in) and wait for health in TS.
+		if (compose.composeType === "nomad-pack") {
+			await registerPackDeployment(compose, deployment.logPath);
+		}
+
 		const finalizeSuccess = async () => {
 			await updateDeploymentStatus(deployment.deploymentId, "done");
-			// Nomad-Pack: patch the just-deployed job's service(s) to provider=consul
-			// + Traefik tags so nomploy's consulCatalog routes the pack's domains
-			// pool-wide (packs default to Nomad-native registration, invisible to
-			// consulCatalog). See setup/pack-domains.
-			if (compose.composeType === "nomad-pack") {
-				const { applyPackJobPatches } = await import(
-					"@nomploy/server/setup/pack-domains"
-				);
-				// Domains + scaling (count/resources/autoscaling) in one re-registration.
-				await applyPackJobPatches(compose).catch((e) =>
-					console.error("pack job patches failed:", e),
-				);
-			}
 			// Phase B: refresh Connect intentions so this project's mesh services
 			// (isolated) get their allow-rules; no-op for non-isolated orgs.
 			if (compose.environment.project.isolated) {

packages/server/src/setup/deploy-monitor.ts

@@ -92,6 +92,29 @@ const jobState = async (jid: string): Promise<RolloutState> => {
 	}
 };
 
+/**
+ * Synchronously poll the given Nomad job id(s) until they reach a terminal
+ * rollout state: "ok" once every job is healthy, "failed" on a definitive
+ * failure of any job, or "pending" if the deadline passes while still rolling.
+ * Reuses the same deployment/allocation signal as monitorNomadRollout
+ * ({@link jobState}). Used by the Nomad-Pack deploy path, which registers in TS
+ * (single registration) and then waits here before finalizing.
+ */
+export const waitForRollout = async (
+	ids: string[],
+	deadlineMs: number = DEFAULT_DEADLINE_MS,
+): Promise<RolloutState> => {
+	if (!ids.length) return "ok";
+	const deadline = Date.now() + deadlineMs;
+	while (Date.now() < deadline) {
+		const states = await Promise.all(ids.map(jobState));
+		if (states.some((s) => s === "failed")) return "failed";
+		if (states.every((s) => s === "ok")) return "ok";
+		await sleep(POLL_INTERVAL_MS);
+	}
+	return "pending";
+};
+
 export interface RolloutMonitorOptions {
 	appName: string;
 	mode: "job" | "pack";
@@ -140,18 +163,7 @@ export const monitorNomadRollout = (opts: RolloutMonitorOptions): void => {
 				);
 				outcome = "ok";
 			} else {
-				while (Date.now() < deadline) {
-					const states = await Promise.all(ids.map(jobState));
-					if (states.some((s) => s === "failed")) {
-						outcome = "failed";
-						break;
-					}
-					if (states.every((s) => s === "ok")) {
-						outcome = "ok";
-						break;
-					}
-					await sleep(POLL_INTERVAL_MS);
-				}
+				outcome = await waitForRollout(ids, deadline - Date.now());
 			}
 		} catch (err) {
 			// Unexpected monitor error: don't fail an otherwise-registered deploy.

packages/server/src/setup/pack-domains.ts

@@ -133,6 +133,13 @@ export const applyDomainsToJob = (
 			// the roll. canary_tags keeps the old alloc serving until the canary
 			// is healthy and promoted, then Nomad swaps in the real tags.
 			s.CanaryTags = s.Tags.length > 0 ? ["traefik.enable=false"] : [];
+			// Graceful drain: on promotion the old alloc deregisters from Consul
+			// then waits before stopping, so consulCatalog/Traefik sees the new
+			// alloc before the old one disappears (overlap, not a gap). Same as the
+			// panel job's shutdown_delay. 5s, in nanoseconds; don't override one the
+			// pack author already set.
+			if (s.Tags.length > 0 && !s.ShutdownDelay)
+				s.ShutdownDelay = 5_000_000_000;
 			changed = true;
 		}
 	}
@@ -184,3 +191,27 @@ export const applyPackJobPatches = async (
 		return d || s;
 	});
 };
+
+/**
+ * Build the mutator applied to a pack's job(s) at their SINGLE registration
+ * (registerRenderedPackJobs): the same domains (Traefik tags + canary_tags +
+ * shutdown_delay) and scaling (count/resources/autoscaling) overrides that
+ * {@link applyPackJobPatches} used to apply as a second re-registration. Folding
+ * them into the one register is what keeps a canary roll a single clean
+ * transition (no 404 window). Resolves domains + the default cert resolver once,
+ * up front, so the returned patch is synchronous.
+ */
+export const buildPackJobPatch = async (
+	compose: Pick<Compose, "appName" | "serviceScaling"> &
+		PackScalingCompose & { domains?: Domain[] },
+): Promise<(job: NomadJob) => void> => {
+	const domains = applyDefaultCertResolver(
+		compose.domains ?? [],
+		await getDefaultCertResolver().catch(() => "letsencrypt"),
+	);
+	const { applyScalingToJob } = await import("./pack-scaling");
+	return (job) => {
+		applyDomainsToJob(job, compose.appName, domains);
+		applyScalingToJob(job, compose);
+	};
+};

packages/server/src/setup/pack-nomad.ts

@@ -4,6 +4,9 @@
 // pack-domains (Traefik tags) and pack-scaling (count/resources/autoscaling)
 // patch the SAME deployed jobs, so the fetch/post/find helpers live here.
 
+import { readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+
 // Nomad reads/writes always go to the control plane (not a resource's serverId).
 const NOMAD = (process.env.NOMAD_ADDRESS || "http://127.0.0.1:4646").replace(
 	/\/$/,
@@ -57,6 +60,21 @@ export const nomadDelete = async (path: string): Promise<void> => {
 	}
 };
 
+/** Parse a job's HCL into canonical JSON via Nomad's /v1/jobs/parse. */
+export const nomadParseHCL = async (hcl: string): Promise<NomadJob> => {
+	const res = await fetch(`${NOMAD}/v1/jobs/parse`, {
+		method: "POST",
+		headers: { "Content-Type": "application/json", ...nomadHeaders() },
+		body: JSON.stringify({ JobHCL: hcl, Canonicalize: true }),
+	});
+	if (!res.ok) {
+		throw new Error(
+			`Nomad POST ${res.status} on /jobs/parse: ${await res.text().catch(() => "")}`,
+		);
+	}
+	return res.json() as Promise<NomadJob>;
+};
+
 // Free-form Nomad job JSON we read and patch in place.
 export type NomadJob = any;
 
@@ -106,3 +124,70 @@ export const patchPackJobs = async (
 		}
 	}
 };
+
+/** The pack.* meta `nomad-pack run` stamps — re-stamped on a direct register so
+ * `nomad-pack destroy` and getPackJobIds still find the jobs. */
+export interface PackMetaCtx {
+	deploymentName: string; // pack.deployment_name (== appName)
+	packName: string; // pack.name
+	registryName: string; // pack.registry (e.g. "nomploy-custom" / "default")
+	version: string; // pack.version (the pinned ref, or "latest")
+}
+
+/** Recursively collect *.nomad files under a render output dir. */
+const findNomadFiles = async (dir: string): Promise<string[]> => {
+	const out: string[] = [];
+	// Infer the Dirent[] element type from the call (annotating with
+	// Awaited<ReturnType<typeof readdir>> picks the Buffer overload → TS error).
+	const entries = await readdir(dir, { withFileTypes: true }).catch(() => null);
+	if (!entries) return out;
+	for (const e of entries) {
+		const p = join(dir, e.name);
+		if (e.isDirectory()) out.push(...(await findNomadFiles(p)));
+		else if (e.name.endsWith(".nomad")) out.push(p);
+	}
+	return out;
+};
+
+/**
+ * Deploy a Nomad-Pack in a SINGLE registration: read the pack's rendered job
+ * files (written by `nomad-pack render --to-dir`), parse each to JSON, re-stamp
+ * the pack.* meta `nomad-pack run` would add (so `nomad-pack destroy` and
+ * getPackJobIds still find them), run `patch` on the job (ingress tags, scaling,
+ * canary_tags, shutdown_delay …), and register it. Returns the job IDs.
+ *
+ * Replaces `nomad-pack run` + a post-deploy re-patch, which was TWO registrations
+ * per deploy — fatal to canary: each registration is its own canary transition,
+ * and consulCatalog drops a service with no passing instance during the churn →
+ * a 404 window. One registration = one clean canary.
+ */
+export const registerRenderedPackJobs = async (
+	renderDir: string,
+	meta: PackMetaCtx,
+	patch: (job: NomadJob) => void,
+): Promise<string[]> => {
+	const files = await findNomadFiles(renderDir);
+	const ids: string[] = [];
+	for (const file of files) {
+		const hcl = await readFile(file, "utf8");
+		if (!hcl.trim()) continue;
+		const parsed = await nomadParseHCL(hcl);
+		if (!parsed || !parsed.ID) continue;
+		parsed.Meta = {
+			...(parsed.Meta ?? {}),
+			"pack.deployment_name": meta.deploymentName,
+			"pack.name": meta.packName,
+			"pack.job": parsed.ID,
+			"pack.registry": meta.registryName,
+			"pack.version": meta.version,
+		};
+		try {
+			patch(parsed);
+		} catch (e) {
+			console.error(`registerRenderedPackJobs: patch ${parsed.ID} failed:`, e);
+		}
+		await nomadPost("/jobs", { Job: parsed });
+		ids.push(parsed.ID);
+	}
+	return ids;
+};