Support controlled package replacement for enrolled executors

cmd/dbl/service.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/netsec-ethz/debuglet/internal/demo"
@@ -18,6 +19,7 @@ const serviceUsage = `Usage:
       [--port 9000] [--grpc-port 9001] [--dispatcher 127.0.0.1:9001]
       [--dispatcher-http 127.0.0.1:9000] [--start=false] [--enable=false] [--root DIR]
       [--enrolled-state /var/lib/debuglet/executors/NAME]
+      [--replace-package /absolute/path/to/installed/full-package]
   dbl service start|stop|status (--role ROLE) [--name NAME]
   dbl service uninstall (--role ROLE) [--name NAME] [--purge]
   dbl service prune --prefix DIR --version VERSION [--component full|cli|dispatcher|executor] [--dry-run]
@@ -43,6 +45,11 @@ retention is configured by the host.
 database already at the named instance's managed directory. Stop its foreground
 daemon first and do not launch it concurrently with installation. See
 docs/operations/executor-onboarding.md for the system-account setup sequence.
+--replace-package selects a verified full package for an already installed,
+enrolled executor. Stop and disable its unit first. Both --start=false and
+--enable=false are required; replacement never starts, enrolls or migrates it.
+The installed service account and state paths are preserved. Use the new CLI
+with an older target directory for rollback, subject to database compatibility.
 `
 
 // serviceDependencies is private so the command exposes no fault-injection
@@ -84,6 +91,7 @@ func productionServiceManager(root string) (service.Manager, error) {
 // serviceOptions are the flags every service subcommand shares.
 type serviceOptions struct {
 	Role, Name, Root, User, Group string
+	ReplacePackage                string
 	Purge                         bool
 	Request                       service.Request
 	Start, Enable                 bool
@@ -130,6 +138,9 @@ func serviceCommandWith(ctx context.Context, args []string, options globalOption
 	if code, ok := checkStagingRoot(name, serviceUsage, local, fs, subcommand, stderr); !ok {
 		return code
 	}
+	if code, ok := checkPackageReplacement(name, &local, fs, stderr); !ok {
+		return code
+	}
 	installer, code, ok := newInstaller(name, local, deps, stderr)
 	if !ok {
 		return code
@@ -139,12 +150,18 @@ func serviceCommandWith(ctx context.Context, args []string, options globalOption
 	switch subcommand {
 	case "install":
 		var assets demo.Assets
+		local.Request.Role, local.Request.Name = role, local.Name
+		local.Request.User, local.Request.Group = local.User, local.Group
+		if local.ReplacePackage != "" {
+			if assets, err = resolveReplacementAssets(local.ReplacePackage, deps); err == nil {
+				report, err = installer.ReplacePackage(ctx, local.Request, assets)
+			}
+			break
+		}
 		if local.Root != "" {
 			local.Start, local.Enable = false, false
 		}
 		if assets, err = resolveServiceAssets(deps); err == nil {
-			local.Request.Role, local.Request.Name = role, local.Name
-			local.Request.User, local.Request.Group = local.User, local.Group
 			report, err = installer.Install(ctx, local.Request, assets, local.Start, local.Enable)
 		}
 	case "start":
@@ -187,6 +204,7 @@ func serviceFlags(fs *flag.FlagSet, local *serviceOptions, subcommand string) {
 		fs.StringVar(&local.Request.DispatcherGRPC, "dispatcher", "", "executor: dispatcher control address as host:port")
 		fs.StringVar(&local.Request.DispatcherHTTP, "dispatcher-http", "", "executor: dispatcher HTTP address as host:port")
 		fs.StringVar(&local.Request.EnrolledState, "enrolled-state", "", "executor: preserve enrolled state at this instance's canonical managed path")
+		fs.StringVar(&local.ReplacePackage, "replace-package", "", "enrolled executor: select an installed full-package directory while stopped and disabled")
 		fs.BoolVar(&local.Start, "start", true, "start the unit and observe its readiness record")
 		fs.BoolVar(&local.Enable, "enable", true, "let the service manager start the unit at boot")
 	case "uninstall":
@@ -215,7 +233,7 @@ func serviceRole(name string, local *serviceOptions, fs *flag.FlagSet, options g
 	blank := false
 	fs.Visit(func(f *flag.Flag) {
 		switch f.Name {
-		case "name", "root", "user", "group", "dispatcher", "dispatcher-http", "enrolled-state":
+		case "name", "root", "user", "group", "dispatcher", "dispatcher-http", "enrolled-state", "replace-package":
 			if strings.TrimSpace(f.Value.String()) == "" {
 				blank = true
 			}
@@ -238,6 +256,40 @@ func defaultInstanceName(role storagecheck.Role) string {
 	return "worker"
 }
 
+func checkPackageReplacement(name string, local *serviceOptions, fs *flag.FlagSet, stderr io.Writer) (int, bool) {
+	if local.ReplacePackage == "" {
+		return exitOK, true
+	}
+	if local.Role != string(storagecheck.Executor) || local.Request.EnrolledState == "" || local.Root != "" {
+		return usageError(name, serviceUsage, stderr, "--replace-package requires an enrolled executor with --enrolled-state and cannot use --root"), false
+	}
+	var startSet, enableSet, userSet, groupSet bool
+	fs.Visit(func(f *flag.Flag) {
+		switch f.Name {
+		case "start":
+			startSet = true
+		case "enable":
+			enableSet = true
+		case "user":
+			userSet = true
+		case "group":
+			groupSet = true
+		}
+	})
+	if !startSet || !enableSet || local.Start || local.Enable {
+		return usageError(name, serviceUsage, stderr, "--replace-package requires explicit --start=false --enable=false"), false
+	}
+	// Replacement preserves the recorded service identity when omitted; the
+	// install command's default account must not overwrite that identity.
+	if !userSet {
+		local.User = ""
+	}
+	if !groupSet {
+		local.Group = ""
+	}
+	return exitOK, true
+}
+
 // checkStagingRoot keeps a staged tree from reaching the host's one service
 // manager. Only the units at their real paths are that manager's; a staged unit
 // of the same name would name the production instance, and a staged unit's
@@ -293,6 +345,24 @@ func resolveServiceAssets(deps serviceDependencies) (demo.Assets, error) {
 	return deps.resolve(exe)
 }
 
+func resolveReplacementAssets(target string, deps serviceDependencies) (demo.Assets, error) {
+	if !filepath.IsAbs(target) {
+		return demo.Assets{}, fmt.Errorf("--replace-package must name an absolute installed full-package directory")
+	}
+	root, err := filepath.EvalSymlinks(target)
+	if err != nil {
+		return demo.Assets{}, err
+	}
+	assets, err := deps.resolve(filepath.Join(root, "bin", "dbl"))
+	if err != nil {
+		return demo.Assets{}, err
+	}
+	if assets.Root != root {
+		return demo.Assets{}, fmt.Errorf("--replace-package selects %s, but bin/dbl resolves to another package at %s", root, assets.Root)
+	}
+	return assets, nil
+}
+
 // writeServiceReport renders the observation for a reader. The caller emits it
 // once, whether or not the operation also failed: an operator has to see what
 // state the host is in either way.

cmd/dbl/service_test.go

@@ -21,8 +21,9 @@ import (
 // live manager. Readiness itself is covered where it is implemented, in
 // internal/demo/service.
 type recordingManager struct {
-	calls   []string
-	enabled map[string]bool
+	calls         []string
+	enabled       map[string]bool
+	unitDirectory string
 	// mainPID, when nonzero, is the process every unit reports running.
 	mainPID int
 }
@@ -60,14 +61,19 @@ func (m *recordingManager) Stop(_ context.Context, unit string) error {
 
 func (m *recordingManager) State(_ context.Context, unit string) (service.UnitState, error) {
 	m.calls = append(m.calls, "state "+unit)
+	fragment := filepath.Join(m.unitDirectory, unit)
 	if m.mainPID != 0 {
-		return service.UnitState{Loaded: true, Active: "active", Sub: "running", Enabled: m.enabled[unit], MainPID: m.mainPID}, nil
+		return service.UnitState{Loaded: true, FragmentPath: fragment, Active: "active", Sub: "running", Enabled: m.enabled[unit], MainPID: m.mainPID}, nil
 	}
-	return service.UnitState{Loaded: true, Active: "inactive", Sub: "dead", Enabled: m.enabled[unit]}, nil
+	return service.UnitState{Loaded: true, FragmentPath: fragment, Active: "inactive", Sub: "dead", Enabled: m.enabled[unit]}, nil
 }
 
 func serviceTestDependencies(t *testing.T, manager service.Manager, root string) serviceDependencies {
 	t.Helper()
+	if recorder, ok := manager.(*recordingManager); ok {
+		// A loaded unit identifies the exact definition the manager read.
+		recorder.unitDirectory = service.UnitDirectory(root)
+	}
 	payload := filepath.Join(t.TempDir(), "lib", "debuglet", "9.9.9")
 	return serviceDependencies{
 		root:       root,
@@ -229,6 +235,173 @@ func TestServiceInstallRefusalPrintsOnlyTheError(t *testing.T) {
 	}
 }
 
+func TestServicePackageReplacementRequiresExplicitOfflineEnrollment(t *testing.T) {
+	base := []string{"install", "--replace-package", "/opt/debuglet/lib/debuglet/2.0.0"}
+	enrolled := []string{"--role", "executor", "--enrolled-state", "/var/lib/debuglet/executors/worker"}
+	for name, flags := range map[string][]string{
+		"dispatcher":                {"--role", "dispatcher", "--enrolled-state", "/var/lib/debuglet/dispatchers/local", "--start=false", "--enable=false"},
+		"not enrolled":              {"--role", "executor", "--start=false", "--enable=false"},
+		"defaults start and enable": enrolled,
+		"missing enable":            append(append([]string{}, enrolled...), "--start=false"),
+		"missing start":             append(append([]string{}, enrolled...), "--enable=false"),
+		"starts":                    append(append([]string{}, enrolled...), "--start=true", "--enable=false"),
+		"enables":                   append(append([]string{}, enrolled...), "--start=false", "--enable=true"),
+		"staging":                   append(append([]string{}, enrolled...), "--start=false", "--enable=false", "--root", t.TempDir()),
+		"blank target":              append(append([]string{}, enrolled...), "--start=false", "--enable=false", "--replace-package", " "),
+	} {
+		t.Run(name, func(t *testing.T) {
+			var stdout, stderr bytes.Buffer
+			deps := serviceDependencies{
+				resolve: func(string) (demo.Assets, error) {
+					t.Fatal("invalid replacement resolved a package")
+					return demo.Assets{}, nil
+				},
+				manager: func(string) (service.Manager, error) {
+					t.Fatal("invalid replacement reached the manager")
+					return nil, nil
+				},
+			}
+			if code := serviceCommandWith(t.Context(), append(append([]string{}, base...), flags...),
+				globalOptions{Output: outputHuman}, &stdout, &stderr, deps); code != exitUsage {
+				t.Fatalf("exit %d, want usage refusal: %s", code, stderr.String())
+			}
+		})
+	}
+}
+
+func TestServicePackageReplacementVerifiesSelectedTarget(t *testing.T) {
+	for _, version := range []string{"previous", "candidate"} {
+		t.Run(version, func(t *testing.T) {
+			target := filepath.Join(t.TempDir(), "lib", "debuglet", version)
+			if err := os.MkdirAll(target, 0755); err != nil {
+				t.Fatal(err)
+			}
+			manager := newRecordingManager()
+			deps := serviceTestDependencies(t, manager, t.TempDir())
+			deps.executable = func() (string, error) { t.Fatal("replacement selected the CLI's own package"); return "", nil }
+			called := false
+			deps.resolve = func(path string) (demo.Assets, error) {
+				called = true
+				if path != filepath.Join(target, "bin", "dbl") {
+					t.Fatalf("resolved %q instead of selected target %q", path, target)
+				}
+				return demo.Assets{}, errors.New("selected package verification failed")
+			}
+			var stdout, stderr bytes.Buffer
+			code := serviceCommandWith(t.Context(), []string{"install", "--role", "executor",
+				"--enrolled-state", "/var/lib/debuglet/executors/worker", "--replace-package", target,
+				"--start=false", "--enable=false"}, globalOptions{Output: outputHuman}, &stdout, &stderr, deps)
+			if code != exitFailure || !called || !strings.Contains(stderr.String(), "selected package verification failed") {
+				t.Fatalf("verification refusal: exit %d, called %v: %s", code, called, stderr.String())
+			}
+			if len(manager.calls) != 0 || stdout.Len() != 0 {
+				t.Fatalf("unverified target reached service actions: %v, output %s", manager.calls, stdout.String())
+			}
+		})
+	}
+}
+
+func TestServicePackageReplacementPreservesOmittedAccountOptions(t *testing.T) {
+	for name, test := range map[string]struct {
+		flags       []string
+		user, group string
+	}{
+		"recorded identity": {},
+		"explicit user":     {flags: []string{"--user", "workeruser"}, user: "workeruser"},
+		"explicit group":    {flags: []string{"--group", "workergroup"}, group: "workergroup"},
+		"explicit identity": {flags: []string{"--user", "workeruser", "--group", "workergroup"}, user: "workeruser", group: "workergroup"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			fs := newCommandFlagSet("service install")
+			var local serviceOptions
+			serviceFlags(fs, &local, "install")
+			args := []string{"--role", "executor", "--enrolled-state", "/var/lib/debuglet/executors/worker",
+				"--replace-package", "/opt/debuglet/lib/debuglet/candidate", "--start=false", "--enable=false"}
+			if err := fs.Parse(append(args, test.flags...)); err != nil {
+				t.Fatal(err)
+			}
+			var stderr bytes.Buffer
+			if _, ok := checkPackageReplacement("dbl service install", &local, fs, &stderr); !ok {
+				t.Fatal(stderr.String())
+			}
+			if local.User != test.user || local.Group != test.group {
+				t.Fatalf("account selection %q:%q, want %q:%q", local.User, local.Group, test.user, test.group)
+			}
+		})
+	}
+}
+
+func TestServicePackageReplacementDoesNotInstallAMissingInstance(t *testing.T) {
+	root := t.TempDir()
+	manager := newRecordingManager()
+	deps := serviceTestDependencies(t, manager, root)
+	assets, err := deps.resolve("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(assets.Root, 0755); err != nil {
+		t.Fatal(err)
+	}
+	var stdout, stderr bytes.Buffer
+	code := serviceCommandWith(t.Context(), []string{"install", "--role", "executor", "--name", "retained",
+		"--enrolled-state", filepath.Join(root, "var", "lib", "debuglet", "executors", "retained"),
+		"--replace-package", assets.Root, "--start=false", "--enable=false"},
+		globalOptions{Output: outputJSON}, &stdout, &stderr, deps)
+	var report service.Report
+	if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
+		t.Fatal(err)
+	}
+	if code != exitFailure || report.Operation != "replace-package" || report.Name != "retained" {
+		t.Fatalf("replacement route: exit %d, report %+v, error %s", code, report, stderr.String())
+	}
+	if _, err := os.Stat(service.RecordPath(root, storagecheck.Executor, "retained")); !os.IsNotExist(err) {
+		t.Fatalf("replacement created an installation record: %v", err)
+	}
+	if len(manager.calls) != 0 {
+		t.Fatalf("missing instance reached the manager: %v", manager.calls)
+	}
+}
+
+func TestServicePackageReplacementRejectsRedirectedExecutable(t *testing.T) {
+	manager := newRecordingManager()
+	deps := serviceTestDependencies(t, manager, t.TempDir())
+	assets, err := deps.resolve("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(filepath.Dir(assets.CLI), 0755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(assets.CLI, []byte("fixture CLI"), 0755); err != nil {
+		t.Fatal(err)
+	}
+	requested := t.TempDir()
+	if err := os.Mkdir(filepath.Join(requested, "bin"), 0755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Symlink(assets.CLI, filepath.Join(requested, "bin", "dbl")); err != nil {
+		t.Fatal(err)
+	}
+	var stdout, stderr bytes.Buffer
+	code := serviceCommandWith(t.Context(), []string{"install", "--role", "executor",
+		"--enrolled-state", "/var/lib/debuglet/executors/worker", "--replace-package", requested,
+		"--start=false", "--enable=false"}, globalOptions{Output: outputHuman}, &stdout, &stderr, deps)
+	if code != exitFailure || !strings.Contains(stderr.String(), "resolves to another package") {
+		t.Fatalf("redirected target: exit %d, error %s", code, stderr.String())
+	}
+	if stdout.Len() != 0 || len(manager.calls) != 0 {
+		t.Fatalf("redirected target reached service actions: %s, %v", stdout.String(), manager.calls)
+	}
+	alias := filepath.Join(t.TempDir(), "package")
+	if err := os.Symlink(assets.Root, alias); err != nil {
+		t.Fatal(err)
+	}
+	selected, err := resolveReplacementAssets(alias, deps)
+	if err != nil || selected.Root != assets.Root {
+		t.Fatalf("valid directory alias refused: %+v, %v", selected, err)
+	}
+}
+
 func TestServiceStatusOfAnUninstalledInstanceFails(t *testing.T) {
 	var stdout, stderr bytes.Buffer
 	code := serviceCommandWith(context.Background(),

docs/cli.md

@@ -308,6 +308,17 @@ See [measurement workflows](measurements.md) for a deliberate linked retry and
 the installed two-executor echo command. Both preserve run identities when a
 submission or cleanup response is uncertain.
 
+For an already enrolled managed executor, `service install --replace-package PATH`
+selects an explicitly verified installed full-package directory. It requires
+`--enrolled-state`, a cleanly stopped and disabled unit, and explicit
+`--start=false --enable=false`. It preserves the recorded service account and
+state without migration or startup. Use a CLI that supports this option even
+when selecting an older target package for rollback; see the
+[controlled replacement procedure](operations/services.md#replace-an-enrolled-executors-package).
+Ordinary enrolled reinstalls retain the recorded package directory and service
+account; use replacement for package changes. Counter-mode changes still use
+the same package and account.
+
 ## Common options
 
 Global options precede the command; command options precede positional arguments.

docs/operations/executor-onboarding.md

@@ -204,6 +204,13 @@ for stopping, removal and package handling. Back up an enrolled executor by
 stopping it cleanly and preserving its entire private state directory; the
 foreground TEST backup command does not include enrolled credentials.
 
+To change the package of this enrolled managed executor, follow
+[controlled package replacement](services.md#replace-an-enrolled-executors-package).
+It requires a clean joined shutdown, a disabled unit, an offline private backup
+and explicit `--replace-package PATH --start=false --enable=false` using a CLI
+that supports replacement. It preserves the existing enrollment and service
+account; do not obtain another setup token or purge the state to upgrade.
+
 ## Restart and recovery
 
 Keep the generated state directory. Restarting with the same configuration,