diff --git a/cmd/dbl/service.go b/cmd/dbl/service.go index 9bbc5627..7187ab54 100644 --- a/cmd/dbl/service.go +++ b/cmd/dbl/service.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "path/filepath" "strings" "github.com/netsec-ethz/debuglet/internal/demo" @@ -18,6 +19,7 @@ const serviceUsage = `Usage: [--port 9000] [--grpc-port 9001] [--dispatcher 127.0.0.1:9001] [--dispatcher-http 127.0.0.1:9000] [--start=false] [--enable=false] [--root DIR] [--enrolled-state /var/lib/debuglet/executors/NAME] + [--replace-package /absolute/path/to/installed/full-package] dbl service start|stop|status (--role ROLE) [--name NAME] dbl service uninstall (--role ROLE) [--name NAME] [--purge] dbl service prune --prefix DIR --version VERSION [--component full|cli|dispatcher|executor] [--dry-run] @@ -43,6 +45,11 @@ retention is configured by the host. database already at the named instance's managed directory. Stop its foreground daemon first and do not launch it concurrently with installation. See docs/operations/executor-onboarding.md for the system-account setup sequence. +--replace-package selects a verified full package for an already installed, +enrolled executor. Stop and disable its unit first. Both --start=false and +--enable=false are required; replacement never starts, enrolls or migrates it. +The installed service account and state paths are preserved. Use the new CLI +with an older target directory for rollback, subject to database compatibility. ` // serviceDependencies is private so the command exposes no fault-injection @@ -84,6 +91,7 @@ func productionServiceManager(root string) (service.Manager, error) { // serviceOptions are the flags every service subcommand shares. type serviceOptions struct { Role, Name, Root, User, Group string + ReplacePackage string Purge bool Request service.Request Start, Enable bool @@ -130,6 +138,9 @@ func serviceCommandWith(ctx context.Context, args []string, options globalOption if code, ok := checkStagingRoot(name, serviceUsage, local, fs, subcommand, stderr); !ok { return code } + if code, ok := checkPackageReplacement(name, &local, fs, stderr); !ok { + return code + } installer, code, ok := newInstaller(name, local, deps, stderr) if !ok { return code @@ -139,12 +150,18 @@ func serviceCommandWith(ctx context.Context, args []string, options globalOption switch subcommand { case "install": var assets demo.Assets + local.Request.Role, local.Request.Name = role, local.Name + local.Request.User, local.Request.Group = local.User, local.Group + if local.ReplacePackage != "" { + if assets, err = resolveReplacementAssets(local.ReplacePackage, deps); err == nil { + report, err = installer.ReplacePackage(ctx, local.Request, assets) + } + break + } if local.Root != "" { local.Start, local.Enable = false, false } if assets, err = resolveServiceAssets(deps); err == nil { - local.Request.Role, local.Request.Name = role, local.Name - local.Request.User, local.Request.Group = local.User, local.Group report, err = installer.Install(ctx, local.Request, assets, local.Start, local.Enable) } case "start": @@ -187,6 +204,7 @@ func serviceFlags(fs *flag.FlagSet, local *serviceOptions, subcommand string) { fs.StringVar(&local.Request.DispatcherGRPC, "dispatcher", "", "executor: dispatcher control address as host:port") fs.StringVar(&local.Request.DispatcherHTTP, "dispatcher-http", "", "executor: dispatcher HTTP address as host:port") fs.StringVar(&local.Request.EnrolledState, "enrolled-state", "", "executor: preserve enrolled state at this instance's canonical managed path") + fs.StringVar(&local.ReplacePackage, "replace-package", "", "enrolled executor: select an installed full-package directory while stopped and disabled") fs.BoolVar(&local.Start, "start", true, "start the unit and observe its readiness record") fs.BoolVar(&local.Enable, "enable", true, "let the service manager start the unit at boot") case "uninstall": @@ -215,7 +233,7 @@ func serviceRole(name string, local *serviceOptions, fs *flag.FlagSet, options g blank := false fs.Visit(func(f *flag.Flag) { switch f.Name { - case "name", "root", "user", "group", "dispatcher", "dispatcher-http", "enrolled-state": + case "name", "root", "user", "group", "dispatcher", "dispatcher-http", "enrolled-state", "replace-package": if strings.TrimSpace(f.Value.String()) == "" { blank = true } @@ -238,6 +256,40 @@ func defaultInstanceName(role storagecheck.Role) string { return "worker" } +func checkPackageReplacement(name string, local *serviceOptions, fs *flag.FlagSet, stderr io.Writer) (int, bool) { + if local.ReplacePackage == "" { + return exitOK, true + } + if local.Role != string(storagecheck.Executor) || local.Request.EnrolledState == "" || local.Root != "" { + return usageError(name, serviceUsage, stderr, "--replace-package requires an enrolled executor with --enrolled-state and cannot use --root"), false + } + var startSet, enableSet, userSet, groupSet bool + fs.Visit(func(f *flag.Flag) { + switch f.Name { + case "start": + startSet = true + case "enable": + enableSet = true + case "user": + userSet = true + case "group": + groupSet = true + } + }) + if !startSet || !enableSet || local.Start || local.Enable { + return usageError(name, serviceUsage, stderr, "--replace-package requires explicit --start=false --enable=false"), false + } + // Replacement preserves the recorded service identity when omitted; the + // install command's default account must not overwrite that identity. + if !userSet { + local.User = "" + } + if !groupSet { + local.Group = "" + } + return exitOK, true +} + // checkStagingRoot keeps a staged tree from reaching the host's one service // manager. Only the units at their real paths are that manager's; a staged unit // of the same name would name the production instance, and a staged unit's @@ -293,6 +345,24 @@ func resolveServiceAssets(deps serviceDependencies) (demo.Assets, error) { return deps.resolve(exe) } +func resolveReplacementAssets(target string, deps serviceDependencies) (demo.Assets, error) { + if !filepath.IsAbs(target) { + return demo.Assets{}, fmt.Errorf("--replace-package must name an absolute installed full-package directory") + } + root, err := filepath.EvalSymlinks(target) + if err != nil { + return demo.Assets{}, err + } + assets, err := deps.resolve(filepath.Join(root, "bin", "dbl")) + if err != nil { + return demo.Assets{}, err + } + if assets.Root != root { + return demo.Assets{}, fmt.Errorf("--replace-package selects %s, but bin/dbl resolves to another package at %s", root, assets.Root) + } + return assets, nil +} + // writeServiceReport renders the observation for a reader. The caller emits it // once, whether or not the operation also failed: an operator has to see what // state the host is in either way. diff --git a/cmd/dbl/service_test.go b/cmd/dbl/service_test.go index 494cf2c5..820f92d6 100644 --- a/cmd/dbl/service_test.go +++ b/cmd/dbl/service_test.go @@ -21,8 +21,9 @@ import ( // live manager. Readiness itself is covered where it is implemented, in // internal/demo/service. type recordingManager struct { - calls []string - enabled map[string]bool + calls []string + enabled map[string]bool + unitDirectory string // mainPID, when nonzero, is the process every unit reports running. mainPID int } @@ -60,14 +61,19 @@ func (m *recordingManager) Stop(_ context.Context, unit string) error { func (m *recordingManager) State(_ context.Context, unit string) (service.UnitState, error) { m.calls = append(m.calls, "state "+unit) + fragment := filepath.Join(m.unitDirectory, unit) if m.mainPID != 0 { - return service.UnitState{Loaded: true, Active: "active", Sub: "running", Enabled: m.enabled[unit], MainPID: m.mainPID}, nil + return service.UnitState{Loaded: true, FragmentPath: fragment, Active: "active", Sub: "running", Enabled: m.enabled[unit], MainPID: m.mainPID}, nil } - return service.UnitState{Loaded: true, Active: "inactive", Sub: "dead", Enabled: m.enabled[unit]}, nil + return service.UnitState{Loaded: true, FragmentPath: fragment, Active: "inactive", Sub: "dead", Enabled: m.enabled[unit]}, nil } func serviceTestDependencies(t *testing.T, manager service.Manager, root string) serviceDependencies { t.Helper() + if recorder, ok := manager.(*recordingManager); ok { + // A loaded unit identifies the exact definition the manager read. + recorder.unitDirectory = service.UnitDirectory(root) + } payload := filepath.Join(t.TempDir(), "lib", "debuglet", "9.9.9") return serviceDependencies{ root: root, @@ -229,6 +235,173 @@ func TestServiceInstallRefusalPrintsOnlyTheError(t *testing.T) { } } +func TestServicePackageReplacementRequiresExplicitOfflineEnrollment(t *testing.T) { + base := []string{"install", "--replace-package", "/opt/debuglet/lib/debuglet/2.0.0"} + enrolled := []string{"--role", "executor", "--enrolled-state", "/var/lib/debuglet/executors/worker"} + for name, flags := range map[string][]string{ + "dispatcher": {"--role", "dispatcher", "--enrolled-state", "/var/lib/debuglet/dispatchers/local", "--start=false", "--enable=false"}, + "not enrolled": {"--role", "executor", "--start=false", "--enable=false"}, + "defaults start and enable": enrolled, + "missing enable": append(append([]string{}, enrolled...), "--start=false"), + "missing start": append(append([]string{}, enrolled...), "--enable=false"), + "starts": append(append([]string{}, enrolled...), "--start=true", "--enable=false"), + "enables": append(append([]string{}, enrolled...), "--start=false", "--enable=true"), + "staging": append(append([]string{}, enrolled...), "--start=false", "--enable=false", "--root", t.TempDir()), + "blank target": append(append([]string{}, enrolled...), "--start=false", "--enable=false", "--replace-package", " "), + } { + t.Run(name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + deps := serviceDependencies{ + resolve: func(string) (demo.Assets, error) { + t.Fatal("invalid replacement resolved a package") + return demo.Assets{}, nil + }, + manager: func(string) (service.Manager, error) { + t.Fatal("invalid replacement reached the manager") + return nil, nil + }, + } + if code := serviceCommandWith(t.Context(), append(append([]string{}, base...), flags...), + globalOptions{Output: outputHuman}, &stdout, &stderr, deps); code != exitUsage { + t.Fatalf("exit %d, want usage refusal: %s", code, stderr.String()) + } + }) + } +} + +func TestServicePackageReplacementVerifiesSelectedTarget(t *testing.T) { + for _, version := range []string{"previous", "candidate"} { + t.Run(version, func(t *testing.T) { + target := filepath.Join(t.TempDir(), "lib", "debuglet", version) + if err := os.MkdirAll(target, 0755); err != nil { + t.Fatal(err) + } + manager := newRecordingManager() + deps := serviceTestDependencies(t, manager, t.TempDir()) + deps.executable = func() (string, error) { t.Fatal("replacement selected the CLI's own package"); return "", nil } + called := false + deps.resolve = func(path string) (demo.Assets, error) { + called = true + if path != filepath.Join(target, "bin", "dbl") { + t.Fatalf("resolved %q instead of selected target %q", path, target) + } + return demo.Assets{}, errors.New("selected package verification failed") + } + var stdout, stderr bytes.Buffer + code := serviceCommandWith(t.Context(), []string{"install", "--role", "executor", + "--enrolled-state", "/var/lib/debuglet/executors/worker", "--replace-package", target, + "--start=false", "--enable=false"}, globalOptions{Output: outputHuman}, &stdout, &stderr, deps) + if code != exitFailure || !called || !strings.Contains(stderr.String(), "selected package verification failed") { + t.Fatalf("verification refusal: exit %d, called %v: %s", code, called, stderr.String()) + } + if len(manager.calls) != 0 || stdout.Len() != 0 { + t.Fatalf("unverified target reached service actions: %v, output %s", manager.calls, stdout.String()) + } + }) + } +} + +func TestServicePackageReplacementPreservesOmittedAccountOptions(t *testing.T) { + for name, test := range map[string]struct { + flags []string + user, group string + }{ + "recorded identity": {}, + "explicit user": {flags: []string{"--user", "workeruser"}, user: "workeruser"}, + "explicit group": {flags: []string{"--group", "workergroup"}, group: "workergroup"}, + "explicit identity": {flags: []string{"--user", "workeruser", "--group", "workergroup"}, user: "workeruser", group: "workergroup"}, + } { + t.Run(name, func(t *testing.T) { + fs := newCommandFlagSet("service install") + var local serviceOptions + serviceFlags(fs, &local, "install") + args := []string{"--role", "executor", "--enrolled-state", "/var/lib/debuglet/executors/worker", + "--replace-package", "/opt/debuglet/lib/debuglet/candidate", "--start=false", "--enable=false"} + if err := fs.Parse(append(args, test.flags...)); err != nil { + t.Fatal(err) + } + var stderr bytes.Buffer + if _, ok := checkPackageReplacement("dbl service install", &local, fs, &stderr); !ok { + t.Fatal(stderr.String()) + } + if local.User != test.user || local.Group != test.group { + t.Fatalf("account selection %q:%q, want %q:%q", local.User, local.Group, test.user, test.group) + } + }) + } +} + +func TestServicePackageReplacementDoesNotInstallAMissingInstance(t *testing.T) { + root := t.TempDir() + manager := newRecordingManager() + deps := serviceTestDependencies(t, manager, root) + assets, err := deps.resolve("") + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(assets.Root, 0755); err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + code := serviceCommandWith(t.Context(), []string{"install", "--role", "executor", "--name", "retained", + "--enrolled-state", filepath.Join(root, "var", "lib", "debuglet", "executors", "retained"), + "--replace-package", assets.Root, "--start=false", "--enable=false"}, + globalOptions{Output: outputJSON}, &stdout, &stderr, deps) + var report service.Report + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatal(err) + } + if code != exitFailure || report.Operation != "replace-package" || report.Name != "retained" { + t.Fatalf("replacement route: exit %d, report %+v, error %s", code, report, stderr.String()) + } + if _, err := os.Stat(service.RecordPath(root, storagecheck.Executor, "retained")); !os.IsNotExist(err) { + t.Fatalf("replacement created an installation record: %v", err) + } + if len(manager.calls) != 0 { + t.Fatalf("missing instance reached the manager: %v", manager.calls) + } +} + +func TestServicePackageReplacementRejectsRedirectedExecutable(t *testing.T) { + manager := newRecordingManager() + deps := serviceTestDependencies(t, manager, t.TempDir()) + assets, err := deps.resolve("") + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(assets.CLI), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(assets.CLI, []byte("fixture CLI"), 0755); err != nil { + t.Fatal(err) + } + requested := t.TempDir() + if err := os.Mkdir(filepath.Join(requested, "bin"), 0755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(assets.CLI, filepath.Join(requested, "bin", "dbl")); err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + code := serviceCommandWith(t.Context(), []string{"install", "--role", "executor", + "--enrolled-state", "/var/lib/debuglet/executors/worker", "--replace-package", requested, + "--start=false", "--enable=false"}, globalOptions{Output: outputHuman}, &stdout, &stderr, deps) + if code != exitFailure || !strings.Contains(stderr.String(), "resolves to another package") { + t.Fatalf("redirected target: exit %d, error %s", code, stderr.String()) + } + if stdout.Len() != 0 || len(manager.calls) != 0 { + t.Fatalf("redirected target reached service actions: %s, %v", stdout.String(), manager.calls) + } + alias := filepath.Join(t.TempDir(), "package") + if err := os.Symlink(assets.Root, alias); err != nil { + t.Fatal(err) + } + selected, err := resolveReplacementAssets(alias, deps) + if err != nil || selected.Root != assets.Root { + t.Fatalf("valid directory alias refused: %+v, %v", selected, err) + } +} + func TestServiceStatusOfAnUninstalledInstanceFails(t *testing.T) { var stdout, stderr bytes.Buffer code := serviceCommandWith(context.Background(), diff --git a/docs/cli.md b/docs/cli.md index b9bb2a5e..acdbfde4 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -308,6 +308,17 @@ See [measurement workflows](measurements.md) for a deliberate linked retry and the installed two-executor echo command. Both preserve run identities when a submission or cleanup response is uncertain. +For an already enrolled managed executor, `service install --replace-package PATH` +selects an explicitly verified installed full-package directory. It requires +`--enrolled-state`, a cleanly stopped and disabled unit, and explicit +`--start=false --enable=false`. It preserves the recorded service account and +state without migration or startup. Use a CLI that supports this option even +when selecting an older target package for rollback; see the +[controlled replacement procedure](operations/services.md#replace-an-enrolled-executors-package). +Ordinary enrolled reinstalls retain the recorded package directory and service +account; use replacement for package changes. Counter-mode changes still use +the same package and account. + ## Common options Global options precede the command; command options precede positional arguments. diff --git a/docs/operations/executor-onboarding.md b/docs/operations/executor-onboarding.md index 7d01e7da..f7bff11c 100644 --- a/docs/operations/executor-onboarding.md +++ b/docs/operations/executor-onboarding.md @@ -204,6 +204,13 @@ for stopping, removal and package handling. Back up an enrolled executor by stopping it cleanly and preserving its entire private state directory; the foreground TEST backup command does not include enrolled credentials. +To change the package of this enrolled managed executor, follow +[controlled package replacement](services.md#replace-an-enrolled-executors-package). +It requires a clean joined shutdown, a disabled unit, an offline private backup +and explicit `--replace-package PATH --start=false --enable=false` using a CLI +that supports replacement. It preserves the existing enrollment and service +account; do not obtain another setup token or purge the state to upgrade. + ## Restart and recovery Keep the generated state directory. Restarting with the same configuration, diff --git a/docs/operations/services.md b/docs/operations/services.md index 9ac86fb0..d577b15d 100644 --- a/docs/operations/services.md +++ b/docs/operations/services.md @@ -25,8 +25,12 @@ package versions. Reinstalling a managed role requires the same package version and source revision as its retained installation record, including after normal uninstall. -Changing the package of an existing generated managed role is not supported by -`dbl service install`. The +Enrolled executors also retain their original installed package directory and +service account/group. Repeat any non-default account options when reinstalling. +An already enrolled executor can use the explicit +[package replacement procedure](#replace-an-enrolled-executors-package) below. +Changing the package of a local generated executor or dispatcher remains +unsupported by `dbl service install`. The [Ansible database-upgrade procedure](../../deploy/README.md#upgrading-a-database) for host-administered daemons does not upgrade this managed installation record. Keep its package and private @@ -59,6 +63,77 @@ the capability-free fallback configuration; see [executor setup](executor-onboar Reference unit files live in [`deploy/systemd`](../../deploy/systemd). +## Replace an enrolled executor's package + +This procedure changes only the package used by an existing enrolled managed +executor. It preserves its service account, canonical state path, TLS identity, +configuration, database, retained output and recorded capability profile. It +does not enroll another executor, change the counter mode, migrate a database, +start the service or enable it at boot. Keep other administrators, automation +and foreground executors from starting or changing this instance throughout +maintenance; the command does not serialize concurrent administrator actions. + +Obtain the operator-approved full package and verify its release trust as +described in [signed releases](releases.md). Retain the previous full package. +Stage the candidate without changing command links, then use its CLI explicitly: + +```sh +DEBUGLET_VERSION=VERSION_PROVIDED_BY_OPERATOR +sudo sh ./install.sh --archive "./debuglet-${DEBUGLET_VERSION}-linux-amd64.tar.gz" \ + --checksums ./SHA256SUMS --version "$DEBUGLET_VERSION" --prefix /usr/local --stage-only +candidate="/usr/local/lib/debuglet/$DEBUGLET_VERSION" +replacement_cli="$candidate/bin/dbl" +sudo "$replacement_cli" --output json drain --role executor --name worker +``` + +Continue only after the drain reports `joined: true` and `enabled: false`. +An incomplete or failed shutdown is not permission to replace the package. +While stopped, make a private offline backup of the entire +`/var/lib/debuglet/executors/worker` directory, the installation record at +`/etc/debuglet/services/executor-worker.json`, and the unit at +`/etc/systemd/system/debuglet-executor-worker.service`. Keep the old package's +exact path and version with that backup. These files include private credentials. + +```sh +sudo "$replacement_cli" service install --role executor --name worker \ + --enrolled-state /var/lib/debuglet/executors/worker \ + --replace-package "$candidate" --start=false --enable=false +``` + +`--replace-package` takes an absolute installed full-package directory, not an +archive or executable. The existing package verifier checks its manifest and +files. Omit `--user` and `--group` to retain the recorded identity; supplied +values must match it. The target executor must accept the current database +through its offline `-check-database` check. An incompatible database or an older +target without that check is refused. Plan any required schema migration +separately using the [offline database procedure](../../deploy/README.md#upgrading-a-database); +do not purge state or edit the installation record to bypass a refusal. + +A successful replacement leaves the unit stopped and disabled; default command +links such as `/usr/local/bin/dbl` are unchanged. Keep using the selected CLI +path below. Explicitly resume it, inspect readiness and the connected node's +version, then run one permitted TEST measurement and inspect its complete output: + +```sh +sudo "$replacement_cli" drain --role executor --name worker --resume +sudo "$replacement_cli" service status --role executor --name worker +``` + +After an interrupted replacement, keep the service stopped and repeat the exact +replacement command with the same verified target. Only the matching interrupted +operation can be completed; an unexplained unit/record mismatch is refused. +Do not manually repair the record or start a mismatched unit. + +For a deliberate rollback, drain and back up again, then use this **new CLI** +with `--replace-package /absolute/path/to/the/retained/previous/full-package` +and the same explicit false flags. The old package's CLI need not support this +option. Rollback selects older executable bytes; it keeps the current identity +and state, requires the old target's schema check to pass, and does not replay +work. Do not restore an old database or cryptographic state over work accepted +since that backup. An incompatible schema needs a separate recovery plan, not +an automatic downgrade. Verify readiness and a fresh permitted TEST result +before returning the node to use. + ## Tested managed profile The installed TEST flow was exercised with this one Linux amd64 combination. diff --git a/internal/demo/service/drain.go b/internal/demo/service/drain.go index 31a38627..4c0d5cf1 100644 --- a/internal/demo/service/drain.go +++ b/internal/demo/service/drain.go @@ -184,6 +184,9 @@ func (i *Installer) Resume(ctx context.Context, role storagecheck.Role, name str report.Active, report.Enabled = state.Active, state.Enabled return report, nil } + if err := i.prepareStart(ctx, p); err != nil { + return report, err + } state, err := i.manager.State(ctx, p.Unit) if err != nil { return report, err diff --git a/internal/demo/service/enrolled.go b/internal/demo/service/enrolled.go index a6dca989..a80f33e0 100644 --- a/internal/demo/service/enrolled.go +++ b/internal/demo/service/enrolled.go @@ -19,87 +19,97 @@ import ( // prepareEnrolledState adopts only the canonical, already enrolled directory. // Configuration, credentials and database are never regenerated or relocated. func (i *Installer) prepareEnrolledState(ctx context.Context, p *Profile, account Account) error { + cfg, err := validateEnrolledState(*p) + if err != nil { + return err + } + state, err := i.manager.State(ctx, p.Unit) + if err != nil { + return err + } + // An identical reinstall may observe its own existing managed process; + // first adoption never ignores a PID, including a unit without a record. + ignorePID := 0 + if p.ExecutorID != "" && state.Running() { + if err := i.verifyRemoval(ctx, *p); err != nil { + return err + } + ignorePID = state.MainPID + } + if err := checkEnrolledProcesses("/proc", *p, cfg.Identity.ExecutorID, ignorePID); err != nil { + return err + } + if err := storagecheck.Check(ctx, storagecheck.Executor, p.DatabasePath); err != nil { + return err + } + if err := i.own(p.StateDir, account); err != nil { + return err + } + // Validate any running unit against its recorded capability set before + // adopting the requested counter mode. Keep the saved configuration intact. + p.PacketCapabilities = cfg.Network.PacketCounter == "auto" + p.ExecutorID = cfg.Identity.ExecutorID + p.DispatcherGRPC, p.DispatcherHTTP = cfg.Dispatcher.Addr, cfg.Dispatcher.YamuxAddr + return writeRecord(*p) +} + +// validateEnrolledState reads the saved identity and trust without changing +// ownership, configuration, schema or the administrator's installed record. +func validateEnrolledState(p Profile) (*executorconfig.ExecutorConfig, error) { for path := p.StateDir; path != p.Root && path != filepath.Dir(path); path = filepath.Dir(path) { info, err := os.Lstat(path) if err != nil || !info.IsDir() { - return fmt.Errorf("enrolled state requires a real directory at %s", path) + return nil, fmt.Errorf("enrolled state requires a real directory at %s", path) } } for _, name := range []string{"service.toml", "executor.sqlite", "ca.crt", "executor.crt", "executor.key"} { info, err := os.Lstat(filepath.Join(p.StateDir, name)) if err != nil || !info.Mode().IsRegular() { - return fmt.Errorf("enrolled state requires a regular %s", name) + return nil, fmt.Errorf("enrolled state requires a regular %s", name) } } data, err := os.ReadFile(p.ConfigPath) if err != nil { - return err + return nil, err } cfg, _, err := executorconfig.DecodeConfig(data) if err != nil { - return err + return nil, err } id, err := uuid.Parse(cfg.Identity.ExecutorID) if err != nil || id == uuid.Nil || id.String() != cfg.Identity.ExecutorID || (p.ExecutorID != "" && p.ExecutorID != cfg.Identity.ExecutorID) { - return errors.New("enrolled configuration has an invalid or changed executor identity") + return nil, errors.New("enrolled configuration has an invalid or changed executor identity") } if cfg.TLS.Disable || cfg.Database.Path != p.DatabasePath || cfg.Credentials.CACert != filepath.Join(p.StateDir, "ca.crt") || cfg.Credentials.ClientCert != filepath.Join(p.StateDir, "executor.crt") || cfg.Credentials.ClientKey != filepath.Join(p.StateDir, "executor.key") || cfg.Credentials.EnrollmentToken != "" { - return errors.New("enrolled configuration must retain TLS and its original credentials and database inside the managed directory") + return nil, errors.New("enrolled configuration must retain TLS and its original credentials and database inside the managed directory") } pair, err := tlsfiles.KeyPair("credentials.client_cert", cfg.Credentials.ClientCert, "credentials.client_key", cfg.Credentials.ClientKey, time.Now()) if err != nil { - return err + return nil, err } if pair.Leaf.IsCA || pair.Leaf.Subject.CommonName != cfg.Identity.ExecutorID { - return errors.New("enrolled certificate does not identify this executor") + return nil, errors.New("enrolled certificate does not identify this executor") } roots, err := tlsfiles.TrustRoots("credentials.ca_cert", cfg.Credentials.CACert, time.Now()) if err != nil { - return err + return nil, err } intermediates := x509.NewCertPool() for _, der := range pair.Certificate[1:] { certificate, err := x509.ParseCertificate(der) if err != nil { - return err + return nil, err } intermediates.AddCert(certificate) } if _, err := pair.Leaf.Verify(x509.VerifyOptions{Roots: roots, Intermediates: intermediates, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err != nil { - return fmt.Errorf("verify enrolled certificate: %w", err) - } - state, err := i.manager.State(ctx, p.Unit) - if err != nil { - return err + return nil, fmt.Errorf("verify enrolled certificate: %w", err) } - // An identical reinstall may observe its own existing managed process; - // first adoption never ignores a PID, including a unit without a record. - ignorePID := 0 - if p.ExecutorID != "" && state.Running() { - if err := i.verifyRemoval(ctx, *p); err != nil { - return err - } - ignorePID = state.MainPID - } - if err := checkEnrolledProcesses("/proc", *p, cfg.Identity.ExecutorID, ignorePID); err != nil { - return err - } - if err := storagecheck.Check(ctx, storagecheck.Executor, p.DatabasePath); err != nil { - return err - } - if err := i.own(p.StateDir, account); err != nil { - return err - } - // Validate any running unit against its recorded capability set before - // adopting the requested counter mode. Keep the saved configuration intact. - p.PacketCapabilities = cfg.Network.PacketCounter == "auto" - p.ExecutorID = cfg.Identity.ExecutorID - p.DispatcherGRPC, p.DispatcherHTTP = cfg.Dispatcher.Addr, cfg.Dispatcher.YamuxAddr - return writeRecord(*p) + return cfg, nil } // This observes already-running daemons; administrators must keep foreground diff --git a/internal/demo/service/install.go b/internal/demo/service/install.go index 0d765654..1d0904fb 100644 --- a/internal/demo/service/install.go +++ b/internal/demo/service/install.go @@ -145,6 +145,13 @@ func (i *Installer) Install(ctx context.Context, request Request, assets demo.As // build never upgrades state in place, so the operator is told which // version owns the directory instead of silently adopting it. if existing, err := ReadRecord(i.root, p.Role, p.Name); err == nil { + if existing.Enrolled && (existing.Version != p.Version || existing.SourceSHA != p.SourceSHA || + existing.PayloadRoot != p.PayloadRoot || existing.Executable != p.Executable) { + return report, fmt.Errorf("enrolled executor %s retains package %s (%s, %s); use --replace-package to select another verified payload after draining", existing.Name, existing.PayloadRoot, existing.Version, existing.SourceSHA) + } + if existing.Enrolled && (existing.User != p.User || existing.Group != p.Group) { + return report, fmt.Errorf("enrolled executor %s retains service account %s:%s; reinstall with that account and group", existing.Name, existing.User, existing.Group) + } if existing.Version != p.Version || existing.SourceSHA != p.SourceSHA { return report, fmt.Errorf("%s %s is installed from version %s (%s); reinstall that version, or install this one under a different --name", existing.Role, existing.Name, existing.Version, existing.SourceSHA) @@ -155,6 +162,19 @@ func (i *Installer) Install(ctx context.Context, request Request, assets demo.As if existing.Enrolled != p.Enrolled { return report, errors.New("installation mode differs from retained state; reinstall enrolled executors with --enrolled-state and never replace their configuration") } + if existing.Enrolled { + if err := i.verifyRemoval(ctx, existing); err != nil { + // A same-package counter-mode reinstall records its capability + // choice before writing the unit. Only that one-field + // intermediate may be repaired here; package/path changes + // still require the explicit replacement operation. + previous := existing + previous.PacketCapabilities = !previous.PacketCapabilities + if err := i.verifyRemoval(ctx, previous); err != nil { + return report, err + } + } + } p.ExecutorID = existing.ExecutorID p.PacketCapabilities = existing.PacketCapabilities } else if !errors.Is(err, os.ErrNotExist) { @@ -500,6 +520,9 @@ func (i *Installer) Start(ctx context.Context, role storagecheck.Role, name stri if err != nil { return report, err } + if err := i.prepareStart(ctx, p); err != nil { + return report, err + } if err := os.Remove(filepath.Join(p.StateDir, demo.OfflineStateFile)); err != nil && !errors.Is(err, fs.ErrNotExist) { return report, err } diff --git a/internal/demo/service/replace.go b/internal/demo/service/replace.go new file mode 100644 index 00000000..8df71ecc --- /dev/null +++ b/internal/demo/service/replace.go @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package service + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + + "github.com/netsec-ethz/debuglet/internal/demo" + "github.com/netsec-ethz/debuglet/internal/executor/scheduler/sqlite" + "github.com/netsec-ethz/debuglet/internal/storagecheck" +) + +// ReplacePackage selects an already verified full payload for a drained, +// disabled enrolled executor. The operator must retain a stopped-state backup +// and exclude concurrent administration. No state is migrated, replayed or +// relocated, and no service is started or enabled. +func (i *Installer) ReplacePackage(ctx context.Context, request Request, target demo.Assets) (Report, error) { + old, report, err := i.load(ctx, "replace-package", request.Role, request.Name) + report.State = "incomplete" + if err != nil { + return report, err + } + if request.Role != storagecheck.Executor || !old.Enrolled || old.ExecutorID == "" || request.EnrolledState != old.StateDir { + return report, errors.New("package replacement requires the existing enrolled executor and its canonical --enrolled-state") + } + if (request.User != "" && request.User != old.User) || (request.Group != "" && request.Group != old.Group) { + return report, errors.New("package replacement cannot change the service account or group") + } + request.Root, request.User, request.Group = i.root, old.User, old.Group + selected, err := Resolve(request, target) + if err != nil { + return report, err + } + if !old.SamePaths(selected) { + return report, errors.New("package replacement cannot change managed paths") + } + account, err := i.lookup(old.User, old.Group) + if err != nil { + return report, err + } + if account.UID == 0 || account.GID == 0 { + return report, errors.New("an enrolled executor must run as an unprivileged service account and group") + } + // All non-package fields come from the retained record, never from a new + // default or the selected payload's preferred capability profile. + candidate := old + candidate.Version, candidate.SourceSHA = selected.Version, selected.SourceSHA + candidate.PayloadRoot, candidate.Executable = selected.PayloadRoot, selected.Executable + report.Version, report.PackagePath = candidate.Version, candidate.PayloadRoot + check := func() error { + if err := ctx.Err(); err != nil { + return err + } + state, err := i.manager.State(ctx, old.Unit) + if err != nil { + return err + } + if !state.Loaded || !state.Joined() || state.MainPID != 0 || state.Enabled { + return errors.New("package replacement requires a disabled executor with successful joined shutdown; drain it first") + } + // A crash before committing the record can leave precisely the unit + // requested by this explicit operation. No other mismatch is adopted. + if _, err := os.Lstat(old.UnitPath); err != nil { + return err + } + if err := i.verifyRemoval(ctx, old); err != nil { + if err := i.verifyRemoval(ctx, candidate); err != nil { + return fmt.Errorf("unit matches neither the recorded nor selected package: %w", err) + } + } + return checkEnrolledProcesses("/proc", old, old.ExecutorID, 0) + } + if err := check(); err != nil { + return report, err + } + cfg, err := validateEnrolledState(old) + if err != nil { + return report, err + } + if (cfg.Network.PacketCounter == "auto") != old.PacketCapabilities || + cfg.Dispatcher.Addr != old.DispatcherGRPC || cfg.Dispatcher.YamuxAddr != old.DispatcherHTTP { + return report, errors.New("package replacement cannot change the recorded counter mode or dispatcher; reinstall the same package to apply configuration changes first") + } + // Only the selected daemon knows which schemas it supports. Its existing + // administration flag reads the database before constructing an executor. + checkCtx, cancel := context.WithTimeout(ctx, ReadyTimeout) + defer cancel() + var diagnostic string + err = sqlite.PreserveDatabaseOwnership(old.DatabasePath, func() error { + cmd := exec.CommandContext(checkCtx, candidate.Executable, "-config", old.ConfigPath, "-check-database") + cmd.Env = []string{"LANG=C", "LC_ALL=C", "TZ=UTC"} + // Output retains only a bounded prefix/suffix of stderr on failure. + _, err := cmd.Output() + var failed *exec.ExitError + if errors.As(err, &failed) { + diagnostic = bounded(string(failed.Stderr)) + } + return errors.Join(err, checkCtx.Err()) + }) + if err != nil { + return report, fmt.Errorf("selected package refused the database check; inspect its -check-database result before replacement: %w: %s", err, diagnostic) + } + if err := check(); err != nil { + return report, err + } + report.Joined = true + changed, err := i.writeUnit(candidate) + if err != nil { + return report, err + } + if changed { + report.Changed = append(report.Changed, "unit") + } + // Publish the record last. Reload failure/cancellation leaves the old + // record and exact candidate unit, recoverable only by explicit selection. + if err := i.manager.Reload(ctx); err != nil { + return report, err + } + if err := check(); err != nil { + return report, err + } + if old != candidate { + if err := writeRecord(candidate); err != nil { + return report, err + } + report.Changed = append(report.Changed, "installed record") + } + report.State, report.Active = "installed", "inactive" + report.Note = "selected package installed; executor remains drained and disabled; resume explicitly after reviewing the retained state" + return report, nil +} + +// prepareStart refuses an interrupted replacement before any enabling or +// marker removal. Reload also prevents the manager starting a stale definition +// after the files were committed but a previous process was interrupted. +func (i *Installer) prepareStart(ctx context.Context, p Profile) error { + if _, err := os.Lstat(p.UnitPath); err != nil { + return err + } + if err := i.verifyRemoval(ctx, p); err != nil { + return err + } + return i.manager.Reload(ctx) +} diff --git a/internal/demo/service/replace_test.go b/internal/demo/service/replace_test.go new file mode 100644 index 00000000..e6ba69b8 --- /dev/null +++ b/internal/demo/service/replace_test.go @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package service + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/netsec-ethz/debuglet/internal/demo" + "github.com/netsec-ethz/debuglet/internal/readiness" + "github.com/netsec-ethz/debuglet/internal/storagecheck" + "github.com/netsec-ethz/debuglet/internal/testtls" +) + +func enrolledReplacementFixture(t *testing.T) (*fixture, Profile, Request) { + t.Helper() + f := newFixture(t) + r := Request{Role: storagecheck.Executor, Name: "worker", Root: f.root, + EnrolledState: StateDirectory(f.root, storagecheck.Executor, "worker"), User: "debuglet-alt", Group: "debuglet-alt"} + p, err := Resolve(r, f.assets) + if err != nil { + t.Fatal(err) + } + p.ExecutorID = uuid.NewString() + p.PacketCapabilities = true + p.DispatcherGRPC, p.DispatcherHTTP = "127.0.0.1:9001", "127.0.0.1:9000" + if err := os.MkdirAll(p.StateDir, 0700); err != nil { + t.Fatal(err) + } + if err := storagecheck.BootstrapFresh(t.Context(), storagecheck.Executor, p.DatabasePath); err != nil { + t.Fatal(err) + } + ca, err := testtls.NewAuthority(t.TempDir(), "ca") + if err != nil { + t.Fatal(err) + } + identity, err := ca.Issue(p.ExecutorID, testtls.Options{Client: true}) + if err != nil { + t.Fatal(err) + } + for name, data := range map[string][]byte{"ca.crt": ca.CertPEM, "executor.crt": identity.CertPEM, "executor.key": identity.KeyPEM, "retained-output": []byte("old completed output\n")} { + if err := os.WriteFile(filepath.Join(p.StateDir, name), data, 0600); err != nil { + t.Fatal(err) + } + } + cfg := demo.ExecutorConfiguration(testVersion, p.ExecutorID, p.DatabasePath, readiness.Record{GRPCAddr: p.DispatcherGRPC, HTTPAddr: p.DispatcherHTTP}) + cfg["tls"] = map[string]any{"disable": false} + cfg["credentials"] = map[string]any{"ca_cert": filepath.Join(p.StateDir, "ca.crt"), "client_cert": filepath.Join(p.StateDir, "executor.crt"), "client_key": filepath.Join(p.StateDir, "executor.key")} + cfg["network"].(map[string]any)["packet_counter"] = "auto" + if err := demo.WriteConfig(p.ConfigPath, cfg); err != nil { + t.Fatal(err) + } + if err := writeRecord(p); err != nil { + t.Fatal(err) + } + if _, err := f.installer.writeUnit(p); err != nil { + t.Fatal(err) + } + if err := f.manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + f.manager.unit(p.Unit) + r.User, r.Group = "", "" // Replacement preserves the retained non-default account. + return f, p, r +} + +func replacementTarget(t *testing.T, f *fixture, p Profile, version, body string) demo.Assets { + t.Helper() + a := f.assets + a.Root = filepath.Join(filepath.Dir(a.Root), version) + a.CLI, a.Executor = filepath.Join(a.Root, "bin", "dbl"), filepath.Join(a.Root, "bin", "debuglet-executor") + a.Manifest.Version = version + if err := os.MkdirAll(filepath.Dir(a.Executor), 0755); err != nil { + t.Fatal(err) + } + script := fmt.Sprintf("#!/bin/sh\n[ \"$#\" = 3 ] && [ \"$1\" = -config ] && [ \"$2\" = %q ] && [ \"$3\" = -check-database ] || exit 99\n[ \"$LANG\" = C ] && [ \"$LC_ALL\" = C ] && [ \"$TZ\" = UTC ] && [ -z \"$DEBUGLET_GUEST_WORKER\" ] || exit 98\n%s\n", p.ConfigPath, body) + if err := os.WriteFile(a.Executor, []byte(script), 0755); err != nil { + t.Fatal(err) + } + return a +} + +func replacementSnapshot(t *testing.T, p Profile, metadata bool) map[string]string { + t.Helper() + out := map[string]string{} + paths, err := filepath.Glob(filepath.Join(p.StateDir, "*")) + if err != nil { + t.Fatal(err) + } + if metadata { + paths = append(paths, p.UnitPath, RecordPath(p.Root, p.Role, p.Name)) + } + for _, path := range paths { + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + out[path] = fmt.Sprintf("%o:%s", info.Mode().Perm(), data) + } + return out +} + +func TestReplacePackagePreservesEnrolledStateAndSupportsOldTarget(t *testing.T) { + f, p, r := enrolledReplacementFixture(t) + t.Setenv("DEBUGLET_GUEST_WORKER", "must not reach schema process") + old := replacementTarget(t, f, p, testVersion, "exit 0") + target := replacementTarget(t, f, p, "1.4.3", "exit 0") + before := replacementSnapshot(t, p, false) + for _, a := range []demo.Assets{target, target, old} { + mark := len(f.manager.recorded()) + report, err := f.installer.ReplacePackage(t.Context(), r, a) + if err != nil || report.State != "installed" || report.Enabled || !report.Joined || report.Ready || report.ExecutorID != p.ExecutorID { + t.Fatalf("replacement: %+v, %v", report, err) + } + got, err := ReadRecord(f.root, p.Role, p.Name) + want := p + want.Version, want.SourceSHA, want.PayloadRoot, want.Executable = a.Manifest.Version, a.Manifest.SourceSHA, a.Root, a.Executor + if err != nil || got != want { + t.Fatalf("record: %+v, want %+v, %v", got, want, err) + } + if !reflect.DeepEqual(before, replacementSnapshot(t, p, false)) || len(f.chowned) != 0 { + t.Fatal("replacement modified state or ownership") + } + for _, call := range f.manager.recorded()[mark:] { + if !strings.HasPrefix(call, "state ") && call != "reload" { + t.Fatalf("replacement drove service: %s", call) + } + } + } + // Identical version/revision in another verified installation is a path change. + other := replacementTarget(t, f, p, "other-install", "exit 0") + other.Manifest = old.Manifest + if _, err := f.installer.ReplacePackage(t.Context(), r, other); err != nil { + t.Fatal(err) + } + got, _ := ReadRecord(f.root, p.Role, p.Name) + if got.Executable != other.Executor || got.PayloadRoot != other.Root { + t.Fatal("target path ignored") + } + if _, err := f.installer.Resume(t.Context(), p.Role, p.Name); err != nil { + t.Fatal(err) + } +} + +func TestReplacePackageRefusalsDoNotMutate(t *testing.T) { + for _, name := range []string{"enabled", "running", "failed", "pid", "account", "group", "role", "path", "empty identity", "identity", "capabilities", "unit", "schema", "unknown flag", "cancelled"} { + t.Run(name, func(t *testing.T) { + f, p, r := enrolledReplacementFixture(t) + body := "exit 0" + ctx := t.Context() + switch name { + case "enabled": + f.manager.unit(p.Unit).enabled = true + case "running": + f.manager.unit(p.Unit).active = "active" + case "failed": + f.manager.unit(p.Unit).active = "failed" + case "pid": + f.manager.unit(p.Unit).pid = 123 + case "account": + r.User = "debuglet" + case "group": + r.Group = "debuglet" + case "role": + r.Role = storagecheck.Dispatcher + case "path": + r.EnrolledState = t.TempDir() + case "empty identity": + broken := p + broken.ExecutorID = "" + if err := writeRecord(broken); err != nil { + t.Fatal(err) + } + case "identity": + data, _ := os.ReadFile(p.ConfigPath) + os.WriteFile(p.ConfigPath, []byte(strings.ReplaceAll(string(data), p.ExecutorID, uuid.NewString())), 0600) + case "capabilities": + data, _ := os.ReadFile(p.ConfigPath) + os.WriteFile(p.ConfigPath, []byte(strings.ReplaceAll(string(data), "auto", "fallback")), 0600) + case "unit": + data, _ := os.ReadFile(p.UnitPath) + os.WriteFile(p.UnitPath, append(data, []byte("# unrelated modification\n")...), 0644) + case "schema": + body = "echo 'older target refuses this schema' >&2; exit 3" + case "unknown flag": + body = "echo 'flag provided but not defined' >&2; exit 2" + case "cancelled": + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + } + target := replacementTarget(t, f, p, "1.4.3", body) + before := replacementSnapshot(t, p, true) + _, err := f.installer.ReplacePackage(ctx, r, target) + if err == nil { + t.Fatal("unsafe replacement accepted") + } + if name == "schema" && !strings.Contains(err.Error(), "older target refuses this schema") { + t.Fatalf("target diagnostic lost: %v", err) + } + if name == "cancelled" && !errors.Is(err, context.Canceled) { + t.Fatalf("cancel cause lost: %v", err) + } + if !reflect.DeepEqual(before, replacementSnapshot(t, p, true)) || len(f.chowned) != 0 { + t.Fatal("refusal changed files or ownership") + } + }) + } +} + +func TestReplacePackageInterruptedPublicationAndStartGuards(t *testing.T) { + f, p, r := enrolledReplacementFixture(t) + target := replacementTarget(t, f, p, "1.4.3", "exit 0") + before := replacementSnapshot(t, p, false) + f.manager.reloadErr = errors.New("manager unavailable") + if _, err := f.installer.ReplacePackage(t.Context(), r, target); err == nil { + t.Fatal("reload failure lost") + } + record, _ := ReadRecord(f.root, p.Role, p.Name) + if record != p || !strings.Contains(f.unitText(p.Role, p.Name), target.Executor) { + t.Fatal("interrupted state was not old record/candidate unit") + } + f.manager.reloadErr = nil + marker := filepath.Join(p.StateDir, demo.OfflineStateFile) + if err := os.WriteFile(marker, []byte("retained marker"), 0600); err != nil { + t.Fatal(err) + } + mark := len(f.manager.recorded()) + if _, err := f.installer.Start(t.Context(), p.Role, p.Name); err == nil { + t.Fatal("Start accepted mismatch") + } + if _, err := f.installer.Resume(t.Context(), p.Role, p.Name); err == nil { + t.Fatal("Resume accepted mismatch") + } + if _, err := f.installer.Install(t.Context(), r, f.assets, false, false); err == nil { + t.Fatal("ordinary install repaired an interrupted replacement") + } + if _, err := os.Stat(marker); err != nil { + t.Fatal("refused start removed marker") + } + for _, call := range f.manager.recorded()[mark:] { + if strings.HasPrefix(call, "start ") || strings.HasPrefix(call, "enable ") { + t.Fatalf("mismatch drove service: %s", call) + } + } + if err := os.Remove(marker); err != nil { + t.Fatal(err) + } + other := replacementTarget(t, f, p, "1.4.4", "exit 0") + if _, err := f.installer.ReplacePackage(t.Context(), r, other); err == nil { + t.Fatal("different target adopted interrupted unit") + } + if _, err := f.installer.ReplacePackage(t.Context(), r, target); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before, replacementSnapshot(t, p, false)) { + t.Fatal("interruption/retry changed state") + } + // A completed commit still reloads before starting: the manager may have + // forgotten its cached definition across an administrator interruption. + f.manager.loaded = map[string]bool{} + if _, err := f.installer.Start(t.Context(), p.Role, p.Name); err != nil { + t.Fatal(err) + } +} + +func TestReplacePackageSchemaCheckDeadline(t *testing.T) { + f, p, r := enrolledReplacementFixture(t) + target := replacementTarget(t, f, p, "1.4.3", "exec /bin/sleep 60") + before := replacementSnapshot(t, p, true) + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + began := time.Now() + _, err := f.installer.ReplacePackage(ctx, r, target) + if !errors.Is(err, context.DeadlineExceeded) || time.Since(began) > 2*time.Second { + t.Fatalf("schema check not bounded: %v", err) + } + if !reflect.DeepEqual(before, replacementSnapshot(t, p, true)) { + t.Fatal("canceled check changed files") + } +} + +func TestEnrolledInstallRetriesRecordBeforeUnit(t *testing.T) { + f, p, request := enrolledReplacementFixture(t) + if err := os.Remove(p.UnitPath); err != nil { + t.Fatal(err) + } + request.User, request.Group = p.User, p.Group + if _, err := f.installer.Install(t.Context(), request, f.assets, false, false); err != nil { + t.Fatalf("first adoption record without unit cannot resume: %v", err) + } + if err := f.installer.verifyRemoval(t.Context(), p); err != nil { + t.Fatal(err) + } +} + +func TestEnrolledInstallRetriesCapabilityOnlyInterruption(t *testing.T) { + for _, automatic := range []bool{false, true} { + t.Run(fmt.Sprintf("auto-%t", automatic), func(t *testing.T) { + f, p, request := enrolledReplacementFixture(t) + request.User, request.Group = p.User, p.Group + // Begin with the opposite complete same-package installation. + p.PacketCapabilities = !automatic + if err := writeRecord(p); err != nil { + t.Fatal(err) + } + if _, err := f.installer.writeUnit(p); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(p.ConfigPath) + if err != nil { + t.Fatal(err) + } + if !automatic { + data = regexp.MustCompile(`(?m)^[ \t]*packet_counter[ \t]*=.*$`).ReplaceAll(data, []byte(`packet_counter = "fallback"`)) + } + if err := os.WriteFile(p.ConfigPath, data, 0600); err != nil { + t.Fatal(err) + } + // Exercise the actual ordinary reinstall stage, then interrupt it + // before writeUnit: new capability record, previous capability unit. + if err := f.installer.prepareEnrolledState(t.Context(), &p, Account{UID: 4242, GID: 4343}); err != nil { + t.Fatal(err) + } + if p.PacketCapabilities != automatic { + t.Fatal("counter change was not recorded") + } + if _, err := f.installer.Start(t.Context(), p.Role, p.Name); err == nil { + t.Fatal("Start accepted capability mismatch") + } + if _, err := f.installer.Resume(t.Context(), p.Role, p.Name); err == nil { + t.Fatal("Resume accepted capability mismatch") + } + if _, err := f.installer.Install(t.Context(), request, f.assets, false, false); err != nil { + t.Fatalf("same-package retry failed: %v", err) + } + if err := f.installer.verifyRemoval(t.Context(), p); err != nil { + t.Fatal(err) + } + if _, err := f.installer.Start(t.Context(), p.Role, p.Name); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestEnrolledInstallRefusesPackageAndAccountChanges(t *testing.T) { + for _, changed := range []string{"package path", "account", "group"} { + t.Run(changed, func(t *testing.T) { + f, p, request := enrolledReplacementFixture(t) + request.User, request.Group = p.User, p.Group + target := f.assets + guidance := p.User + ":" + p.Group + switch changed { + case "package path": + target = replacementTarget(t, f, p, "second-prefix", "exit 0") + target.Manifest = f.assets.Manifest + guidance = "--replace-package" + case "account": + request.User = "debuglet" + case "group": + request.Group = "debuglet" + } + before := replacementSnapshot(t, p, true) + _, err := f.installer.Install(t.Context(), request, target, false, false) + if err == nil || !strings.Contains(err.Error(), guidance) { + t.Fatalf("changed %s lacks retention guidance: %v", changed, err) + } + if changed == "package path" && !strings.Contains(err.Error(), p.PayloadRoot) { + t.Fatalf("original package path missing: %v", err) + } + if !reflect.DeepEqual(before, replacementSnapshot(t, p, true)) || len(f.chowned) != 0 { + t.Fatal("refused change modified state or ownership") + } + }) + } +}