fix(workspaceindex): skip git-ignored and build-cache dirs in Scan only

internal/repomap/repomap_test.go

@@ -90,7 +90,9 @@ func TestScanBuildsDeterministicSnapshot(t *testing.T) {
 func TestScanDoesNotFollowSymlinkedDirectories(t *testing.T) {
 	root := t.TempDir()
 	writeFile(t, root, "main.go", "package main\n")
-	target := filepath.Join(root, "target")
+	// Not named "target": outside a git work tree Scan skips that name as Cargo build
+	// output, and the real directory must be scanned for this test to exercise the link.
+	target := filepath.Join(root, "real-dir")
 	writeFile(t, target, "hidden.go", "package hidden\n")
 	link := filepath.Join(root, "linked")
 	if err := os.Symlink(target, link); err != nil {
@@ -101,7 +103,7 @@ func TestScanDoesNotFollowSymlinkedDirectories(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Scan: %v", err)
 	}
-	want := []string{"main.go", "target/hidden.go"}
+	want := []string{"main.go", "real-dir/hidden.go"}
 	if !reflect.DeepEqual(pathsOf(got.Files), want) {
 		t.Fatalf("Files=%v want %v", pathsOf(got.Files), want)
 	}

internal/workspaceindex/gitignore.go

@@ -0,0 +1,92 @@
+package workspaceindex
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+	"time"
+)
+
+// gitIgnoredPathsTimeout bounds how long Scan waits on git before giving up and falling
+// back to the non-git behavior. Scan also runs for the per-turn workspace seed, so a slow or
+// degenerate tree (network filesystem, huge monorepo) must not stall a turn.
+const gitIgnoredPathsTimeout = 5 * time.Second
+
+// gitIgnoredPaths asks git which paths under root it considers ignored, honoring every
+// source git itself resolves (nested .gitignore files, .git/info/exclude,
+// core.excludesFile). Returns a set of paths relative to root: whole ignored directories
+// collapsed to one entry plus individually ignored files, using forward slashes and no
+// trailing slash (the `rel` form Scan computes).
+//
+// Returns (nil, false) when root is not inside a git work tree, git is missing or older
+// than the floor below, or a command fails or times out. Scan then behaves as it does
+// outside git. A scan must never fail or hang because of this lookup.
+//
+// Git version floor: 2.16. `status --ignored=<mode>` landed in 2.16 and `--porcelain=v1`
+// in 2.11; an older git rejects the arguments, which lands on the (nil, false) path.
+// GIT_OPTIONAL_LOCKS (2.15) is an environment variable, so an older git ignores it.
+//
+// Why `status --ignored=matching` and not `ls-files --others --ignored --directory`:
+// ls-files stops descending at the first untracked directory, so an ignored directory or
+// file nested inside one (svc/build-out/, app/cache.db) is never reported. status descends,
+// collapses a fully ignored directory to a single entry, and lists partially ignored files
+// individually. `-uno` is not usable as a speedup: with it, status reports no ignored
+// entries at all.
+func gitIgnoredPaths(root string) (map[string]bool, bool) {
+	ctx, cancel := context.WithTimeout(context.Background(), gitIgnoredPathsTimeout)
+	defer cancel()
+
+	// status reports paths relative to the repository top level even when run from a
+	// subdirectory, while Scan's `rel` is relative to root. show-prefix is root's own
+	// location inside the repo ("" at the top level, "svc/" for <repo>/svc), and also the
+	// cheapest way to learn whether this is a git work tree at all.
+	prefixOut, err := gitCommand(ctx, root, "rev-parse", "--show-prefix").Output()
+	if err != nil {
+		return nil, false
+	}
+	prefix := strings.TrimSpace(string(prefixOut))
+
+	// -z: NUL-terminated and unquoted. Without it git C-quotes non-ASCII bytes
+	// (core.quotePath), e.g. "caf\303\251/", which would never match the raw `rel`.
+	// The trailing "." limits the report to root when root is a repository subdirectory.
+	out, err := gitCommand(ctx, root, "status", "--porcelain=v1", "-z", "--ignored=matching", ".").Output()
+	if err != nil {
+		return nil, false
+	}
+
+	paths := make(map[string]bool)
+	for entry := range bytes.SplitSeq(out, []byte{0}) {
+		// Ignored entries are "!! <path>"; untracked "??", changes, and a rename's
+		// second NUL field are skipped.
+		rest, ok := bytes.CutPrefix(entry, []byte("!! "))
+		if !ok {
+			continue
+		}
+		rel, ok := strings.CutPrefix(string(rest), prefix)
+		if !ok {
+			continue
+		}
+		rel = path.Clean(strings.TrimSuffix(rel, "/"))
+		if rel == "." || rel == "" {
+			continue
+		}
+		paths[rel] = true
+	}
+	return paths, true
+}
+
+// gitCommand builds a git invocation with a fixed argv (no shell) rooted at dir.
+//
+// GIT_OPTIONAL_LOCKS=0 stops `git status` from taking .git/index.lock to refresh the
+// index stat cache. Scan runs on every turn, often while the user or the agent is running
+// git in the same repository; an opportunistic lock there can make a concurrent
+// `git commit` or `git add` fail with "index.lock exists".
+func gitCommand(ctx context.Context, dir string, args ...string) *exec.Cmd {
+	// #nosec G204 -- fixed argv, no shell; dir is the caller's own absolute path.
+	cmd := exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...)
+	cmd.Env = append(os.Environ(), "GIT_OPTIONAL_LOCKS=0")
+	return cmd
+}

internal/workspaceindex/gitignore_test.go

@@ -0,0 +1,253 @@
+package workspaceindex
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// isolateGitConfig keeps the git child processes away from the developer's real global and
+// system config: a global core.excludesFile would change what counts as ignored and make
+// these tests depend on the machine. GIT_CONFIG_GLOBAL needs git >= 2.32; HOME and
+// XDG_CONFIG_HOME cover older git and macOS/Windows home resolution.
+func isolateGitConfig(t *testing.T) {
+	t.Helper()
+	home := t.TempDir()
+	t.Setenv("HOME", home)
+	t.Setenv("USERPROFILE", home)
+	t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config"))
+	t.Setenv("GIT_CONFIG_GLOBAL", os.DevNull)
+	t.Setenv("GIT_CONFIG_NOSYSTEM", "1")
+}
+
+// stopGitDiscoveryAbove keeps git from finding an enclosing repository above root, so a
+// TMPDIR that happens to live inside a git work tree cannot turn a "not a repo" fixture
+// into one.
+func stopGitDiscoveryAbove(t *testing.T, root string) {
+	t.Helper()
+	t.Setenv("GIT_CEILING_DIRECTORIES", filepath.Dir(root))
+}
+
+// requireGitIgnoredMatching skips below git 2.16, where `status --ignored=matching` landed.
+// Older git takes Scan's non-git path, which the fallback tests cover separately.
+func requireGitIgnoredMatching(t *testing.T) {
+	t.Helper()
+	out, err := exec.Command("git", "version").Output()
+	if err != nil {
+		t.Skipf("git unavailable: %v", err)
+	}
+	var major, minor int
+	if _, err := fmt.Sscanf(strings.TrimPrefix(strings.TrimSpace(string(out)), "git version "), "%d.%d", &major, &minor); err != nil {
+		t.Skipf("cannot parse %q: %v", out, err)
+	}
+	if major < 2 || (major == 2 && minor < 16) {
+		t.Skipf("needs git >= 2.16 for status --ignored=matching, have %d.%d", major, minor)
+	}
+}
+
+func runGit(t *testing.T, dir string, args ...string) {
+	t.Helper()
+	cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=zero-test", "-c", "user.email=zero-test@example.invalid", "-c", "commit.gpgsign=false"}, args...)...)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("git %v: %v: %s", args, err, out)
+	}
+}
+
+func initGitRepo(t *testing.T, root, gitignore string) {
+	t.Helper()
+	isolateGitConfig(t)
+	requireGitIgnoredMatching(t)
+	stopGitDiscoveryAbove(t, root)
+	runGit(t, root, "init", "-q")
+	if gitignore != "" {
+		writeFile(t, root, ".gitignore", gitignore)
+	}
+}
+
+// A virtualenv or cache directory whose name no fixed list anticipates must still be
+// skipped when the repository's own .gitignore excludes it. Reported against a real repo
+// where `zero repo-map` spent its whole file budget inside a `.locust_env/` virtualenv and
+// never reached the repo's own README.md.
+func TestScanHonorsRepoGitignore(t *testing.T) {
+	root := t.TempDir()
+	initGitRepo(t, root, ".locust_env/\n*.db\ncafé-cache/\n")
+
+	writeFile(t, root, "README.md", "# Example\n")
+	writeFile(t, root, "app/main.py", "print('hi')\n")
+	writeFile(t, root, "app/café.py", "x = 1\n")
+	writeFile(t, root, ".locust_env/lib/python3.13/site-packages/flask/sansio/README.md", "vendored")
+	writeFile(t, root, "audit_results.db", "sqlite")
+	// Ignored file inside an untracked directory: `ls-files --directory` would miss it.
+	writeFile(t, root, "app/cache.db", "sqlite")
+	// Non-ASCII ignored directory: only matches because the lookup uses -z (no C-quoting).
+	writeFile(t, root, "café-cache/blob.json", "{}")
+
+	got, err := Scan(root, Options{MaxDepth: DefaultMaxDepth})
+	if err != nil {
+		t.Fatalf("Scan: %v", err)
+	}
+	want := []string{".gitignore", "README.md", "app/café.py", "app/main.py"}
+	if !reflect.DeepEqual(pathsOf(got.Files), want) {
+		t.Fatalf("Files=%v want %v", pathsOf(got.Files), want)
+	}
+}
+
+// Scanning a repository subdirectory must resolve ignore rules relative to that
+// subdirectory (Scan's `rel`), not relative to the repository top level.
+func TestScanHonorsRepoGitignoreFromSubdirectoryRoot(t *testing.T) {
+	repo := t.TempDir()
+	initGitRepo(t, repo, "build-out/\n")
+	writeFile(t, repo, "svc/handler.go", "package svc\n")
+	writeFile(t, repo, "svc/build-out/gen.go", "package gen\n")
+
+	got, err := Scan(filepath.Join(repo, "svc"), Options{MaxDepth: DefaultMaxDepth})
+	if err != nil {
+		t.Fatalf("Scan: %v", err)
+	}
+	if want := []string{"handler.go"}; !reflect.DeepEqual(pathsOf(got.Files), want) {
+		t.Fatalf("Files=%v want %v", pathsOf(got.Files), want)
+	}
+}
+
+// Inside a git work tree, git decides. A tracked directory that merely shares a name with
+// a build-cache directory (a Go package called "target", say) is real source and must be
+// scanned; the fixed list is only a stand-in for when git cannot be asked.
+func TestScanKeepsTrackedBuildCacheNamedDirInsideGitRepo(t *testing.T) {
+	root := t.TempDir()
+	initGitRepo(t, root, "")
+	writeFile(t, root, "main.go", "package main\n")
+	writeFile(t, root, "target/target.go", "package target\n")
+	runGit(t, root, "add", ".")
+	runGit(t, root, "commit", "-q", "-m", "fixture")
+
+	got, err := Scan(root, Options{MaxDepth: DefaultMaxDepth})
+	if err != nil {
+		t.Fatalf("Scan: %v", err)
+	}
+	if want := []string{"main.go", "target/target.go"}; !reflect.DeepEqual(pathsOf(got.Files), want) {
+		t.Fatalf("Files=%v want %v", pathsOf(got.Files), want)
+	}
+}
+
+// Scanning a directory that the enclosing repository ignores as a whole must still list
+// its contents: git reports the root itself as ignored, which is not a reason to skip
+// everything under it.
+func TestScanOfIgnoredRootStillListsItsFiles(t *testing.T) {
+	repo := t.TempDir()
+	initGitRepo(t, repo, "out/\n")
+	writeFile(t, repo, "out/report.md", "# report\n")
+	writeFile(t, repo, "out/data/rows.csv", "a,b\n")
+
+	got, err := Scan(filepath.Join(repo, "out"), Options{MaxDepth: DefaultMaxDepth})
+	if err != nil {
+		t.Fatalf("Scan: %v", err)
+	}
+	if want := []string{"data/rows.csv", "report.md"}; !reflect.DeepEqual(pathsOf(got.Files), want) {
+		t.Fatalf("Files=%v want %v", pathsOf(got.Files), want)
+	}
+}
+
+// Outside a git work tree, Scan falls back to the fixed build/cache list so the file
+// budget is not spent on build output. Reported against a repo where
+// rust/*/target/debug/.fingerprint alone held hundreds of files: with a small budget the
+// real source must still be reached and the scan must not report truncation.
+func TestScanSkipsBuildCacheDirsOutsideGit(t *testing.T) {
+	isolateGitConfig(t)
+	root := t.TempDir()
+	stopGitDiscoveryAbove(t, root)
+	if _, ok := gitIgnoredPaths(root); ok {
+		t.Fatalf("gitIgnoredPaths(non-repo) reported a work tree")
+	}
+
+	writeFile(t, root, "src/lib.rs", "pub fn f() {}\n")
+	writeFile(t, root, "app/main.py", "print('hi')\n")
+	for i := range 10 {
+		writeFile(t, root, fmt.Sprintf("rust/crate/target/debug/.fingerprint/foo-%d/lib-foo.d", i), "x")
+	}
+	writeFile(t, root, "app/__pycache__/main.cpython-312.pyc", "x")
+	writeFile(t, root, ".venv/lib/site-packages/pkg.py", "x")
+	writeFile(t, root, "Venv/lib/pkg.py", "x")
+	writeFile(t, root, ".terraform/providers/provider.json", "x")
+	writeFile(t, root, ".pytest_cache/v/cache/nodeids", "x")
+	writeFile(t, root, ".mypy_cache/3.12/mod.json", "x")
+	writeFile(t, root, ".ruff_cache/content", "x")
+
+	got, err := Scan(root, Options{MaxFiles: 3, MaxDepth: DefaultMaxDepth})
+	if err != nil {
+		t.Fatalf("Scan: %v", err)
+	}
+	if want := []string{"app/main.py", "src/lib.rs"}; !reflect.DeepEqual(pathsOf(got.Files), want) {
+		t.Fatalf("Files=%v want %v", pathsOf(got.Files), want)
+	}
+	if got.Truncated {
+		t.Fatalf("Truncated=true: build-cache files consumed the file budget")
+	}
+}
+
+// When git cannot run at all, Scan must not error and must behave exactly as outside a
+// repository, even inside one: .gitignore is not consulted, the fixed list applies.
+func TestScanFallsBackWhenGitUnavailable(t *testing.T) {
+	root := t.TempDir()
+	initGitRepo(t, root, ".locust_env/\n")
+	writeFile(t, root, "main.go", "package main\n")
+	writeFile(t, root, ".locust_env/lib/x.py", "kept: git is not available to honor .gitignore")
+	writeFile(t, root, "target/debug/app.d", "skipped by the fixed list")
+
+	t.Setenv("PATH", t.TempDir())
+	if paths, ok := gitIgnoredPaths(root); ok || paths != nil {
+		t.Fatalf("gitIgnoredPaths without git = (%v, %v), want (nil, false)", paths, ok)
+	}
+	got, err := Scan(root, Options{MaxDepth: DefaultMaxDepth})
+	if err != nil {
+		t.Fatalf("Scan: %v", err)
+	}
+	if want := []string{".gitignore", ".locust_env/lib/x.py", "main.go"}; !reflect.DeepEqual(pathsOf(got.Files), want) {
+		t.Fatalf("Files=%v want %v", pathsOf(got.Files), want)
+	}
+}
+
+// ShouldSkipDir is shared with glob, grep, list_directory and path autocomplete. The
+// build/cache names Scan skips must stay visible there, so a question about a dependency in
+// .venv or generated code under target can still be answered.
+func TestShouldSkipDirLeavesScanOnlyDirsVisibleToTools(t *testing.T) {
+	for _, name := range []string{"target", "__pycache__", ".venv", "venv", ".pytest_cache", ".terraform", ".mypy_cache", ".ruff_cache"} {
+		if ShouldSkipDir(name) {
+			t.Fatalf("ShouldSkipDir(%q)=true: tools would stop seeing it", name)
+		}
+		if !isScanOnlySkipDir(name) {
+			t.Fatalf("isScanOnlySkipDir(%q)=false want true", name)
+		}
+	}
+	for _, name := range []string{"Target", " target ", "__PYCACHE__", ".VENV"} {
+		if !isScanOnlySkipDir(name) {
+			t.Fatalf("isScanOnlySkipDir(%q)=false want true (case/whitespace-insensitive)", name)
+		}
+	}
+	for _, name := range []string{"src", "targets", "venv2", "internal"} {
+		if isScanOnlySkipDir(name) {
+			t.Fatalf("isScanOnlySkipDir(%q)=true want false", name)
+		}
+	}
+}
+
+// The ignored-path lookup runs every turn, often next to a git command the user or agent
+// is running in the same repository. It must not take the optional index lock, and a
+// GIT_OPTIONAL_LOCKS value inherited from the environment must not re-enable it.
+func TestGitCommandDisablesOptionalLocks(t *testing.T) {
+	t.Setenv("GIT_OPTIONAL_LOCKS", "1")
+	cmd := gitCommand(context.Background(), t.TempDir(), "status")
+	value := ""
+	for _, entry := range cmd.Env {
+		if after, ok := strings.CutPrefix(entry, "GIT_OPTIONAL_LOCKS="); ok {
+			value = after
+		}
+	}
+	if value != "0" {
+		t.Fatalf("effective GIT_OPTIONAL_LOCKS=%q want \"0\"", value)
+	}
+}