diff --git a/internal/repomap/repomap_test.go b/internal/repomap/repomap_test.go index 7809e32ca..629a4ec76 100644 --- a/internal/repomap/repomap_test.go +++ b/internal/repomap/repomap_test.go @@ -90,7 +90,9 @@ func TestScanBuildsDeterministicSnapshot(t *testing.T) { func TestScanDoesNotFollowSymlinkedDirectories(t *testing.T) { root := t.TempDir() writeFile(t, root, "main.go", "package main\n") - target := filepath.Join(root, "target") + // Not named "target": outside a git work tree Scan skips that name as Cargo build + // output, and the real directory must be scanned for this test to exercise the link. + target := filepath.Join(root, "real-dir") writeFile(t, target, "hidden.go", "package hidden\n") link := filepath.Join(root, "linked") if err := os.Symlink(target, link); err != nil { @@ -101,7 +103,7 @@ func TestScanDoesNotFollowSymlinkedDirectories(t *testing.T) { if err != nil { t.Fatalf("Scan: %v", err) } - want := []string{"main.go", "target/hidden.go"} + want := []string{"main.go", "real-dir/hidden.go"} if !reflect.DeepEqual(pathsOf(got.Files), want) { t.Fatalf("Files=%v want %v", pathsOf(got.Files), want) } diff --git a/internal/workspaceindex/gitignore.go b/internal/workspaceindex/gitignore.go new file mode 100644 index 000000000..f8eff94c4 --- /dev/null +++ b/internal/workspaceindex/gitignore.go @@ -0,0 +1,92 @@ +package workspaceindex + +import ( + "bytes" + "context" + "os" + "os/exec" + "path" + "strings" + "time" +) + +// gitIgnoredPathsTimeout bounds how long Scan waits on git before giving up and falling +// back to the non-git behavior. Scan also runs for the per-turn workspace seed, so a slow or +// degenerate tree (network filesystem, huge monorepo) must not stall a turn. +const gitIgnoredPathsTimeout = 5 * time.Second + +// gitIgnoredPaths asks git which paths under root it considers ignored, honoring every +// source git itself resolves (nested .gitignore files, .git/info/exclude, +// core.excludesFile). Returns a set of paths relative to root: whole ignored directories +// collapsed to one entry plus individually ignored files, using forward slashes and no +// trailing slash (the `rel` form Scan computes). +// +// Returns (nil, false) when root is not inside a git work tree, git is missing or older +// than the floor below, or a command fails or times out. Scan then behaves as it does +// outside git. A scan must never fail or hang because of this lookup. +// +// Git version floor: 2.16. `status --ignored=` landed in 2.16 and `--porcelain=v1` +// in 2.11; an older git rejects the arguments, which lands on the (nil, false) path. +// GIT_OPTIONAL_LOCKS (2.15) is an environment variable, so an older git ignores it. +// +// Why `status --ignored=matching` and not `ls-files --others --ignored --directory`: +// ls-files stops descending at the first untracked directory, so an ignored directory or +// file nested inside one (svc/build-out/, app/cache.db) is never reported. status descends, +// collapses a fully ignored directory to a single entry, and lists partially ignored files +// individually. `-uno` is not usable as a speedup: with it, status reports no ignored +// entries at all. +func gitIgnoredPaths(root string) (map[string]bool, bool) { + ctx, cancel := context.WithTimeout(context.Background(), gitIgnoredPathsTimeout) + defer cancel() + + // status reports paths relative to the repository top level even when run from a + // subdirectory, while Scan's `rel` is relative to root. show-prefix is root's own + // location inside the repo ("" at the top level, "svc/" for /svc), and also the + // cheapest way to learn whether this is a git work tree at all. + prefixOut, err := gitCommand(ctx, root, "rev-parse", "--show-prefix").Output() + if err != nil { + return nil, false + } + prefix := strings.TrimSpace(string(prefixOut)) + + // -z: NUL-terminated and unquoted. Without it git C-quotes non-ASCII bytes + // (core.quotePath), e.g. "caf\303\251/", which would never match the raw `rel`. + // The trailing "." limits the report to root when root is a repository subdirectory. + out, err := gitCommand(ctx, root, "status", "--porcelain=v1", "-z", "--ignored=matching", ".").Output() + if err != nil { + return nil, false + } + + paths := make(map[string]bool) + for entry := range bytes.SplitSeq(out, []byte{0}) { + // Ignored entries are "!! "; untracked "??", changes, and a rename's + // second NUL field are skipped. + rest, ok := bytes.CutPrefix(entry, []byte("!! ")) + if !ok { + continue + } + rel, ok := strings.CutPrefix(string(rest), prefix) + if !ok { + continue + } + rel = path.Clean(strings.TrimSuffix(rel, "/")) + if rel == "." || rel == "" { + continue + } + paths[rel] = true + } + return paths, true +} + +// gitCommand builds a git invocation with a fixed argv (no shell) rooted at dir. +// +// GIT_OPTIONAL_LOCKS=0 stops `git status` from taking .git/index.lock to refresh the +// index stat cache. Scan runs on every turn, often while the user or the agent is running +// git in the same repository; an opportunistic lock there can make a concurrent +// `git commit` or `git add` fail with "index.lock exists". +func gitCommand(ctx context.Context, dir string, args ...string) *exec.Cmd { + // #nosec G204 -- fixed argv, no shell; dir is the caller's own absolute path. + cmd := exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...) + cmd.Env = append(os.Environ(), "GIT_OPTIONAL_LOCKS=0") + return cmd +} diff --git a/internal/workspaceindex/gitignore_test.go b/internal/workspaceindex/gitignore_test.go new file mode 100644 index 000000000..589ec6cef --- /dev/null +++ b/internal/workspaceindex/gitignore_test.go @@ -0,0 +1,253 @@ +package workspaceindex + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" +) + +// isolateGitConfig keeps the git child processes away from the developer's real global and +// system config: a global core.excludesFile would change what counts as ignored and make +// these tests depend on the machine. GIT_CONFIG_GLOBAL needs git >= 2.32; HOME and +// XDG_CONFIG_HOME cover older git and macOS/Windows home resolution. +func isolateGitConfig(t *testing.T) { + t.Helper() + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + t.Setenv("GIT_CONFIG_GLOBAL", os.DevNull) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") +} + +// stopGitDiscoveryAbove keeps git from finding an enclosing repository above root, so a +// TMPDIR that happens to live inside a git work tree cannot turn a "not a repo" fixture +// into one. +func stopGitDiscoveryAbove(t *testing.T, root string) { + t.Helper() + t.Setenv("GIT_CEILING_DIRECTORIES", filepath.Dir(root)) +} + +// requireGitIgnoredMatching skips below git 2.16, where `status --ignored=matching` landed. +// Older git takes Scan's non-git path, which the fallback tests cover separately. +func requireGitIgnoredMatching(t *testing.T) { + t.Helper() + out, err := exec.Command("git", "version").Output() + if err != nil { + t.Skipf("git unavailable: %v", err) + } + var major, minor int + if _, err := fmt.Sscanf(strings.TrimPrefix(strings.TrimSpace(string(out)), "git version "), "%d.%d", &major, &minor); err != nil { + t.Skipf("cannot parse %q: %v", out, err) + } + if major < 2 || (major == 2 && minor < 16) { + t.Skipf("needs git >= 2.16 for status --ignored=matching, have %d.%d", major, minor) + } +} + +func runGit(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=zero-test", "-c", "user.email=zero-test@example.invalid", "-c", "commit.gpgsign=false"}, args...)...) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } +} + +func initGitRepo(t *testing.T, root, gitignore string) { + t.Helper() + isolateGitConfig(t) + requireGitIgnoredMatching(t) + stopGitDiscoveryAbove(t, root) + runGit(t, root, "init", "-q") + if gitignore != "" { + writeFile(t, root, ".gitignore", gitignore) + } +} + +// A virtualenv or cache directory whose name no fixed list anticipates must still be +// skipped when the repository's own .gitignore excludes it. Reported against a real repo +// where `zero repo-map` spent its whole file budget inside a `.locust_env/` virtualenv and +// never reached the repo's own README.md. +func TestScanHonorsRepoGitignore(t *testing.T) { + root := t.TempDir() + initGitRepo(t, root, ".locust_env/\n*.db\ncafé-cache/\n") + + writeFile(t, root, "README.md", "# Example\n") + writeFile(t, root, "app/main.py", "print('hi')\n") + writeFile(t, root, "app/café.py", "x = 1\n") + writeFile(t, root, ".locust_env/lib/python3.13/site-packages/flask/sansio/README.md", "vendored") + writeFile(t, root, "audit_results.db", "sqlite") + // Ignored file inside an untracked directory: `ls-files --directory` would miss it. + writeFile(t, root, "app/cache.db", "sqlite") + // Non-ASCII ignored directory: only matches because the lookup uses -z (no C-quoting). + writeFile(t, root, "café-cache/blob.json", "{}") + + got, err := Scan(root, Options{MaxDepth: DefaultMaxDepth}) + if err != nil { + t.Fatalf("Scan: %v", err) + } + want := []string{".gitignore", "README.md", "app/café.py", "app/main.py"} + if !reflect.DeepEqual(pathsOf(got.Files), want) { + t.Fatalf("Files=%v want %v", pathsOf(got.Files), want) + } +} + +// Scanning a repository subdirectory must resolve ignore rules relative to that +// subdirectory (Scan's `rel`), not relative to the repository top level. +func TestScanHonorsRepoGitignoreFromSubdirectoryRoot(t *testing.T) { + repo := t.TempDir() + initGitRepo(t, repo, "build-out/\n") + writeFile(t, repo, "svc/handler.go", "package svc\n") + writeFile(t, repo, "svc/build-out/gen.go", "package gen\n") + + got, err := Scan(filepath.Join(repo, "svc"), Options{MaxDepth: DefaultMaxDepth}) + if err != nil { + t.Fatalf("Scan: %v", err) + } + if want := []string{"handler.go"}; !reflect.DeepEqual(pathsOf(got.Files), want) { + t.Fatalf("Files=%v want %v", pathsOf(got.Files), want) + } +} + +// Inside a git work tree, git decides. A tracked directory that merely shares a name with +// a build-cache directory (a Go package called "target", say) is real source and must be +// scanned; the fixed list is only a stand-in for when git cannot be asked. +func TestScanKeepsTrackedBuildCacheNamedDirInsideGitRepo(t *testing.T) { + root := t.TempDir() + initGitRepo(t, root, "") + writeFile(t, root, "main.go", "package main\n") + writeFile(t, root, "target/target.go", "package target\n") + runGit(t, root, "add", ".") + runGit(t, root, "commit", "-q", "-m", "fixture") + + got, err := Scan(root, Options{MaxDepth: DefaultMaxDepth}) + if err != nil { + t.Fatalf("Scan: %v", err) + } + if want := []string{"main.go", "target/target.go"}; !reflect.DeepEqual(pathsOf(got.Files), want) { + t.Fatalf("Files=%v want %v", pathsOf(got.Files), want) + } +} + +// Scanning a directory that the enclosing repository ignores as a whole must still list +// its contents: git reports the root itself as ignored, which is not a reason to skip +// everything under it. +func TestScanOfIgnoredRootStillListsItsFiles(t *testing.T) { + repo := t.TempDir() + initGitRepo(t, repo, "out/\n") + writeFile(t, repo, "out/report.md", "# report\n") + writeFile(t, repo, "out/data/rows.csv", "a,b\n") + + got, err := Scan(filepath.Join(repo, "out"), Options{MaxDepth: DefaultMaxDepth}) + if err != nil { + t.Fatalf("Scan: %v", err) + } + if want := []string{"data/rows.csv", "report.md"}; !reflect.DeepEqual(pathsOf(got.Files), want) { + t.Fatalf("Files=%v want %v", pathsOf(got.Files), want) + } +} + +// Outside a git work tree, Scan falls back to the fixed build/cache list so the file +// budget is not spent on build output. Reported against a repo where +// rust/*/target/debug/.fingerprint alone held hundreds of files: with a small budget the +// real source must still be reached and the scan must not report truncation. +func TestScanSkipsBuildCacheDirsOutsideGit(t *testing.T) { + isolateGitConfig(t) + root := t.TempDir() + stopGitDiscoveryAbove(t, root) + if _, ok := gitIgnoredPaths(root); ok { + t.Fatalf("gitIgnoredPaths(non-repo) reported a work tree") + } + + writeFile(t, root, "src/lib.rs", "pub fn f() {}\n") + writeFile(t, root, "app/main.py", "print('hi')\n") + for i := range 10 { + writeFile(t, root, fmt.Sprintf("rust/crate/target/debug/.fingerprint/foo-%d/lib-foo.d", i), "x") + } + writeFile(t, root, "app/__pycache__/main.cpython-312.pyc", "x") + writeFile(t, root, ".venv/lib/site-packages/pkg.py", "x") + writeFile(t, root, "Venv/lib/pkg.py", "x") + writeFile(t, root, ".terraform/providers/provider.json", "x") + writeFile(t, root, ".pytest_cache/v/cache/nodeids", "x") + writeFile(t, root, ".mypy_cache/3.12/mod.json", "x") + writeFile(t, root, ".ruff_cache/content", "x") + + got, err := Scan(root, Options{MaxFiles: 3, MaxDepth: DefaultMaxDepth}) + if err != nil { + t.Fatalf("Scan: %v", err) + } + if want := []string{"app/main.py", "src/lib.rs"}; !reflect.DeepEqual(pathsOf(got.Files), want) { + t.Fatalf("Files=%v want %v", pathsOf(got.Files), want) + } + if got.Truncated { + t.Fatalf("Truncated=true: build-cache files consumed the file budget") + } +} + +// When git cannot run at all, Scan must not error and must behave exactly as outside a +// repository, even inside one: .gitignore is not consulted, the fixed list applies. +func TestScanFallsBackWhenGitUnavailable(t *testing.T) { + root := t.TempDir() + initGitRepo(t, root, ".locust_env/\n") + writeFile(t, root, "main.go", "package main\n") + writeFile(t, root, ".locust_env/lib/x.py", "kept: git is not available to honor .gitignore") + writeFile(t, root, "target/debug/app.d", "skipped by the fixed list") + + t.Setenv("PATH", t.TempDir()) + if paths, ok := gitIgnoredPaths(root); ok || paths != nil { + t.Fatalf("gitIgnoredPaths without git = (%v, %v), want (nil, false)", paths, ok) + } + got, err := Scan(root, Options{MaxDepth: DefaultMaxDepth}) + if err != nil { + t.Fatalf("Scan: %v", err) + } + if want := []string{".gitignore", ".locust_env/lib/x.py", "main.go"}; !reflect.DeepEqual(pathsOf(got.Files), want) { + t.Fatalf("Files=%v want %v", pathsOf(got.Files), want) + } +} + +// ShouldSkipDir is shared with glob, grep, list_directory and path autocomplete. The +// build/cache names Scan skips must stay visible there, so a question about a dependency in +// .venv or generated code under target can still be answered. +func TestShouldSkipDirLeavesScanOnlyDirsVisibleToTools(t *testing.T) { + for _, name := range []string{"target", "__pycache__", ".venv", "venv", ".pytest_cache", ".terraform", ".mypy_cache", ".ruff_cache"} { + if ShouldSkipDir(name) { + t.Fatalf("ShouldSkipDir(%q)=true: tools would stop seeing it", name) + } + if !isScanOnlySkipDir(name) { + t.Fatalf("isScanOnlySkipDir(%q)=false want true", name) + } + } + for _, name := range []string{"Target", " target ", "__PYCACHE__", ".VENV"} { + if !isScanOnlySkipDir(name) { + t.Fatalf("isScanOnlySkipDir(%q)=false want true (case/whitespace-insensitive)", name) + } + } + for _, name := range []string{"src", "targets", "venv2", "internal"} { + if isScanOnlySkipDir(name) { + t.Fatalf("isScanOnlySkipDir(%q)=true want false", name) + } + } +} + +// The ignored-path lookup runs every turn, often next to a git command the user or agent +// is running in the same repository. It must not take the optional index lock, and a +// GIT_OPTIONAL_LOCKS value inherited from the environment must not re-enable it. +func TestGitCommandDisablesOptionalLocks(t *testing.T) { + t.Setenv("GIT_OPTIONAL_LOCKS", "1") + cmd := gitCommand(context.Background(), t.TempDir(), "status") + value := "" + for _, entry := range cmd.Env { + if after, ok := strings.CutPrefix(entry, "GIT_OPTIONAL_LOCKS="); ok { + value = after + } + } + if value != "0" { + t.Fatalf("effective GIT_OPTIONAL_LOCKS=%q want \"0\"", value) + } +} diff --git a/internal/workspaceindex/workspaceindex.go b/internal/workspaceindex/workspaceindex.go index badfe5ca1..8526c428f 100644 --- a/internal/workspaceindex/workspaceindex.go +++ b/internal/workspaceindex/workspaceindex.go @@ -63,6 +63,18 @@ func Scan(root string, options Options) (Summary, error) { maxDepth = DefaultMaxDepth } + // Inside a git work tree, git's own ignore rules decide what else to skip; outside one + // (or when git is unavailable or too old), a fixed list of build/cache directory names + // stands in. Both apply to Scan only: ShouldSkipDir is shared with glob, grep, + // list_directory and path autocomplete, which must keep seeing these directories. + ignored, hasIgnored := gitIgnoredPaths(cleanRoot) + skipScanDir := func(rel, name string) bool { + if hasIgnored { + return ignored[rel] + } + return isScanOnlySkipDir(name) + } + files := []File{} dirs := map[string]struct{}{} maxDepthSeen := 0 @@ -83,7 +95,7 @@ func Scan(root string, options Options) (Summary, error) { rel = filepath.ToSlash(rel) if entry.IsDir() { - if ShouldSkipDir(entry.Name()) || isSymlink(entry) { + if ShouldSkipDir(entry.Name()) || isSymlink(entry) || skipScanDir(rel, entry.Name()) { return filepath.SkipDir } depth := pathDepth(rel) @@ -98,7 +110,7 @@ func Scan(root string, options Options) (Summary, error) { return nil } - if isSymlink(entry) || ShouldSkipFile(rel) { + if isSymlink(entry) || ShouldSkipFile(rel) || (hasIgnored && ignored[rel]) { return nil } depth := FileDepth(rel) @@ -175,6 +187,18 @@ func ShouldSkipDir(name string) bool { } } +// isScanOnlySkipDir names build and cache directories Scan skips when it cannot ask git +// what is ignored. It is deliberately separate from ShouldSkipDir: a tracked package named +// "target" or a question about a dependency inside .venv must stay visible to the tools. +func isScanOnlySkipDir(name string) bool { + switch strings.ToLower(strings.TrimSpace(name)) { + case "target", "__pycache__", ".venv", "venv", ".pytest_cache", ".terraform", ".mypy_cache", ".ruff_cache": + return true + default: + return false + } +} + func ShouldSkipFile(rel string) bool { base := strings.ToLower(path.Base(filepath.ToSlash(rel))) switch base { diff --git a/internal/workspaceseed/workspaceseed.go b/internal/workspaceseed/workspaceseed.go index 140620375..4ae0a5094 100644 --- a/internal/workspaceseed/workspaceseed.go +++ b/internal/workspaceseed/workspaceseed.go @@ -83,7 +83,9 @@ func Build(input Input) Seed { } // BuildFromWorkspace scans the local filesystem with workspaceindex and builds -// a seed from the resulting file list. It performs no git operations. +// a seed from the resulting file list. Its only git operation is the read-only +// ignored-path lookup inside workspaceindex.Scan; branch and dirty state come +// from the caller's GitInfo. func BuildFromWorkspace(root string, git GitInfo) (Seed, error) { summary, err := workspaceindex.Scan(root, workspaceindex.Options{ MaxFiles: workspaceindex.DefaultMaxFiles,