From c1030d01c5741bb261c21bddab1820521aafba61 Mon Sep 17 00:00:00 2001 From: Marvin Wichmann Date: Thu, 27 Aug 2026 15:19:07 +0200 Subject: [PATCH] Add pnpm recipe Builds pnpm from its self-contained linux-x64 release archive and injects a bin/pnpm wrapper so the artifact is consumed exactly like yarn's. Covers pnpm 11.0.0 and newer: v11.0.0 is the first release to publish pnpm-linux-x64.tar.gz. Earlier releases attach a bare pnpm-linux-x64 binary rather than an archive and are out of range. Verified end to end against 11.24.0 and 12.0.0. The npm registry package is not used, because it stopped being self-contained. Through pnpm 11 it bundled a complete dist/ tree, but as of pnpm 12 it is a ~1 MB stub: its preinstall script downloads a platform-native binary, and the bin/pnpm.mjs Corepack shim fetches that same binary on first run. Neither is available to a buildpack staging offline, so sourcing from npm would keep working right up until the first pnpm 12 build and then produce an artifact that only runs with network access. The release archive ships the native binary outright. The archive is flat, so nothing is stripped and a bin/pnpm wrapper is injected to provide the bin/ layout. It resolves $0 through readlink -f because buildpacks symlink bin/ entries into their own bin directory, where the binary is not a sibling. InjectFile gains a mode-aware variant for this; the existing entry point keeps its 0644 default. --- README.md | 2 +- cmd/binary-builder/main.go | 1 + internal/archive/archive.go | 17 ++-- internal/archive/archive_test.go | 60 ++++++++++++++ internal/recipe/recipe_helpers_test.go | 39 ++++++++++ internal/recipe/recipe_test.go | 103 +++++++++++++++++++++++++ internal/recipe/repack.go | 29 +++++-- internal/recipe/simple.go | 53 +++++++++++++ test/exerciser/exerciser_test.go | 9 +++ 9 files changed, 299 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5e079431..801eb949 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ A Go tool for building binaries used by Cloud Foundry buildpacks. | Apache HTTPD | cflinuxfs4, cflinuxfs5 | | Bundler | cflinuxfs4, cflinuxfs5 | | RubyGems | cflinuxfs4, cflinuxfs5 | -| Yarn / Bower / Composer | cflinuxfs4, cflinuxfs5 | +| Yarn / pnpm / Bower / Composer | cflinuxfs4, cflinuxfs5 | | Pip / Pipenv / Setuptools | cflinuxfs4, cflinuxfs5 | | OpenJDK / Zulu / SAPMachine | cflinuxfs4, cflinuxfs5 | | .NET SDK / Runtime / ASP.NET Core | cflinuxfs4, cflinuxfs5 | diff --git a/cmd/binary-builder/main.go b/cmd/binary-builder/main.go index 6e6d355b..5a5761b3 100644 --- a/cmd/binary-builder/main.go +++ b/cmd/binary-builder/main.go @@ -328,6 +328,7 @@ func buildRegistry() *recipe.Registry { reg.Register(&recipe.PipenvRecipe{Fetcher: f}) reg.Register(&recipe.BowerRecipe{Fetcher: f}) reg.Register(&recipe.YarnRecipe{Fetcher: f}) + reg.Register(&recipe.PnpmRecipe{Fetcher: f}) reg.Register(&recipe.RubygemsRecipe{Fetcher: f}) reg.Register(&recipe.MinicondaRecipe{Fetcher: f}) reg.Register(&recipe.DotnetSDKRecipe{Fetcher: f}) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index b558029e..0ad73b8f 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -269,10 +269,17 @@ func StripIncorrectWordsYAML(path string) error { return StripFiles(path, "incorrect_words.yaml") } -// InjectFile adds a file with the given name and content into an existing -// gzipped tarball. The file is appended at the archive root (no directory -// prefix). Typically used to inject sources.yml into an artifact tarball. +// InjectFile adds a non-executable file with the given name and content into an +// existing gzipped tarball. Typically used to inject sources.yml into an +// artifact tarball. func InjectFile(tarPath, filename string, content []byte) error { + return InjectFileWithMode(tarPath, filename, content, 0644) +} + +// InjectFileWithMode is InjectFile with an explicit file mode. Use it for +// entries that must be executable, such as wrapper scripts placed in bin/. +// filename may contain a directory prefix (e.g. "bin/pnpm"). +func InjectFileWithMode(tarPath, filename string, content []byte, mode int64) error { data, err := os.ReadFile(tarPath) if err != nil { return fmt.Errorf("reading %s: %w", tarPath, err) @@ -309,10 +316,10 @@ func InjectFile(tarPath, filename string, content []byte) error { } } - // Append the new file at the archive root. + // Append the new file, relative to the archive root. hdr := &tar.Header{ Name: "./" + filename, - Mode: 0644, + Mode: mode, Size: int64(len(content)), Typeflag: tar.TypeReg, } diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 3b2671cb..601f57b8 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -183,6 +183,66 @@ func TestStripIncorrectWordsYAML(t *testing.T) { assert.NotContains(t, entries, "incorrect_words.yaml") } +// readTarEntry returns the header and content of the named entry, matching +// either the bare name or its "./"-prefixed form. +func readTarEntry(t *testing.T, path, name string) (*tar.Header, string) { + t.Helper() + data, err := os.ReadFile(path) + require.NoError(t, err) + + gr, err := gzip.NewReader(bytes.NewReader(data)) + require.NoError(t, err) + defer gr.Close() + + tr := tar.NewReader(gr) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + if hdr.Name != name && hdr.Name != "./"+name { + continue + } + content, err := io.ReadAll(tr) + require.NoError(t, err) + return hdr, string(content) + } + + t.Fatalf("readTarEntry: %q not found in %s", name, path) + return nil, "" +} + +func TestInjectFileDefaultsToNonExecutable(t *testing.T) { + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "test.tgz") + require.NoError(t, os.WriteFile(path, createTestTarball(t, map[string]string{"bin/ruby": "binary"}), 0644)) + + require.NoError(t, archive.InjectFile(path, "sources.yml", []byte("---\n"))) + + // Existing entries survive. + assert.Contains(t, listTarEntries(t, path), "bin/ruby") + + hdr, content := readTarEntry(t, path, "sources.yml") + assert.Equal(t, "---\n", content) + assert.Equal(t, int64(0644), hdr.Mode) +} + +func TestInjectFileWithModePreservesModeAndPath(t *testing.T) { + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "test.tgz") + require.NoError(t, os.WriteFile(path, createTestTarball(t, map[string]string{"bin/pnpm.mjs": "entrypoint"}), 0644)) + + require.NoError(t, archive.InjectFileWithMode(path, "bin/pnpm", []byte("#!/bin/sh\n"), 0755)) + + hdr, content := readTarEntry(t, path, "bin/pnpm") + assert.Equal(t, "#!/bin/sh\n", content) + assert.Equal(t, int64(0755), hdr.Mode, "injected wrapper must be executable") + + // The directory prefix is kept rather than flattened to the archive root. + assert.NotContains(t, listTarEntries(t, path), "./pnpm") +} + func TestStripTopLevelDirFromZip(t *testing.T) { tmpDir := t.TempDir() path := filepath.Join(tmpDir, "test.zip") diff --git a/internal/recipe/recipe_helpers_test.go b/internal/recipe/recipe_helpers_test.go index 25a4abbd..3d788984 100644 --- a/internal/recipe/recipe_helpers_test.go +++ b/internal/recipe/recipe_helpers_test.go @@ -3,6 +3,7 @@ package recipe_test import ( "archive/tar" "compress/gzip" + "io" "os" "path/filepath" "strings" @@ -118,6 +119,44 @@ func useTempWorkDir(t *testing.T) string { return tmp } +// tarEntry returns the header and content of the named entry in a gzipped +// tarball, matching either the bare name or its "./"-prefixed form. It returns +// a nil header when the entry is absent. +func tarEntry(t *testing.T, path, name string) (*tar.Header, string) { + t.Helper() + + f, err := os.Open(path) + if err != nil { + t.Fatalf("tarEntry: open %s: %v", path, err) + } + defer f.Close() + + gr, err := gzip.NewReader(f) + if err != nil { + t.Fatalf("tarEntry: gzip %s: %v", path, err) + } + defer gr.Close() + + tr := tar.NewReader(gr) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil, "" + } + if err != nil { + t.Fatalf("tarEntry: read %s: %v", path, err) + } + if hdr.Name != name && hdr.Name != "./"+name { + continue + } + content, err := io.ReadAll(tr) + if err != nil { + t.Fatalf("tarEntry: read %q: %v", name, err) + } + return hdr, string(content) + } +} + // writeFakeArtifact creates a minimal valid .tgz at in the current // working directory. The tarball contains a single dummy file so that // archive.StripTopLevelDir / StripIncorrectWordsYAML don't fail. diff --git a/internal/recipe/recipe_test.go b/internal/recipe/recipe_test.go index 0ff53cc4..830c7740 100644 --- a/internal/recipe/recipe_test.go +++ b/internal/recipe/recipe_test.go @@ -400,6 +400,109 @@ func TestYarnRecipeNameAndArtifact(t *testing.T) { assert.Equal(t, "noarch", r.Artifact().Arch) } +// ── PnpmRecipe ──────────────────────────────────────────────────────────────── + +func pnpmReleaseURLFor(version string) string { + return "https://github.com/pnpm/pnpm/releases/download/v" + version + "/pnpm-linux-x64.tar.gz" +} + +// pnpmSupportedVersions spans the supported range. v11.0.0 is the first release +// to publish pnpm-linux-x64.tar.gz at all; 12.0.0 is the first major after pnpm +// gutted its npm package, which is why that package is not the source here. +// Both archives have the same flat layout, so one recipe must serve both. +var pnpmSupportedVersions = []string{"11.0.0", "11.24.0", "12.0.0"} + +func TestPnpmRecipeNameAndArtifact(t *testing.T) { + r := &recipe.PnpmRecipe{} + assert.Equal(t, "pnpm", r.Name()) + assert.Equal(t, "linux", r.Artifact().OS) + // The release archive bundles a native executable, so it is not noarch. + assert.Equal(t, "x64", r.Artifact().Arch) +} + +func TestPnpmRecipeStripsVPrefix(t *testing.T) { + for _, version := range pnpmSupportedVersions { + t.Run(version, func(t *testing.T) { + f := newFakeFetcher() + url := pnpmReleaseURLFor(version) + + src := newInput("pnpm", "v"+version, url) + r := &recipe.PnpmRecipe{Fetcher: f} + outData := &output.OutData{} + require.NoError(t, r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), outData)) + + require.Len(t, f.DownloadedURLs, 1) + assert.Equal(t, url, f.DownloadedURLs[0].URL) + // File on disk uses the stripped version so findIntermediateArtifact matches. + assert.Equal(t, filepath.Join(os.TempDir(), "pnpm-"+version+".tar.gz"), f.DownloadedURLs[0].Dest) + assert.Equal(t, version, outData.Version) + // src.Version must NOT be mutated — callers after Build rely on the original. + assert.Equal(t, "v"+version, src.Version) + }) + } +} + +func TestPnpmRecipeInjectsExecutableBinWrapper(t *testing.T) { + for _, version := range pnpmSupportedVersions { + t.Run(version, func(t *testing.T) { + f := newFakeFetcher() + + dest := filepath.Join(os.TempDir(), "pnpm-"+version+".tar.gz") + t.Cleanup(func() { _ = os.Remove(dest) }) + + src := newInput("pnpm", "v"+version, pnpmReleaseURLFor(version)) + r := &recipe.PnpmRecipe{Fetcher: f} + require.NoError(t, r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), &output.OutData{})) + + hdr, content := tarEntry(t, dest, "bin/pnpm") + require.NotNil(t, hdr, "artifact must contain a bin/pnpm entry") + + // The release archive has no bin/ dir at all — the plain name only + // exists because we inject it, and it is useless unless executable. + assert.Equal(t, int64(0755), hdr.Mode) + // Buildpacks symlink bin/ entries elsewhere, so $0 must be resolved. + assert.Contains(t, content, "readlink -f") + assert.Contains(t, content, `exec "$basedir/../pnpm"`) + // The binary is native: pulling in an interpreter would be a regression. + assert.NotContains(t, content, "node") + }) + } +} + +func TestPnpmRecipeKeepsArchiveFlat(t *testing.T) { + for _, version := range pnpmSupportedVersions { + t.Run(version, func(t *testing.T) { + f := newFakeFetcher() + + dest := filepath.Join(os.TempDir(), "pnpm-"+version+".tar.gz") + t.Cleanup(func() { _ = os.Remove(dest) }) + + src := newInput("pnpm", "v"+version, pnpmReleaseURLFor(version)) + r := &recipe.PnpmRecipe{Fetcher: f} + require.NoError(t, r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), &output.OutData{})) + + // The upstream archive is already flat. Stripping a top-level + // directory would silently discard the native binary, so the fake's + // "fake-top/" entry must still be present. + hdr, _ := tarEntry(t, dest, "fake-top/") + assert.NotNil(t, hdr, "recipe must not strip a top-level directory") + }) + } +} + +func TestPnpmRecipePropagatesDownloadError(t *testing.T) { + f := newFakeFetcher() + url := pnpmReleaseURLFor("12.0.0") + f.ErrMap[url] = errors.New("boom") + + src := newInput("pnpm", "v12.0.0", url) + r := &recipe.PnpmRecipe{Fetcher: f} + err := r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), &output.OutData{}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "downloading pnpm") +} + // ── PyPISourceRecipe ────────────────────────────────────────────────────────── func TestPyPISourceRecipeFilenameFromURL(t *testing.T) { diff --git a/internal/recipe/repack.go b/internal/recipe/repack.go index c56e5a07..75cab0d2 100644 --- a/internal/recipe/repack.go +++ b/internal/recipe/repack.go @@ -37,6 +37,10 @@ type RepackRecipe struct { // If nil, the default is "-.". // PyPI sdist recipes use this to infer the filename from the URL's last path segment. DestFilename func(version, url string) string + // AfterRepack runs once the archive has been downloaded and stripped, with + // the path to the local artifact. Use it for per-dep transformations that + // would otherwise have to recompute the destination filename. + AfterRepack func(dest string) error } func (r *RepackRecipe) Name() string { return r.DepName } @@ -60,17 +64,26 @@ func (r *RepackRecipe) Build(ctx context.Context, _ *stack.Stack, src *source.In return fmt.Errorf("downloading %s: %w", r.DepName, err) } - if !r.StripTopLevelDir { - return nil + if r.StripTopLevelDir { + // Use dest (already fragment-free) rather than src.URL to detect zip archives. + // PyPI download URLs may contain a #sha256=… fragment that would fool a + // suffix check on the raw URL. + var err error + if strings.HasSuffix(dest, ".zip") { + err = archive.StripTopLevelDirFromZip(dest) + } else { + err = archive.StripTopLevelDir(dest) + } + if err != nil { + return err + } } - // Use dest (already fragment-free) rather than src.URL to detect zip archives. - // PyPI download URLs may contain a #sha256=… fragment that would fool a - // suffix check on the raw URL. - if strings.HasSuffix(dest, ".zip") { - return archive.StripTopLevelDirFromZip(dest) + if r.AfterRepack != nil { + return r.AfterRepack(dest) } - return archive.StripTopLevelDir(dest) + + return nil } // inferExt returns the file extension for a download URL, recognising .tar.gz diff --git a/internal/recipe/simple.go b/internal/recipe/simple.go index 86d10a04..37afc7fe 100644 --- a/internal/recipe/simple.go +++ b/internal/recipe/simple.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/cloudfoundry/binary-builder/internal/archive" "github.com/cloudfoundry/binary-builder/internal/fetch" "github.com/cloudfoundry/binary-builder/internal/output" "github.com/cloudfoundry/binary-builder/internal/runner" @@ -47,6 +48,58 @@ func (y *YarnRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Inpu }).Build(ctx, s, src, r, out) } +// pnpmWrapper is injected as bin/pnpm. The release archive lays the native +// binary out flat at the archive root, so this wrapper supplies the bin/ +// layout consumers already expect from yarn. +// +// $0 is resolved with readlink -f because buildpacks symlink the artifact's +// bin/ entries into their own bin directory — without it $0's dirname points at +// the symlink's directory, where the binary does not exist. This mirrors what +// upstream yarn's own bin/yarn wrapper does. No interpreter is involved: the +// target is a native executable. +const pnpmWrapper = `#!/bin/sh +basedir=$(dirname "$(readlink -f "$0" 2>/dev/null || echo "$0")") +exec "$basedir/../pnpm" "$@" +` + +// PnpmRecipe downloads pnpm's self-contained linux-x64 release archive and +// injects a bin/pnpm wrapper. +// +// The npm registry tarball is deliberately not used. As of pnpm 12 that package +// is a ~1 MB stub: its preinstall script downloads a platform-native binary, and +// the bin/pnpm.mjs Corepack shim fetches the same binary on first run. A +// buildpack cannot depend on either in an offline or air-gapped deployment. The +// release archive ships the native binary outright, so the artifact is +// self-contained and needs no network access at staging time. +// +// The archive is linux-x64 and glibc-linked, which covers every current +// cflinuxfs stack; musl and other architectures are published separately and are +// not built here. +type PnpmRecipe struct { + Fetcher fetch.Fetcher +} + +func (p *PnpmRecipe) Name() string { return "pnpm" } +func (p *PnpmRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} +func (p *PnpmRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, r runner.Runner, out *output.OutData) error { + return (&RepackRecipe{ + DepName: "pnpm", + Meta: ArtifactMeta{OS: "linux", Arch: "x64"}, + Fetcher: p.Fetcher, + // Release tags carry a "v" prefix; the archive itself is already flat, + // so there is no top-level directory to strip. + StripVersionPrefix: "v", + AfterRepack: func(dest string) error { + if err := archive.InjectFileWithMode(dest, "bin/pnpm", []byte(pnpmWrapper), 0755); err != nil { + return fmt.Errorf("pnpm: injecting bin/pnpm wrapper: %w", err) + } + return nil + }, + }).Build(ctx, s, src, r, out) +} + // PyPISourceRecipe downloads a PyPI source tarball and strips its top-level // directory. It covers any dep published as a plain sdist on PyPI (e.g. // setuptools, flit-core) where no compilation step is required. diff --git a/test/exerciser/exerciser_test.go b/test/exerciser/exerciser_test.go index 9c24ec8d..88cabeef 100644 --- a/test/exerciser/exerciser_test.go +++ b/test/exerciser/exerciser_test.go @@ -215,6 +215,15 @@ func TestYarnBinary(t *testing.T) { assertContains(t, out, os.Getenv("VERSION")) } +func TestPnpmBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + // The upstream archive has no bin/ dir — bin/pnpm is the wrapper the recipe + // injects, so this also asserts it is present, executable, and resolves the + // native binary next to it. + out := runInContainer(t, a, s, "./bin/pnpm", "--version") + assertContains(t, out, os.Getenv("VERSION")) +} + func TestRubygemsFiles(t *testing.T) { a, s := artifact(t), stackEnv(t) out := runInContainer(t, a, s, "bash", "-c", "ls rubygems-*/")