diff --git a/README.md b/README.md index 769af0e4f..489bd72cf 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,48 @@ Building buildpack (version: 0.0.0, stack: cflinuxfs4, cached: true, output: bui The offline package will be significantly larger (1.0-1.2 GB depending on cached dependencies) as it includes all JRE versions and framework agents specified in `manifest.yml`. +#### Selective Dependency Packaging + +For air-gapped environments or security-conscious deployments, you can build a smaller offline package that contains only a named subset of dependencies using packaging profiles or explicit exclusions. + +**Using a profile** (defined in `manifest.yml`): + +```bash +# Minimal: JDKs, CF utilities, Tomcat, and common frameworks only (~28 dependencies) +$ ./scripts/package.sh --cached --profile minimal + +# Standard: core + open-source APM, OTel, and JDBC drivers (~32 dependencies) +$ ./scripts/package.sh --cached --profile standard +``` + +**Ad-hoc exclusions** (no profile required): + +```bash +# Exclude specific agents you don't have licences for +$ ./scripts/package.sh --cached --exclude jrebel,your-kit-profiler,jprofiler-profiler +``` + +**Combining a profile with overrides**: + +```bash +# Start from standard profile but also drop jacoco +$ ./scripts/package.sh --cached --profile standard --exclude jacoco + +# Start from minimal profile but add back jprofiler for triage builds +$ ./scripts/package.sh --cached --profile minimal --include jprofiler-profiler +``` + +The output zip filename reflects the profile/exclusion applied so that different variants can coexist: + +| Invocation | Output filename | +|---|---| +| `--cached` | `java_buildpack-cached-cflinuxfs4-v.zip` | +| `--cached --profile minimal` | `java_buildpack-cached-minimal-cflinuxfs4-v.zip` | +| `--cached --exclude newrelic` | `java_buildpack-cached-custom-cflinuxfs4-v.zip` | +| `--cached --profile minimal --include jprofiler-profiler` | `java_buildpack-cached-minimal+custom-cflinuxfs4-v.zip` | + +> **Note**: `--profile`, `--exclude`, and `--include` are only valid with `--cached`. Using them on an uncached build is an error. `--include` requires `--profile` to be set. + ### Package Versioning To specify a version number when creating a package, use the `--version` flag: @@ -260,6 +302,9 @@ OPTIONS --cached cache the buildpack dependencies (default: false) --stack specifies the stack (default: cflinuxfs4) --output output file path (default: build/buildpack.zip) + --profile packaging profile from manifest.yml (e.g. minimal, standard) + --exclude comma-separated dependency names to exclude (cached only) + --include comma-separated dependency names to restore, overriding profile exclusions (cached only) ``` ### Customizing Dependencies @@ -289,7 +334,7 @@ dependencies: # Online package with version 5.0.0 $ ./scripts/package.sh --version 5.0.0 -# Offline package with version 5.0.0 +# Offline package with version 5.0.0 (all dependencies) $ ./scripts/package.sh --version 5.0.0 --cached # Package for specific stack @@ -297,6 +342,18 @@ $ ./scripts/package.sh --stack cflinuxfs4 --cached # Custom output location $ ./scripts/package.sh --version 5.0.0 --cached --output /tmp/my-buildpack.zip + +# Offline package with minimal profile (JDKs + CF utilities only) +$ ./scripts/package.sh --version 5.0.0 --cached --profile minimal + +# Offline package with standard profile (core + open-source observability) +$ ./scripts/package.sh --version 5.0.0 --cached --profile standard + +# Exclude specific dependencies without a profile +$ ./scripts/package.sh --version 5.0.0 --cached --exclude jrebel,your-kit-profiler + +# Minimal profile, but add back jprofiler for this specific build +$ ./scripts/package.sh --version 5.0.0 --cached --profile minimal --include jprofiler-profiler ``` ## Running Tests diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md index 3400ea3ab..9db4c1bbd 100644 --- a/docs/DEVELOPING.md +++ b/docs/DEVELOPING.md @@ -482,6 +482,28 @@ Create a package with all dependencies cached (no internet required at runtime): **Output:** `build/buildpack.zip` (~500MB, varies based on cached dependencies) +#### Selective Dependency Packaging (Profiles) + +For environments that only need a subset of dependencies, use packaging profiles or +explicit exclusions to reduce the offline package size: + +```bash +# Minimal: JDKs, CF utilities, Tomcat only (~28 deps, much smaller download) +./scripts/package.sh --version 1.0.0 --cached --profile minimal + +# Standard: core + open-source APM, OTel, JDBC (~32 deps) +./scripts/package.sh --version 1.0.0 --cached --profile standard + +# Ad-hoc: exclude specific agents (no profile needed) +./scripts/package.sh --version 1.0.0 --cached --exclude jrebel,your-kit-profiler + +# Restore one dep excluded by a profile +./scripts/package.sh --version 1.0.0 --cached --profile minimal --include jprofiler-profiler +``` + +Profiles are declared in the `packaging_profiles` section of `manifest.yml`. See +[Selective Dependency Packaging](selective-dependency-packaging.md) for full details. + ### Package Options ```bash @@ -496,6 +518,12 @@ Create a package with all dependencies cached (no internet required at runtime): # Offline with custom stack ./scripts/package.sh --version 1.0.0 --cached --stack cflinuxfs4 + +# Offline with minimal profile +./scripts/package.sh --version 1.0.0 --cached --profile minimal + +# Offline excluding specific dependencies +./scripts/package.sh --version 1.0.0 --cached --exclude datadog-javaagent,newrelic ``` ### Automated Packaging (CI/CD) diff --git a/docs/buildpack-modes.md b/docs/buildpack-modes.md index 103550470..a19c16cf7 100644 --- a/docs/buildpack-modes.md +++ b/docs/buildpack-modes.md @@ -44,6 +44,39 @@ The "Offline Mode" buildpack is a self-contained packaging of either the "Easy M You can download specific versions of the "Offline Mode" buildpack to use with the `create-buildpack` and `update-buildpack` Cloud Foundry CLI commands. To find these, navigate to the [Java Buildpack Releases page][v] and download one of the `java-buildpack-offline-v.zip` file. In order to package a modified "Offline Mode" buildpack, refer to [Building Packages][p]. To add the buildpack to an instance of Cloud Foundry, use the `cf create-buildpack java-buildpack java-buildpack-offline-v.zip` command. For more details refer to the [Cloud Foundry buildpack documentation][b]. +### Selective Offline Packaging + +The full offline package bundles every dependency in `manifest.yml` (~47 binaries, 1.0-1.2 GB). For +air-gapped environments that only need a subset, you can build a smaller offline package using +**packaging profiles** or explicit exclusions. + +**Using a profile** (defined in `manifest.yml`'s `packaging_profiles` section): + +```bash +# Minimal: JDKs, CF utilities, Tomcat, and common frameworks only (~28 deps) +$ ./scripts/package.sh --cached --profile minimal + +# Standard: core + open-source APM, OTel, and JDBC drivers (~32 deps) +$ ./scripts/package.sh --cached --profile standard +``` + +**Ad-hoc exclusions** without a profile: + +```bash +# Remove specific agents you don't have licences for +$ ./scripts/package.sh --cached --exclude jrebel,your-kit-profiler,jprofiler-profiler +``` + +**Overriding a profile** to restore a specific dependency: + +```bash +# Start from minimal but include jprofiler for triage builds +$ ./scripts/package.sh --cached --profile minimal --include jprofiler-profiler +``` + +For full details on profiles, validation rules, and output filename conventions, see +[Selective Dependency Packaging](selective-dependency-packaging.md). + [b]: http://docs.pivotal.io/pivotalcf/adminguide/buildpacks.html [c]: ../README.md#configuration-and-extension diff --git a/docs/selective-dependency-packaging.md b/docs/selective-dependency-packaging.md new file mode 100644 index 000000000..154b1b4d3 --- /dev/null +++ b/docs/selective-dependency-packaging.md @@ -0,0 +1,886 @@ +# Selective Dependency Packaging + +**Status**: Proposed +**Date**: 2026-04-01 +**Affects**: `libbuildpack/packager`, all CF buildpacks + +--- + +## Table of Contents + +1. [Problem Statement](#1-problem-statement) +2. [Goals and Non-Goals](#2-goals-and-non-goals) +3. [Current Architecture](#3-current-architecture) +4. [Proposed Architecture](#4-proposed-architecture) +5. [Design Decisions](#5-design-decisions) +6. [manifest.yml Changes](#6-manifestyml-changes) +7. [libbuildpack/packager Changes](#7-libbuildpackpackager-changes) +8. [scripts/package.sh Changes](#8-scriptspackagesh-changes) +9. [java-buildpack Adoption](#9-java-buildpack-adoption) +10. [Implementation Plan](#10-implementation-plan) +11. [Testing Strategy](#11-testing-strategy) +12. [Rollout Strategy](#12-rollout-strategy) +13. [Open Questions](#13-open-questions) + +--- + +## 1. Problem Statement + +Running `scripts/package.sh --cached` produces an **offline buildpack** — a zip that contains every +dependency declared in `manifest.yml`. For the java-buildpack this means 47 binaries are bundled, +covering every JRE distribution, every APM agent, every profiler, and every JDBC driver, regardless +of whether the target platform will ever use them. + +**Concrete consequences**: + +- The resulting zip is very large, making it slow to upload and store. +- Air-gapped environments that only use, say, OpenJDK + Tomcat are forced to carry agents for + Datadog, New Relic, JRebel, YourKit, SkyWalking, and dozens of other tools they will never need. +- Operators cannot tailor a buildpack to their security posture (e.g., excluding a commercial agent + they don't have a license for). + +**This is not a java-buildpack-only problem.** Eight of the thirteen CF buildpacks have ten or more +dependencies (python: 23, ruby: 22, dotnet-core: 20, php: 16, go: 13, nginx: 12, nodejs: 11) and +face the same trade-off when building cached/offline releases. + +--- + +## 2. Goals and Non-Goals + +### Goals + +- Allow operators to build a cached buildpack that contains only a **named subset** of dependencies. +- Support both **ad-hoc exclusion** (`--exclude dep-a,dep-b`) and **named profiles** (`--profile minimal`). +- Profiles are declared inside `manifest.yml` of each buildpack — no global registry needed. +- The feature lives in **`libbuildpack/packager`** so every buildpack inherits it automatically. +- **Fully backward compatible**: buildpacks that do not use the new flags are completely unaffected. + +### Non-Goals + +- Runtime dependency filtering (what the running buildpack installs for an app) — this is purely a + *packaging-time* concern. +- Changing how `buildpack-packager` handles stacks — that mechanism is orthogonal and unchanged. +- Automatic profile selection based on platform configuration. +- A centralised profile registry shared across buildpacks. + +--- + +## 3. Current Architecture + +### 3.1 Packaging pipeline (today) + +``` +scripts/package.sh --cached + └─ buildpack-packager build + --version= + --cached=true + --stack=cflinuxfs4 + └─ packager.Package(bpDir, cacheDir, version, stack, cached=true) + ├─ validates stack against manifest + ├─ runs pre_package script + ├─ for every dependency that matches the stack: + │ ├─ downloadDependency() ← downloads ALL deps + │ └─ SHA256 verify + └─ ZipFiles() → java_buildpack-cached-cflinuxfs4-v.zip +``` + +### 3.2 Dependency declaration in manifest.yml (today) + +```yaml +dependencies: + - name: datadog-javaagent + version: 1.42.1 + uri: https://repo1.maven.org/... + sha256: e703547f... + cf_stacks: + - cflinuxfs4 +``` + +Each dependency entry has: `name`, `version`, `uri`, `sha256`, `cf_stacks`. +There is no concept of optionality, grouping, or profiles. + +### 3.3 Shared scripts + +`scripts/package.sh` and `scripts/.util/tools.sh` are **byte-for-byte identical** across all 13 +buildpacks (differing only in the default `stack=` value). Any new flag added to `buildpack-packager` +needs only a trivial one-line change in the shared script template to become available everywhere. + +--- + +## 4. Proposed Architecture + +### 4.1 Overview + +Three complementary mechanisms are added, all optional: + +| Mechanism | Flag | Where defined | Use case | +|---|---|---|---| +| Ad-hoc exclusion | `--exclude dep-a,dep-b` | CLI only | One-off builds, CI overrides | +| Named profiles | `--profile minimal` | `manifest.yml` | Reusable, versioned subsets | +| Profile override | `--include dep-a` | CLI only | Restore specific deps excluded by a profile | + +All are purely *packaging-time* filters. At runtime the buildpack behaves identically — components +that rely on a dependency that was excluded simply will not find it and will not activate (the same +as they would in an uncached buildpack where the network is unavailable). + +### 4.2 End-to-end flow (proposed) + +``` +scripts/package.sh --cached --profile minimal + └─ buildpack-packager build + --version= + --cached=true + --stack=cflinuxfs4 + --profile=minimal ← NEW + └─ packager.PackageWithOptions(bpDir, cacheDir, version, stack, cached=true, + PackageOptions{Profile:"minimal", Exclude:[], Include:[]}) + ├─ resolveExclusions(manifest, profile="minimal", exclude=[], include=[]) + │ └─ returns map[string]struct{} of dep names to skip + ├─ for every dependency that matches the stack AND is not excluded: + │ ├─ downloadDependency() ← only selected deps + │ └─ SHA256 verify + └─ ZipFiles() → java_buildpack-cached-cflinuxfs4-minimal-v.zip + +scripts/package.sh --cached --profile minimal --include jprofiler-profiler + └─ buildpack-packager build + --version= + --cached=true + --stack=cflinuxfs4 + --profile=minimal + --include=jprofiler-profiler ← NEW + └─ packager.PackageWithOptions(...) + ├─ resolveExclusions(manifest, profile="minimal", exclude=[], include=["jprofiler-profiler"]) + │ ├─ computes profile exclusions → removes jprofiler-profiler from excluded set + │ └─ returns map without jprofiler-profiler + ├─ for every dependency that matches the stack AND is not excluded: + │ ├─ downloadDependency() ← minimal deps + jprofiler-profiler + │ └─ SHA256 verify + └─ ZipFiles() → java_buildpack-cached-cflinuxfs4-minimal+custom-v.zip +``` + +### 4.3 Zip filename convention + +The output filename gains a profile or exclusion suffix so that different variants can coexist: + +| Invocation | Output filename | +|---|---| +| `--cached` | `java_buildpack-cached-cflinuxfs4-v1.2.3.zip` | +| `--cached --profile minimal` | `java_buildpack-cached-cflinuxfs4-minimal-v1.2.3.zip` | +| `--cached --exclude newrelic` | `java_buildpack-cached-cflinuxfs4-custom-v1.2.3.zip` | +| `--cached --profile minimal --include jprofiler-profiler` | `java_buildpack-cached-cflinuxfs4-minimal+custom-v1.2.3.zip` | +| `--cached --profile minimal --exclude groovy` | `java_buildpack-cached-cflinuxfs4-minimal+custom-v1.2.3.zip` | + +--- + +## 5. Design Decisions + +### 5.1 Why profiles live in manifest.yml, not a separate file + +`manifest.yml` is already the single source of truth for dependency metadata. Keeping profiles there +means: + +- Profile definitions are versioned alongside the dependencies they reference. +- `buildpack-packager summary` can be extended to also list profiles. +- No new file format needs to be discovered or parsed by tooling. + +### 5.2 Why `--exclude` takes dependency *names* not *indices* + +Names are stable across manifest updates. Indices change whenever a dependency is added or removed. +Using names also makes CI scripts and documentation self-documenting. + +### 5.3 Why profiles use `exclude` lists rather than `include` lists + +The manifest already declares the full set of available dependencies. Exclusion lists are shorter +and require less maintenance: when a new optional dependency is added to the manifest it is +automatically part of all profiles unless explicitly excluded. An inclusion-based profile would +require every profile to be updated each time a new core dependency is added. + +The `minimal` profile is the one exception that benefits most from this: it excludes the long tail +of optional agents, and the "include everything" case is simply the absence of any profile. + +### 5.4 Why the feature belongs in libbuildpack, not per-buildpack scripts + +All buildpacks share the same `buildpack-packager` binary (installed via `go install ...@latest`). +Adding the feature to the packager makes it available to every buildpack immediately, with only a +trivial script change per buildpack to expose the new flags. The alternative — implementing YAML +manipulation in each buildpack's `package.sh` — would be duplicated across 13 repos and harder to +keep consistent. + +### 5.5 Mutual exclusion: --profile and --exclude can be combined + +`--profile minimal --exclude groovy` is valid. The profile's exclusion list is computed first, then +the `--exclude` list is unioned with it. This allows operators to start from a profile and trim +further for a specific deployment. + +### 5.6 --include overrides profile exclusions (CLI only) + +`--profile minimal --include jprofiler-profiler` is valid. The profile's exclusion list is computed +first, then any names in `--include` are removed from that set — effectively restoring those +dependencies into the build. This allows operators to start from a profile and selectively add back +specific deps without defining a new profile. + +`--include` and `--exclude` can both be passed alongside `--profile`. Order of resolution: +1. Profile's `exclude` list is applied. +2. `--exclude` CLI additions are unioned in. +3. `--include` CLI overrides are removed from the set. + +`--include` without `--profile` is a no-op (nothing was excluded to begin with) but is treated as a +warning rather than a hard error, since it is not necessarily a mistake in a scripted environment. + +### 5.7 Unknown dependency names are errors + +If `--exclude datadog-javaagent` is passed but `datadog-javaagent` does not exist in the manifest, +`buildpack-packager` exits non-zero. This catches typos early rather than silently producing a zip +that happens to be missing something unexpected. + +Same rule applies to profiles: referencing an unknown profile name is a hard error. + +--- + +## 6. manifest.yml Changes + +### 6.1 New top-level field: `packaging_profiles` + +```yaml +# manifest.yml (excerpt — new section added near the top) + +packaging_profiles: + minimal: + description: "JDKs and core CF utilities only. No APM agents, profilers, or JDBC drivers." + exclude: + - datadog-javaagent + - elastic-apm-agent + - azure-application-insights + - skywalking-agent + - splunk-otel-javaagent + - google-stackdriver-profiler + - open-telemetry-javaagent + - contrast-security + - newrelic + - sealights-agent + - jacoco + - jrebel + - your-kit-profiler + - jprofiler-profiler + - java-memory-assistant + - java-memory-assistant-cleanup + - luna-security-provider + - postgresql-jdbc + - mariadb-jdbc + + standard: + description: "Core + open-source APM/observability. No commercial profilers or security providers." + exclude: + - jrebel + - your-kit-profiler + - jprofiler-profiler + - contrast-security + - sealights-agent + - luna-security-provider + - java-memory-assistant + - java-memory-assistant-cleanup +``` + +No changes to the `dependencies:` entries themselves. Existing dependency declarations remain +unchanged so that the full set is still packaged when no profile or exclude flag is given. + +### 6.2 YAML schema for packaging_profiles + +``` +packaging_profiles: + : # string, no spaces, used as CLI value + description: # human-readable, shown in --help / summary + exclude: # list of dependency names (must exist in manifest) + - + - ... +``` + +--- + +## 7. libbuildpack/packager Changes + +### 7.1 models.go — new struct fields + +```go +// PackagingProfile defines a named dependency exclusion set for use at packaging time. +type PackagingProfile struct { + Description string `yaml:"description"` + Exclude []string `yaml:"exclude"` +} + +// Manifest — add PackagingProfiles field +type Manifest struct { + Language string `yaml:"language"` + Stack string `yaml:"stack"` + IncludeFiles []string `yaml:"include_files"` + PrePackage string `yaml:"pre_package"` + Dependencies Dependencies `yaml:"dependencies"` + Defaults []struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + } `yaml:"default_versions"` + PackagingProfiles map[string]PackagingProfile `yaml:"packaging_profiles"` // NEW +} +``` + +### 7.2 packager.go — exclusion resolution and filtering + +New unexported helper `resolveExclusions`: + +```go +// resolveExclusions returns the set of dependency names that should be skipped +// during packaging. Resolution order: +// 1. Profile's exclude list (if a profile is named). +// 2. Explicit --exclude names are unioned in. +// 3. Explicit --include names are removed (overrides profile exclusions). +// +// An error is returned if the profile name is unknown or if any exclude/include +// name does not exist in the manifest. +func resolveExclusions(manifest Manifest, profile string, exclude []string, include []string) (map[string]struct{}, error) { + // 1. Start with profile exclusions + result := make(map[string]struct{}) + if profile != "" { + p, ok := manifest.PackagingProfiles[profile] + if !ok { + return nil, fmt.Errorf("packaging profile %q not found in manifest", profile) + } + for _, name := range p.Exclude { + result[name] = struct{}{} + } + } + + // 2. Union with explicitly excluded names + for _, name := range exclude { + result[name] = struct{}{} + } + + // 3. Remove explicitly included names (overrides profile) + for _, name := range include { + delete(result, name) + } + + // 4. Validate: every exclude/include name must exist in the manifest + depNames := make(map[string]struct{}) + for _, d := range manifest.Dependencies { + depNames[d.Name] = struct{}{} + } + for _, name := range append(exclude, include...) { + if _, ok := depNames[name]; !ok { + return nil, fmt.Errorf("dependency %q not found in manifest", name) + } + } + + return result, nil +} +``` + +`PackageOptions` struct and updated `Package` / `PackageWithOptions` signatures: + +```go +type PackageOptions struct { + Profile string + Exclude []string + Include []string // deps to restore after profile exclusions are applied +} + +func PackageWithOptions(bpDir, cacheDir, version, stack string, cached bool, opts PackageOptions) (string, error) + +// Package delegates to PackageWithOptions with zero-value opts for backward compat +func Package(bpDir, cacheDir, version, stack string, cached bool) (string, error) { + return PackageWithOptions(bpDir, cacheDir, version, stack, cached, PackageOptions{}) +} +``` + +Updated inner dependency loop (the only logic change inside `PackageWithOptions`): + +```go + // Resolve which deps to skip BEFORE the download loop + excluded, err := resolveExclusions(manifest, opts.Profile, opts.Exclude, opts.Include) + if err != nil { + return "", err + } + + for idx, d := range manifest.Dependencies { + // Skip excluded dependencies entirely — they are not downloaded + // and are not written into the packaged manifest.yml + if _, skip := excluded[d.Name]; skip { + continue + } + + for _, s := range d.Stacks { + if stack == "" || s == stack { + dependencyMap := deps[idx] + if cached { + if file, err := downloadDependency(d, cacheDir); err != nil { + return "", err + } else { + updateDependencyMap(dependencyMap, file) + files = append(files, file) + } + } + if stack != "" { + delete(dependencyMap.(map[interface{}]interface{}), "cf_stacks") + } + dependenciesForStack = append(dependenciesForStack, dependencyMap) + break + } + } + } +``` + +Filename suffix logic (appended after the existing `cachedPart` / `stackPart` computation): + +```go + profilePart := "" + if opts.Profile != "" { + profilePart = "-" + opts.Profile + if len(opts.Exclude) > 0 || len(opts.Include) > 0 { + profilePart += "+custom" + } + } else if len(opts.Exclude) > 0 || len(opts.Include) > 0 { + profilePart = "-custom" + } + + fileName := fmt.Sprintf( + "%s_buildpack%s%s%s-v%s.zip", + manifest.Language, cachedPart, profilePart, stackPart, version, + ) +``` + +### 7.3 buildpack-packager/main.go — new CLI flags + +```go +type buildCmd struct { + cached bool + anyStack bool + version string + cacheDir string + stack string + profile string // NEW + exclude string // NEW: comma-separated, parsed before calling PackageWithOptions + include string // NEW: comma-separated, parsed before calling PackageWithOptions +} + +func (b *buildCmd) SetFlags(f *flag.FlagSet) { + f.StringVar(&b.version, "version", "", "version to build as") + f.BoolVar(&b.cached, "cached", false, "include dependencies") + f.StringVar(&b.cacheDir, "cachedir", packager.CacheDir, "cache dir") + f.StringVar(&b.stack, "stack", "", "stack to package buildpack for") + f.BoolVar(&b.anyStack, "any-stack", false, "package buildpack for any stack") + f.StringVar(&b.profile, "profile", "", "packaging profile defined in manifest.yml") // NEW + f.StringVar(&b.exclude, "exclude", "", "comma-separated dependency names to exclude") // NEW + f.StringVar(&b.include, "include", "", "comma-separated dependency names to include, overriding profile exclusions") // NEW +} + +func (b *buildCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { + // ... existing validation ... + + // Parse exclude and include lists + parseCSV := func(s string) []string { + var out []string + for _, name := range strings.Split(s, ",") { + name = strings.TrimSpace(name) + if name != "" { + out = append(out, name) + } + } + return out + } + + opts := packager.PackageOptions{ + Profile: b.profile, + Exclude: parseCSV(b.exclude), + Include: parseCSV(b.include), + } + + zipFile, err := packager.PackageWithOptions(".", b.cacheDir, b.version, b.stack, b.cached, opts) + // ... rest unchanged ... +} +``` + +Updated `Usage()` string: + +``` +build -stack |-any-stack [-cached] [-version ] + [-cachedir ] [-profile ] [-exclude ] + [-include ]: + + Creates a zip file from the current buildpack directory. + + -profile Name of a packaging profile defined in manifest.yml's + packaging_profiles section. Profiles declare which dependencies + to exclude from the cached zip. + + -exclude Comma-separated list of dependency names to exclude, in addition + to any exclusions implied by -profile. Names must exist in + manifest.yml. Example: -exclude datadog-javaagent,newrelic + + -include Comma-separated list of dependency names to force-include, + overriding exclusions implied by -profile. Useful for starting + from a restrictive profile and adding back a single dep. + Example: -profile minimal -include jprofiler-profiler +``` + +### 7.4 summary.go — list available profiles + +The `buildpack-packager summary` subcommand should be extended to print available profiles when the +manifest contains a `packaging_profiles` section: + +``` +Packaged binaries: +... + +Default binary versions: +... + +Packaging profiles: + minimal JDKs and core CF utilities only. No APM agents, profilers, or JDBC drivers. + standard Core + open-source APM/observability. No commercial profilers or security providers. +``` + +Implementation: iterate `manifest.PackagingProfiles` in sorted key order, print name + description. + +### 7.5 Backward compatibility + +The new `profile` and `exclude` parameters are added at the **end** of the `Package()` signature. +All existing callers (other buildpack tests and tools that call `packager.Package` directly) must +be updated to pass empty values: + +```go +// Before +packager.Package(bpDir, cacheDir, version, stack, cached) + +// After +packager.Package(bpDir, cacheDir, version, stack, cached, "", nil) +``` + +Since `libbuildpack` is a Go module consumed via `go install ...@latest`, this is a breaking change +to the Go API. Two options: + +**Option A — Update signature, update all callers in the same PR.** +Clean, no shims. Requires coordinating one PR across `libbuildpack` and any internal tooling that +calls `Package()` directly (currently only `buildpack-packager/main.go` and test files in +`libbuildpack` itself). + +**Option B — Introduce a new function `PackageWithOptions`.** +```go +type PackageOptions struct { + Profile string + Exclude []string +} + +func PackageWithOptions(bpDir, cacheDir, version, stack string, cached bool, opts PackageOptions) (string, error) + +// Package delegates to PackageWithOptions with zero-value opts for backward compat +func Package(bpDir, cacheDir, version, stack string, cached bool) (string, error) { + return PackageWithOptions(bpDir, cacheDir, version, stack, cached, PackageOptions{}) +} +``` + +**Recommendation**: Option B. It keeps the existing `Package()` function intact and avoids a +flag day across all consumers. + +--- + +## 8. scripts/package.sh Changes + +Each buildpack's `scripts/package.sh` needs three additions: + +1. Parse `--profile`, `--exclude`, and `--include` in the `while` loop. +2. Forward them to `buildpack-packager`. + +```bash +function main() { + local stack version cached output profile exclude include + stack="cflinuxfs4" + cached="false" + output="${ROOTDIR}/build/buildpack.zip" + profile="" # NEW + exclude="" # NEW + include="" # NEW + + while [[ "${#}" != 0 ]]; do + case "${1}" in + # ... existing cases unchanged ... + + --profile) # NEW + profile="${2}" + shift 2 + ;; + + --exclude) # NEW + exclude="${2}" + shift 2 + ;; + + --include) # NEW + include="${2}" + shift 2 + ;; + + # ... + esac + done + + package::buildpack "${version}" "${cached}" "${stack}" "${output}" "${profile}" "${exclude}" "${include}" +} + +function package::buildpack() { + local version cached stack output profile exclude include + version="${1}" + cached="${2}" + stack="${3}" + output="${4}" + profile="${5}" # NEW + exclude="${6}" # NEW + include="${7}" # NEW + + # ... existing setup ... + + local profile_flag="" exclude_flag="" include_flag="" + [[ -n "${profile}" ]] && profile_flag="--profile=${profile}" + [[ -n "${exclude}" ]] && exclude_flag="--exclude=${exclude}" + [[ -n "${include}" ]] && include_flag="--include=${include}" + + local file + file="$( + "${ROOTDIR}/.bin/buildpack-packager" build \ + "--version=${version}" \ + "--cached=${cached}" \ + "${stack_flag}" \ + ${profile_flag:+"${profile_flag}"} \ + ${exclude_flag:+"${exclude_flag}"} \ + ${include_flag:+"${include_flag}"} \ + | xargs -n1 | grep -e '\.zip$' + )" + + mv "${file}" "${output}" +} +``` + +Updated `usage()`: + +``` +package.sh --version [OPTIONS] +Packages the buildpack into a .zip file. +OPTIONS + --help -h prints the command usage + --version specifies the version number + --cached bundle dependencies (default: false) + --stack target stack (default: cflinuxfs4) + --output output path (default: build/buildpack.zip) + --profile packaging profile from manifest.yml + --exclude additional dependencies to exclude + --include dependencies to restore, overriding profile exclusions +``` + +--- + +## 9. java-buildpack Adoption + +### 9.1 manifest.yml profiles + +The following profiles are proposed for the java-buildpack. The dependency categorisation used +here mirrors the analysis of the 47 dependencies in the current `manifest.yml`. + +**Core (never excluded by any profile)**: +- JDKs: `openjdk`, `zulu`, `sapmachine` (all versions) +- CF utilities: `jvmkill`, `memory-calculator`, `auto-reconfiguration`, `java-cfenv`, + `client-certificate-mapper`, `metric-writer`, `container-security-provider`, + `cf-metrics-exporter` +- Tomcat family: `tomcat`, `tomcat-access-logging-support`, `tomcat-lifecycle-support`, + `tomcat-logging-support` +- Other frameworks: `groovy`, `spring-boot-cli` + +**`minimal` profile** — excludes everything that requires a commercial license or serves a +single vendor's ecosystem: +```yaml + minimal: + description: "JDKs, CF utilities, Tomcat, and common frameworks only." + exclude: + - datadog-javaagent + - elastic-apm-agent + - azure-application-insights + - skywalking-agent + - splunk-otel-javaagent + - google-stackdriver-profiler + - open-telemetry-javaagent + - contrast-security + - newrelic + - sealights-agent + - jacoco + - jrebel + - your-kit-profiler + - jprofiler-profiler + - java-memory-assistant + - java-memory-assistant-cleanup + - luna-security-provider + - postgresql-jdbc + - mariadb-jdbc +``` +Result: 47 → 28 dependencies bundled. + +**`standard` profile** — adds open-source observability (OTel, JaCoCo) and JDBC drivers, removes +commercial profilers and specialist security providers: +```yaml + standard: + description: "Core + open-source APM, OTel, and JDBC drivers. No commercial agents or profilers." + exclude: + - datadog-javaagent + - elastic-apm-agent + - azure-application-insights + - skywalking-agent + - splunk-otel-javaagent + - google-stackdriver-profiler + - contrast-security + - newrelic + - sealights-agent + - jrebel + - your-kit-profiler + - jprofiler-profiler + - java-memory-assistant + - java-memory-assistant-cleanup + - luna-security-provider +``` +Result: 47 → 32 dependencies bundled. + +### 9.2 Typical usage examples + +```bash +# Current behaviour — unchanged +./scripts/package.sh --cached + +# Air-gapped environment, only OpenJDK + Tomcat needed +./scripts/package.sh --cached --profile minimal + +# Standard ops team buildpack — OTel and JDBC included, commercial agents excluded +./scripts/package.sh --cached --profile standard + +# Standard profile but also drop jacoco (not needed on this foundation) +./scripts/package.sh --cached --profile standard --exclude jacoco + +# One-off: full cached buildpack minus the two agents we don't have licences for +./scripts/package.sh --cached --exclude jrebel,your-kit-profiler,jprofiler-profiler + +# Standard profile, but this foundation also needs jprofiler for triage +./scripts/package.sh --cached --profile standard --include jprofiler-profiler +``` + +--- + +## 10. Implementation Plan + +The work is broken into three sequential phases. Phases 1 and 2 are in `libbuildpack`, Phase 3 is +in `java-buildpack` (and optionally in other buildpacks). + +### Phase 1 — libbuildpack core (packager library) + +| # | File | Change | Notes | +|---|---|---|---| +| 1.1 | `packager/models.go` | Add `PackagingProfile` struct and `PackagingProfiles` field on `Manifest` | ~15 lines | +| 1.2 | `packager/packager.go` | Add `resolveExclusions()` helper (profile + exclude + include logic) | ~40 lines | +| 1.3 | `packager/packager.go` | Add `PackageOptions` struct, `PackageWithOptions`, update `Package` to delegate | ~20 lines | +| 1.4 | `packager/packager.go` | Apply exclusion filter in dependency loop, update filename logic | ~15 lines | +| 1.5 | `packager/summary.go` | Print `packaging_profiles` section in `Summary()` | ~20 lines | +| 1.6 | `packager/packager_test.go` | Test cases for exclude, include, profile, combined, unknown name errors | ~100 lines | +| 1.7 | `packager/models_test.go` | Test `resolveExclusions` edge cases | ~50 lines | + +**Entry criteria**: existing tests pass on `main`. +**Exit criteria**: all new tests pass, `packager.Package()` signature unchanged, `PackageWithOptions` works. + +### Phase 2 — buildpack-packager CLI + +| # | File | Change | Notes | +|---|---|---|---| +| 2.1 | `packager/buildpack-packager/main.go` | Add `--profile`, `--exclude`, and `--include` flags to `buildCmd` | ~30 lines | +| 2.2 | `packager/buildpack-packager/main.go` | Parse comma-separated `--exclude` and `--include` into `[]string` | ~15 lines | +| 2.3 | `packager/buildpack-packager/main.go` | Update `Usage()` string | ~15 lines | + +**Exit criteria**: `buildpack-packager build --help` shows new flags; manual smoke test against +java-buildpack `manifest.yml` produces expected zip sizes. + +### Phase 3 — java-buildpack adoption + +| # | File | Change | Notes | +|---|---|---|---| +| 3.1 | `manifest.yml` | Add `packaging_profiles` section with `minimal` and `standard` | ~40 lines | +| 3.2 | `scripts/package.sh` | Add `--profile` / `--exclude` / `--include` flag parsing and forwarding | ~20 lines | +| 3.3 | `scripts/package.sh` | Update `usage()` | ~5 lines | + +**Exit criteria**: +- `./scripts/package.sh --cached --profile minimal` produces a zip with 28 dependencies. +- `./scripts/package.sh --cached --profile minimal --include jprofiler-profiler` produces a zip with 29 dependencies. +- `./scripts/package.sh --cached` produces a zip with 47 dependencies (unchanged). +- `buildpack-packager summary` lists the two profiles. + +### Phase 4 (optional) — other buildpacks + +Any buildpack team can independently add a `packaging_profiles` section to their `manifest.yml` +and the two-line script update to `scripts/package.sh`. No further changes to `libbuildpack` are +required. + +--- + +## 11. Testing Strategy + +### Unit tests (libbuildpack) + +| Scenario | Expected outcome | +|---|---| +| `PackageWithOptions` called with no profile, no exclude, no include | All stack-matching deps bundled (existing behaviour) | +| `PackageWithOptions` called with `exclude=["dep-a"]` | `dep-a` absent from zip manifest and not downloaded | +| `PackageWithOptions` called with valid `profile="minimal"` | Profile's exclude list applied correctly | +| `PackageWithOptions` called with `profile` + extra `exclude` | Union of both exclude lists applied | +| `PackageWithOptions` called with `profile` + `include` | Named dep restored; rest of profile exclusions still applied | +| `PackageWithOptions` called with `profile` + `exclude` + `include` | exclude adds, include removes from profile exclusions | +| `PackageWithOptions` called with `include` but no `profile` | No-op (nothing was excluded); warning emitted | +| `PackageWithOptions` called with unknown `profile` name | Returns error containing profile name | +| `PackageWithOptions` called with `exclude` containing unknown dep name | Returns error containing dep name | +| `PackageWithOptions` called with `include` containing unknown dep name | Returns error containing dep name | +| `Package` called (legacy signature) | Delegates to `PackageWithOptions` with zero opts; full behaviour unchanged | +| Zip filename — profile only | Contains `-` segment, no `+custom` | +| Zip filename — profile + include or exclude | Contains `-+custom` segment | +| Zip filename — exclude only (no profile) | Contains `-custom` segment | +| Zip filename — neither | Original filename (backward compat) | + +New fixture: `packager/fixtures/with_profiles/manifest.yml` — a minimal manifest with a +`packaging_profiles` section used by the new tests. + +### Integration / smoke tests (java-buildpack CI) + +The existing `ci/package-test.sh` script can be extended to: + +1. Build `--profile minimal` and assert the zip does **not** contain `dependencies/*/dd-java-agent*`. +2. Build `--cached` (no profile) and assert the zip **does** contain that file. +3. Build `--exclude datadog-javaagent` and assert the same. + +These can run without downloading real binaries by mocking the packager's HTTP client (as the +existing packager tests already do via `httpmock`). + +--- + +## 12. Rollout Strategy + +1. **Land Phase 1+2 in `libbuildpack`** as a single PR. Tagging a new release is not strictly + required because all buildpacks use `@latest`, but a tag is recommended for traceability. + +2. **Land Phase 3 in `java-buildpack`** once the `libbuildpack` PR is merged and the binary + installed at `.bin/buildpack-packager` is refreshed in CI. + +3. **Communicate to other buildpack teams** that `--profile`, `--exclude`, and `--include` are now available. + Each team can adopt on their own schedule by adding `packaging_profiles` to their manifest. + +4. **No operator action required** for existing deployments. Operators who build the buildpack + without `--profile`, `--exclude`, or `--include` get identical output to today. + +--- + +## 13. Open Questions + +| # | Question | Options | Decision | +|---|---|---|---| +| Q1 | Should `--exclude`/`--include` on an uncached buildpack be an error or a no-op? | Error vs no-op with warning | Recommend: **no-op with a warning** — the flags are meaningless for uncached builds but not necessarily a mistake | +| Q2 | Should profile names be validated for character set? (e.g., no spaces, no slashes) | Yes (reject invalid names) vs no | Recommend: **yes**, restrict to `[a-z0-9_-]+` to keep filenames safe | +| Q3 | Should excluded dependencies be completely absent from the packaged `manifest.yml`? | Absent (cleaner, smaller manifest) vs present with a flag | Recommend: **absent** — a smaller manifest also means faster version resolution at staging time | +| Q4 | Should `packaging_profiles` entries be validated at `buildpack-packager summary` time even when not building? | Yes (catches stale exclusion lists) vs no | Recommend: **yes**, warn if a profile excludes a name not in `dependencies` | +| Q5 | Should we also support `include` lists in profiles (whitelist model in manifest.yml)? | Yes (more explicit) vs no (requires updating all profiles when a new dep is added) | Recommend: **no for now** — the CLI `--include` flag covers the override use case without complicating the manifest schema | diff --git a/manifest.yml b/manifest.yml index e1492243a..37f8fa5a0 100644 --- a/manifest.yml +++ b/manifest.yml @@ -13,6 +13,47 @@ include_files: - bin/supply - manifest.yml pre_package: scripts/build.sh +packaging_profiles: + minimal: + description: "JDKs, CF utilities, Tomcat, and common frameworks only. No APM agents, profilers, or JDBC drivers." + exclude: + - datadog-javaagent + - elastic-apm-agent + - azure-application-insights + - skywalking-agent + - splunk-otel-javaagent + - google-stackdriver-profiler + - open-telemetry-javaagent + - contrast-security + - newrelic + - sealights-agent + - jacoco + - jrebel + - your-kit-profiler + - jprofiler-profiler + - java-memory-assistant + - java-memory-assistant-cleanup + - luna-security-provider + - postgresql-jdbc + - mariadb-jdbc + standard: + description: "Core + open-source APM, OTel, and JDBC drivers. No commercial agents or profilers." + exclude: + - datadog-javaagent + - elastic-apm-agent + - azure-application-insights + - skywalking-agent + - splunk-otel-javaagent + - google-stackdriver-profiler + - contrast-security + - newrelic + - sealights-agent + - jrebel + - your-kit-profiler + - jprofiler-profiler + - java-memory-assistant + - java-memory-assistant-cleanup + - luna-security-provider default_versions: - name: openjdk version: 17.x diff --git a/scripts/package.sh b/scripts/package.sh index aab0b6bb8..161b073f9 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -14,10 +14,13 @@ source "${ROOTDIR}/scripts/.util/tools.sh" source "${ROOTDIR}/scripts/.util/print.sh" function main() { - local stack version cached output + local stack version cached output profile exclude include stack="cflinuxfs4" cached="false" output="${ROOTDIR}/build/buildpack.zip" + profile="" + exclude="" + include="" while [[ "${#}" != 0 ]]; do case "${1}" in @@ -41,6 +44,21 @@ function main() { shift 2 ;; + --profile) + profile="${2}" + shift 2 + ;; + + --exclude) + exclude="${2}" + shift 2 + ;; + + --include) + include="${2}" + shift 2 + ;; + --help|-h) shift 1 usage @@ -62,7 +80,7 @@ function main() { echo "No version specified, using VERSION file: ${version}" fi - package::buildpack "${version}" "${cached}" "${stack}" "${output}" + package::buildpack "${version}" "${cached}" "${stack}" "${output}" "${profile}" "${exclude}" "${include}" } @@ -76,15 +94,21 @@ OPTIONS --cached cache the buildpack dependencies (default: false) --stack specifies the stack (default: cflinuxfs4) --output output file path (default: build/buildpack.zip) + --profile packaging profile from manifest.yml (e.g. minimal, standard) + --exclude comma-separated dependency names to exclude (cached only) + --include comma-separated dependency names to restore, overriding profile exclusions (cached only) USAGE } function package::buildpack() { - local version cached stack output + local version cached stack output profile exclude include version="${1}" cached="${2}" stack="${3}" output="${4}" + profile="${5:-}" + exclude="${6:-}" + include="${7:-}" mkdir -p "$(dirname "${output}")" @@ -98,12 +122,20 @@ function package::buildpack() { stack_flag="--stack=${stack}" fi + local profile_flag="" exclude_flag="" include_flag="" + [[ -n "${profile}" ]] && profile_flag="--profile=${profile}" + [[ -n "${exclude}" ]] && exclude_flag="--exclude=${exclude}" + [[ -n "${include}" ]] && include_flag="--include=${include}" + local file file="$( "${ROOTDIR}/.bin/buildpack-packager" build \ "--version=${version}" \ "--cached=${cached}" \ "${stack_flag}" \ + ${profile_flag:+"${profile_flag}"} \ + ${exclude_flag:+"${exclude_flag}"} \ + ${include_flag:+"${include_flag}"} \ | xargs -n1 | grep -e '\.zip$' )"