From b1741e547d67ed2bafaf03deb77e7edc718f8aa5 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 9 Sep 2026 04:08:48 +0200 Subject: [PATCH 1/6] rules_go: make instrumented artifacts deterministic --- .github/workflows/shared-validation.yml | 7 +- CONTRIBUTING.md | 5 +- third_party/rgo/v0_60_0/base.CHANGED_FILES.md | 5 +- .../rgo/v0_60_0/base/go/tools/builders/asm.go | 9 +- .../base/go/tools/builders/compilepkg.go | 78 ++++- .../base/go/tools/builders/compilepkg_test.go | 78 +++++ .../base/go/tools/builders/importcfg.go | 2 +- .../base/go/tools/builders/module_proxy.go | 21 +- .../tools/builders/orchestrion_skip_test.go | 11 +- .../v0_60_0/base/go/tools/builders/stdlib.go | 5 + third_party/rgo/v0_61_1/base.CHANGED_FILES.md | 5 +- .../rgo/v0_61_1/base/go/tools/builders/asm.go | 9 +- .../base/go/tools/builders/compilepkg.go | 78 ++++- .../base/go/tools/builders/compilepkg_test.go | 78 +++++ .../base/go/tools/builders/importcfg.go | 2 +- .../base/go/tools/builders/module_proxy.go | 21 +- .../tools/builders/orchestrion_skip_test.go | 11 +- .../v0_61_1/base/go/tools/builders/stdlib.go | 5 + third_party/rgo/v0_62_0/base.CHANGED_FILES.md | 5 +- .../rgo/v0_62_0/base/go/tools/builders/asm.go | 9 +- .../base/go/tools/builders/compilepkg.go | 78 ++++- .../base/go/tools/builders/compilepkg_test.go | 78 +++++ .../base/go/tools/builders/importcfg.go | 2 +- .../base/go/tools/builders/module_proxy.go | 21 +- .../tools/builders/orchestrion_skip_test.go | 11 +- .../v0_62_0/base/go/tools/builders/stdlib.go | 5 + third_party/rgo/v0_63_0/base.CHANGED_FILES.md | 5 +- .../rgo/v0_63_0/base/go/tools/builders/asm.go | 9 +- .../base/go/tools/builders/compilepkg.go | 78 ++++- .../base/go/tools/builders/compilepkg_test.go | 78 +++++ .../base/go/tools/builders/importcfg.go | 2 +- .../base/go/tools/builders/module_proxy.go | 21 +- .../tools/builders/orchestrion_skip_test.go | 11 +- .../v0_63_0/base/go/tools/builders/stdlib.go | 5 + .../v0_60_0/base/0001-full-delta.patch | 257 ++++++++++++++-- .../v0_61_1/base/0001-full-delta.patch | 257 ++++++++++++++-- .../v0_62_0/base/0001-full-delta.patch | 257 ++++++++++++++-- .../v0_63_0/base/0001-full-delta.patch | 255 ++++++++++++++-- .../profiles/workspace_runtime.json | 1 + tools/dev/verify_rules_go_profiles.py | 276 +++++++++++++++++- .../python/test_rules_go_profile_tools.py | 107 +++++++ 41 files changed, 2045 insertions(+), 213 deletions(-) diff --git a/.github/workflows/shared-validation.yml b/.github/workflows/shared-validation.yml index d83de416..2f1bff13 100644 --- a/.github/workflows/shared-validation.yml +++ b/.github/workflows/shared-validation.yml @@ -256,7 +256,7 @@ jobs: run: python3 tools/dev/check_release_archive_contents.py rules-go-fork-profile-shard: - name: rules-go-fork-profile-shard (${{ matrix.upstream }}) + name: rules-go-fork-profile-shard (${{ matrix.upstream }}, ${{ matrix.runner }}) needs: rules-go-fork-drift-matrix if: ${{ !inputs.docs_only && inputs.run_rules_go_fork_drift }} strategy: @@ -264,7 +264,10 @@ jobs: max-parallel: 4 matrix: upstream: ${{ fromJSON(needs.rules-go-fork-drift-matrix.outputs.upstreams) }} - runs-on: ubuntu-latest + runner: + - ubuntu-latest + - ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 82c3d025..ee498ad7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -228,7 +228,10 @@ This product includes software developed at Datadog - Utility/lint lanes: - module version alignment check (`tools/dev/check_module_versions.py`) - `.bazelversion` parity check (`tools/dev/check_bazelversion_sync.py`) - - global fork drift checks plus one consumer patch-profile shard per supported `rules_go` upstream + - global fork drift checks plus x86-64 and ARM64 consumer patch-profile shards + for every supported `rules_go` upstream; each profile shard rebuilds the + cache-critical Test Optimization actions twice from isolated output roots + and requires identical action keys and output bytes - shell scripts, PowerShell, Buildifier, gofmt, schema sync checks, fixture JSON checks, and Python tooling tests - Workflow dependency pinning: - Keep GitHub Actions pinned by commit SHA and preserve the `# vX.Y.Z` comment. diff --git a/third_party/rgo/v0_60_0/base.CHANGED_FILES.md b/third_party/rgo/v0_60_0/base.CHANGED_FILES.md index ab3d351e..bb1f80df 100644 --- a/third_party/rgo/v0_60_0/base.CHANGED_FILES.md +++ b/third_party/rgo/v0_60_0/base.CHANGED_FILES.md @@ -12,8 +12,8 @@ This file is generated. Do not edit by hand. ## Summary -- Total changed paths: `55` -- Modified files: `31` +- Total changed paths: `56` +- Modified files: `32` - Added files: `24` - Removed files: `0` @@ -37,6 +37,7 @@ This file is generated. Do not edit by hand. - `go/private/rules/transition.bzl` - `go/tools/builders/BUILD.bazel` - `go/tools/builders/ar.go` +- `go/tools/builders/asm.go` - `go/tools/builders/builder.go` - `go/tools/builders/cc.go` - `go/tools/builders/compilepkg.go` diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/asm.go b/third_party/rgo/v0_60_0/base/go/tools/builders/asm.go index 6704be70..5e7d3404 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/asm.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/asm.go @@ -111,7 +111,7 @@ func buildSymabisFile(goenv *env, packagePath string, sFiles, hFiles []fileInfo, return symabisName, err } -func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, outPath string) error { +func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, trimPath, outPath string) error { args := goenv.goTool("asm") args = append(args, asmFlags...) // The package path has to be specified as of Go 1.19 or the resulting @@ -121,10 +121,13 @@ func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, outPath args = append(args, "-p", packagePath) } args = append(args, ASM_DEFINES...) - args = append(args, "-trimpath", ".") + if trimPath == "" { + trimPath = abs(".") + } + args = append(args, "-trimpath", trimPath) args = append(args, "-o", outPath) args = append(args, "--", srcPath) - absArgs(args, []string{"-I", "-o", "-trimpath"}) + absArgs(args, []string{"-I", "-o"}) return goenv.runCommand(args) } diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/compilepkg.go b/third_party/rgo/v0_60_0/base/go/tools/builders/compilepkg.go index cb5b10ac..5356b08c 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/compilepkg.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/compilepkg.go @@ -210,7 +210,7 @@ func publishSyntheticTestmainPackagefiles(data []byte, outputDir string) ([]byte } published[publishedPath] = true } - lines[index] = "packagefile " + packagePath + "=" + publishedPath + lines[index] = "packagefile " + packagePath + "=" + execrootRelativePath(publishedPath) } return []byte(strings.Join(lines, "\n")), nil } @@ -770,7 +770,7 @@ func compileArchive( } for i, sSrc := range srcs.sSrcs { obj := filepath.Join(workDir, fmt.Sprintf("s%d.o", i)) - if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, obj); err != nil { + if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, "", obj); err != nil { return err } objFiles = append(objFiles, obj) @@ -1353,7 +1353,7 @@ func modulePackageCommandEnv(goenv *env, exportRoot string) ([]string, error) { env = setEnv(env, "GIT_CONFIG_GLOBAL", os.DevNull) env = setEnv(env, "GIT_CONFIG_NOSYSTEM", "1") env = setEnv(env, "GIT_TERMINAL_PROMPT", "0") - env = ensureGoFlagsModMode(env) + env = ensureSyntheticModuleGoFlags(env) if getEnv(env, "HOME") == "" { homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -1923,13 +1923,17 @@ func writeSharedSyntheticTestmainHelperBundle(outputDir string, compiled map[str if published := publishedPaths[source]; published != "" { return published, nil } - digest, err := fullDigestFile(source) + data, err := os.ReadFile(source) if err != nil { - return "", fmt.Errorf("digest shared synthetic helper archive %s: %w", source, err) + return "", fmt.Errorf("read shared synthetic helper archive %s: %w", source, err) } + if err := normalizePublishedGoArchiveBuildID(data); err != nil { + return "", fmt.Errorf("normalize shared synthetic helper archive %s: %w", source, err) + } + digest := fmt.Sprintf("%x", sha256.Sum256(data)) destination := filepath.Join(outputDir, "archives", digest+".a") if _, err := os.Stat(destination); errors.Is(err, os.ErrNotExist) { - if err := copyArchiveFile(source, destination); err != nil { + if err := os.WriteFile(destination, data, 0o644); err != nil { return "", fmt.Errorf("publish shared synthetic helper archive %s: %w", source, err) } } else if err != nil { @@ -1970,6 +1974,49 @@ func writeSharedSyntheticTestmainHelperBundle(outputDir string, compiled map[str return writeJSONAtomically(filepath.Join(outputDir, cacheManifestFileName), manifest) } +// normalizePublishedGoArchiveBuildID removes execroot-specific action IDs from +// a copied helper archive. The Go cache keeps the original archive and build ID; +// only the declared Bazel output receives the stable content/content form. +func normalizePublishedGoArchiveBuildID(data []byte) error { + if !bytes.HasPrefix(data, []byte("!\n")) { + return nil + } + header := data + if len(header) > 1024 { + header = header[:1024] + } + const marker = "\nbuild id \"" + start := bytes.Index(header, []byte(marker)) + if start < 0 { + return nil + } + start += len(marker) + end := bytes.IndexByte(header[start:], '"') + if end < 0 { + return errors.New("unterminated Go archive build ID") + } + end += start + buildID := string(header[start:end]) + if buildID == "" { + return nil + } + parts := strings.Split(buildID, "/") + if len(parts) != 2 || len(parts[0]) != len(parts[1]) { + return fmt.Errorf("unexpected Go archive build ID %q", buildID) + } + stableBuildID := parts[1] + "/" + parts[1] + for offset := 0; ; { + index := bytes.Index(data[offset:], []byte(buildID)) + if index < 0 { + break + } + index += offset + copy(data[index:index+len(buildID)], stableBuildID) + offset = index + len(buildID) + } + return nil +} + // reusableSharedSyntheticTestmainHelperBundle returns the declared shared // bundle only when it covers the requested roots and does not overlap archives // supplied by the current Bazel target. Overlap falls back to the existing @@ -2358,7 +2405,7 @@ func compileSyntheticTestmainSourcePackage(goenv *env, pack, workDir, moduleDir, stem := fmt.Sprintf("%x", sha256.Sum256([]byte(meta.ImportPath)))[:16] outLinkobjPath := filepath.Join(archiveDir, stem+".a") outInterfacePath := filepath.Join(archiveDir, stem+".iface.a") - trimPath, err := createTrimPath() + trimPath, err := syntheticSourceTrimPath(meta.Dir, meta.ImportPath) if err != nil { return "", "", fmt.Errorf("create trimpath for %s: %w", pkg, err) } @@ -2401,7 +2448,7 @@ func compileSyntheticTestmainSourcePackage(goenv *env, pack, workDir, moduleDir, } for i, sSrc := range filteredSrcs.sSrcs { obj := filepath.Join(packageWorkDir, fmt.Sprintf("s%d.o", i)) - if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, obj); err != nil { + if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, trimPath, obj); err != nil { return "", "", fmt.Errorf("assemble synthetic helper %s: %w", pkg, err) } objFiles = append(objFiles, obj) @@ -2429,6 +2476,21 @@ func compileSyntheticTestmainSourcePackage(goenv *env, pack, workDir, moduleDir, return result.compilePath, result.linkPath, nil } +// syntheticSourceTrimPath gives module sources a stable logical prefix. The +// physical GOMODCACHE belongs to the action sandbox and must not reach archives +// that Bazel may publish to a shared cache. +func syntheticSourceTrimPath(sourceDir, importPath string) (string, error) { + trimPath, err := createTrimPath() + if err != nil { + return "", err + } + sourceDir = strings.TrimSpace(sourceDir) + if sourceDir == "" { + return trimPath, nil + } + return trimPath + ";" + abs(sourceDir) + "=>" + strings.TrimSpace(importPath), nil +} + func shouldSkipOrchestrionForImportPath(importPath string) bool { if strings.HasPrefix(importPath, "github.com/bazelbuild/rules_go/go/tools/") { return true diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/compilepkg_test.go b/third_party/rgo/v0_60_0/base/go/tools/builders/compilepkg_test.go index 320fb66e..91065d66 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/compilepkg_test.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/compilepkg_test.go @@ -140,6 +140,51 @@ func TestSharedSyntheticTestmainHelperBundleRoundTrip(t *testing.T) { } } +func TestSharedSyntheticTestmainHelperBundleNormalizesGoBuildIDs(t *testing.T) { + const contentID = "cccccccccccccccccccc" + archive := func(actionID string) []byte { + buildID := actionID + "/" + contentID + return []byte("!\narchive header\ngo object test\nbuild id \"" + buildID + "\"\npayload build id \"" + buildID + "\"\n") + } + + sourceDir := t.TempDir() + firstSource := filepath.Join(sourceDir, "first.a") + secondSource := filepath.Join(sourceDir, "second.a") + if err := os.WriteFile(firstSource, archive("aaaaaaaaaaaaaaaaaaaa"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(secondSource, archive("bbbbbbbbbbbbbbbbbbbb"), 0o644); err != nil { + t.Fatal(err) + } + + bundleDir := filepath.Join(t.TempDir(), "shared-helpers") + compiled := map[string]compiledModuleArchive{ + "example.com/first": {compilePath: firstSource, linkPath: firstSource}, + "example.com/second": {compilePath: secondSource, linkPath: secondSource}, + } + if err := writeSharedSyntheticTestmainHelperBundle(bundleDir, compiled, orchestrionModeTestOptimization); err != nil { + t.Fatal(err) + } + + manifest, err := readSyntheticTestmainHelperManifest(bundleDir) + if err != nil { + t.Fatal(err) + } + firstPath := manifest.Packages["example.com/first"].CompilePath + secondPath := manifest.Packages["example.com/second"].CompilePath + if firstPath != secondPath { + t.Fatalf("equivalent archives published at different paths: %q != %q", firstPath, secondPath) + } + published, err := os.ReadFile(filepath.Join(bundleDir, firstPath)) + if err != nil { + t.Fatal(err) + } + wantBuildID := contentID + "/" + contentID + if got := strings.Count(string(published), wantBuildID); got != 2 { + t.Fatalf("normalized build ID occurs %d times, want 2", got) + } +} + func TestSharedSyntheticTestmainHelperBundleFallsBackOnTargetOverlap(t *testing.T) { rootPackage := syntheticTestmainRootPackagesTestOptimization[0].packagePath sourceDir := t.TempDir() @@ -539,6 +584,39 @@ func TestSyntheticTestmainHelperModuleCacheRootIsStable(t *testing.T) { } } +func TestSyntheticModuleGoFlagsRequireTrimpath(t *testing.T) { + tests := []struct { + name string + flags string + want string + }{ + {name: "empty", want: "-mod=mod -trimpath"}, + {name: "preserve module mode", flags: "-mod=readonly", want: "-mod=readonly -trimpath"}, + {name: "preserve trimpath", flags: "-trimpath -mod=vendor", want: "-trimpath -mod=vendor"}, + {name: "override disabled trimpath", flags: "-trimpath=false", want: "-mod=mod -trimpath=false -trimpath"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + env := ensureSyntheticModuleGoFlags([]string{"GOFLAGS=" + test.flags}) + if got := getEnv(env, "GOFLAGS"); got != test.want { + t.Fatalf("GOFLAGS = %q, want %q", got, test.want) + } + }) + } +} + +func TestSyntheticSourceTrimPathReplacesModuleCacheRoot(t *testing.T) { + sourceDir := filepath.Join(t.TempDir(), "pkg", "mod", "example.com", "helper@v1.0.0") + got, err := syntheticSourceTrimPath(sourceDir, "example.com/helper") + if err != nil { + t.Fatalf("syntheticSourceTrimPath error: %v", err) + } + want := abs(sourceDir) + "=>example.com/helper" + if !strings.Contains(got, want) { + t.Fatalf("syntheticSourceTrimPath = %q, want mapping %q", got, want) + } +} + // TestWithSyntheticTestmainModuleCacheEnvRestoresEnvironment verifies that the // synthetic helper wrapper points subprocess cache env vars at the helper // cache root during execution and restores the caller's environment after. diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/importcfg.go b/third_party/rgo/v0_60_0/base/go/tools/builders/importcfg.go index b6298e6c..7c19d36a 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/importcfg.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/importcfg.go @@ -787,7 +787,7 @@ func resolveModuleExportsForPackagesWithRoot(goenv *env, packages []string, orch if err != nil { return nil, fmt.Errorf("prepare module resolution env: %w", err) } - cmd.Env = ensureGoFlagsModMode(cmd.Env) + cmd.Env = ensureSyntheticModuleGoFlags(cmd.Env) if getEnv(cmd.Env, "HOME") == "" { homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") if err := os.MkdirAll(homePath, 0o755); err != nil { diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/module_proxy.go b/third_party/rgo/v0_60_0/base/go/tools/builders/module_proxy.go index b44c0f3b..91b0ba4b 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/module_proxy.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/module_proxy.go @@ -83,15 +83,22 @@ func normalizeGoModuleResolutionEnv(env []string) ([]string, error) { return env, nil } -func ensureGoFlagsModMode(env []string) []string { +// ensureSyntheticModuleGoFlags keeps helper module builds offline-compatible +// and removes their temporary module-cache roots from compiled archives. +func ensureSyntheticModuleGoFlags(env []string) []string { goFlags := strings.TrimSpace(getEnv(env, "GOFLAGS")) - if strings.Contains(goFlags, "-mod=") { - return env + if !strings.Contains(goFlags, "-mod=") { + goFlags = strings.TrimSpace("-mod=mod " + goFlags) } - if goFlags == "" { - goFlags = "-mod=mod" - } else { - goFlags = "-mod=mod " + goFlags + trimPathEnabled := false + for _, flag := range strings.Fields(goFlags) { + if flag == "-trimpath" || flag == "-trimpath=true" { + trimPathEnabled = true + break + } + } + if !trimPathEnabled { + goFlags += " -trimpath" } return setEnv(env, "GOFLAGS", goFlags) } diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_skip_test.go b/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_skip_test.go index 32c0cf4b..00d37c1f 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_skip_test.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion_skip_test.go @@ -294,6 +294,14 @@ func TestSyntheticHelperManifestKeepsExternalAbsoluteDependencies(t *testing.T) func TestSyntheticTestmainManifestPublishesExternalArchives(t *testing.T) { execroot := t.TempDir() + previousDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(execroot); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(previousDir) }) externalArchive := filepath.Join(t.TempDir(), "helper.a") if err := os.WriteFile(externalArchive, []byte("helper archive"), 0o644); err != nil { t.Fatal(err) @@ -336,7 +344,8 @@ func TestSyntheticTestmainManifestPublishesExternalArchives(t *testing.T) { t.Fatalf("published helper archive count = %d, want 1", len(entries)) } publishedArchive := filepath.Join(helperOutputDir, entries[0].Name()) - if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedArchive) { + publishedRelative := execrootRelativePath(publishedArchive) + if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedRelative) { t.Fatalf("manifest does not reference published helper %q:\n%s", publishedArchive, gotManifest) } publishedData, err := os.ReadFile(publishedArchive) diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib.go b/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib.go index 65244aa7..8ea63311 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/stdlib.go @@ -270,6 +270,11 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } } installArgs := goenv.goCmd("install", "-toolexec", toolexec) + if *orchestrion != "" { + // Helper module exports use Go's reproducible build mode. Compile the + // woven stdlib with the same flag so imported package fingerprints match. + installArgs = append(installArgs, "-trimpath") + } if len(build.Default.BuildTags) > 0 { installArgs = append(installArgs, "-tags", strings.Join(build.Default.BuildTags, ",")) } diff --git a/third_party/rgo/v0_61_1/base.CHANGED_FILES.md b/third_party/rgo/v0_61_1/base.CHANGED_FILES.md index 91b81cec..d5c85f66 100644 --- a/third_party/rgo/v0_61_1/base.CHANGED_FILES.md +++ b/third_party/rgo/v0_61_1/base.CHANGED_FILES.md @@ -12,8 +12,8 @@ This file is generated. Do not edit by hand. ## Summary -- Total changed paths: `55` -- Modified files: `31` +- Total changed paths: `56` +- Modified files: `32` - Added files: `24` - Removed files: `0` @@ -37,6 +37,7 @@ This file is generated. Do not edit by hand. - `go/private/rules/transition.bzl` - `go/tools/builders/BUILD.bazel` - `go/tools/builders/ar.go` +- `go/tools/builders/asm.go` - `go/tools/builders/builder.go` - `go/tools/builders/cc.go` - `go/tools/builders/compilepkg.go` diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/asm.go b/third_party/rgo/v0_61_1/base/go/tools/builders/asm.go index 6704be70..5e7d3404 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/asm.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/asm.go @@ -111,7 +111,7 @@ func buildSymabisFile(goenv *env, packagePath string, sFiles, hFiles []fileInfo, return symabisName, err } -func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, outPath string) error { +func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, trimPath, outPath string) error { args := goenv.goTool("asm") args = append(args, asmFlags...) // The package path has to be specified as of Go 1.19 or the resulting @@ -121,10 +121,13 @@ func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, outPath args = append(args, "-p", packagePath) } args = append(args, ASM_DEFINES...) - args = append(args, "-trimpath", ".") + if trimPath == "" { + trimPath = abs(".") + } + args = append(args, "-trimpath", trimPath) args = append(args, "-o", outPath) args = append(args, "--", srcPath) - absArgs(args, []string{"-I", "-o", "-trimpath"}) + absArgs(args, []string{"-I", "-o"}) return goenv.runCommand(args) } diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/compilepkg.go b/third_party/rgo/v0_61_1/base/go/tools/builders/compilepkg.go index cb5b10ac..5356b08c 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/compilepkg.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/compilepkg.go @@ -210,7 +210,7 @@ func publishSyntheticTestmainPackagefiles(data []byte, outputDir string) ([]byte } published[publishedPath] = true } - lines[index] = "packagefile " + packagePath + "=" + publishedPath + lines[index] = "packagefile " + packagePath + "=" + execrootRelativePath(publishedPath) } return []byte(strings.Join(lines, "\n")), nil } @@ -770,7 +770,7 @@ func compileArchive( } for i, sSrc := range srcs.sSrcs { obj := filepath.Join(workDir, fmt.Sprintf("s%d.o", i)) - if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, obj); err != nil { + if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, "", obj); err != nil { return err } objFiles = append(objFiles, obj) @@ -1353,7 +1353,7 @@ func modulePackageCommandEnv(goenv *env, exportRoot string) ([]string, error) { env = setEnv(env, "GIT_CONFIG_GLOBAL", os.DevNull) env = setEnv(env, "GIT_CONFIG_NOSYSTEM", "1") env = setEnv(env, "GIT_TERMINAL_PROMPT", "0") - env = ensureGoFlagsModMode(env) + env = ensureSyntheticModuleGoFlags(env) if getEnv(env, "HOME") == "" { homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -1923,13 +1923,17 @@ func writeSharedSyntheticTestmainHelperBundle(outputDir string, compiled map[str if published := publishedPaths[source]; published != "" { return published, nil } - digest, err := fullDigestFile(source) + data, err := os.ReadFile(source) if err != nil { - return "", fmt.Errorf("digest shared synthetic helper archive %s: %w", source, err) + return "", fmt.Errorf("read shared synthetic helper archive %s: %w", source, err) } + if err := normalizePublishedGoArchiveBuildID(data); err != nil { + return "", fmt.Errorf("normalize shared synthetic helper archive %s: %w", source, err) + } + digest := fmt.Sprintf("%x", sha256.Sum256(data)) destination := filepath.Join(outputDir, "archives", digest+".a") if _, err := os.Stat(destination); errors.Is(err, os.ErrNotExist) { - if err := copyArchiveFile(source, destination); err != nil { + if err := os.WriteFile(destination, data, 0o644); err != nil { return "", fmt.Errorf("publish shared synthetic helper archive %s: %w", source, err) } } else if err != nil { @@ -1970,6 +1974,49 @@ func writeSharedSyntheticTestmainHelperBundle(outputDir string, compiled map[str return writeJSONAtomically(filepath.Join(outputDir, cacheManifestFileName), manifest) } +// normalizePublishedGoArchiveBuildID removes execroot-specific action IDs from +// a copied helper archive. The Go cache keeps the original archive and build ID; +// only the declared Bazel output receives the stable content/content form. +func normalizePublishedGoArchiveBuildID(data []byte) error { + if !bytes.HasPrefix(data, []byte("!\n")) { + return nil + } + header := data + if len(header) > 1024 { + header = header[:1024] + } + const marker = "\nbuild id \"" + start := bytes.Index(header, []byte(marker)) + if start < 0 { + return nil + } + start += len(marker) + end := bytes.IndexByte(header[start:], '"') + if end < 0 { + return errors.New("unterminated Go archive build ID") + } + end += start + buildID := string(header[start:end]) + if buildID == "" { + return nil + } + parts := strings.Split(buildID, "/") + if len(parts) != 2 || len(parts[0]) != len(parts[1]) { + return fmt.Errorf("unexpected Go archive build ID %q", buildID) + } + stableBuildID := parts[1] + "/" + parts[1] + for offset := 0; ; { + index := bytes.Index(data[offset:], []byte(buildID)) + if index < 0 { + break + } + index += offset + copy(data[index:index+len(buildID)], stableBuildID) + offset = index + len(buildID) + } + return nil +} + // reusableSharedSyntheticTestmainHelperBundle returns the declared shared // bundle only when it covers the requested roots and does not overlap archives // supplied by the current Bazel target. Overlap falls back to the existing @@ -2358,7 +2405,7 @@ func compileSyntheticTestmainSourcePackage(goenv *env, pack, workDir, moduleDir, stem := fmt.Sprintf("%x", sha256.Sum256([]byte(meta.ImportPath)))[:16] outLinkobjPath := filepath.Join(archiveDir, stem+".a") outInterfacePath := filepath.Join(archiveDir, stem+".iface.a") - trimPath, err := createTrimPath() + trimPath, err := syntheticSourceTrimPath(meta.Dir, meta.ImportPath) if err != nil { return "", "", fmt.Errorf("create trimpath for %s: %w", pkg, err) } @@ -2401,7 +2448,7 @@ func compileSyntheticTestmainSourcePackage(goenv *env, pack, workDir, moduleDir, } for i, sSrc := range filteredSrcs.sSrcs { obj := filepath.Join(packageWorkDir, fmt.Sprintf("s%d.o", i)) - if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, obj); err != nil { + if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, trimPath, obj); err != nil { return "", "", fmt.Errorf("assemble synthetic helper %s: %w", pkg, err) } objFiles = append(objFiles, obj) @@ -2429,6 +2476,21 @@ func compileSyntheticTestmainSourcePackage(goenv *env, pack, workDir, moduleDir, return result.compilePath, result.linkPath, nil } +// syntheticSourceTrimPath gives module sources a stable logical prefix. The +// physical GOMODCACHE belongs to the action sandbox and must not reach archives +// that Bazel may publish to a shared cache. +func syntheticSourceTrimPath(sourceDir, importPath string) (string, error) { + trimPath, err := createTrimPath() + if err != nil { + return "", err + } + sourceDir = strings.TrimSpace(sourceDir) + if sourceDir == "" { + return trimPath, nil + } + return trimPath + ";" + abs(sourceDir) + "=>" + strings.TrimSpace(importPath), nil +} + func shouldSkipOrchestrionForImportPath(importPath string) bool { if strings.HasPrefix(importPath, "github.com/bazelbuild/rules_go/go/tools/") { return true diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/compilepkg_test.go b/third_party/rgo/v0_61_1/base/go/tools/builders/compilepkg_test.go index 320fb66e..91065d66 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/compilepkg_test.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/compilepkg_test.go @@ -140,6 +140,51 @@ func TestSharedSyntheticTestmainHelperBundleRoundTrip(t *testing.T) { } } +func TestSharedSyntheticTestmainHelperBundleNormalizesGoBuildIDs(t *testing.T) { + const contentID = "cccccccccccccccccccc" + archive := func(actionID string) []byte { + buildID := actionID + "/" + contentID + return []byte("!\narchive header\ngo object test\nbuild id \"" + buildID + "\"\npayload build id \"" + buildID + "\"\n") + } + + sourceDir := t.TempDir() + firstSource := filepath.Join(sourceDir, "first.a") + secondSource := filepath.Join(sourceDir, "second.a") + if err := os.WriteFile(firstSource, archive("aaaaaaaaaaaaaaaaaaaa"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(secondSource, archive("bbbbbbbbbbbbbbbbbbbb"), 0o644); err != nil { + t.Fatal(err) + } + + bundleDir := filepath.Join(t.TempDir(), "shared-helpers") + compiled := map[string]compiledModuleArchive{ + "example.com/first": {compilePath: firstSource, linkPath: firstSource}, + "example.com/second": {compilePath: secondSource, linkPath: secondSource}, + } + if err := writeSharedSyntheticTestmainHelperBundle(bundleDir, compiled, orchestrionModeTestOptimization); err != nil { + t.Fatal(err) + } + + manifest, err := readSyntheticTestmainHelperManifest(bundleDir) + if err != nil { + t.Fatal(err) + } + firstPath := manifest.Packages["example.com/first"].CompilePath + secondPath := manifest.Packages["example.com/second"].CompilePath + if firstPath != secondPath { + t.Fatalf("equivalent archives published at different paths: %q != %q", firstPath, secondPath) + } + published, err := os.ReadFile(filepath.Join(bundleDir, firstPath)) + if err != nil { + t.Fatal(err) + } + wantBuildID := contentID + "/" + contentID + if got := strings.Count(string(published), wantBuildID); got != 2 { + t.Fatalf("normalized build ID occurs %d times, want 2", got) + } +} + func TestSharedSyntheticTestmainHelperBundleFallsBackOnTargetOverlap(t *testing.T) { rootPackage := syntheticTestmainRootPackagesTestOptimization[0].packagePath sourceDir := t.TempDir() @@ -539,6 +584,39 @@ func TestSyntheticTestmainHelperModuleCacheRootIsStable(t *testing.T) { } } +func TestSyntheticModuleGoFlagsRequireTrimpath(t *testing.T) { + tests := []struct { + name string + flags string + want string + }{ + {name: "empty", want: "-mod=mod -trimpath"}, + {name: "preserve module mode", flags: "-mod=readonly", want: "-mod=readonly -trimpath"}, + {name: "preserve trimpath", flags: "-trimpath -mod=vendor", want: "-trimpath -mod=vendor"}, + {name: "override disabled trimpath", flags: "-trimpath=false", want: "-mod=mod -trimpath=false -trimpath"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + env := ensureSyntheticModuleGoFlags([]string{"GOFLAGS=" + test.flags}) + if got := getEnv(env, "GOFLAGS"); got != test.want { + t.Fatalf("GOFLAGS = %q, want %q", got, test.want) + } + }) + } +} + +func TestSyntheticSourceTrimPathReplacesModuleCacheRoot(t *testing.T) { + sourceDir := filepath.Join(t.TempDir(), "pkg", "mod", "example.com", "helper@v1.0.0") + got, err := syntheticSourceTrimPath(sourceDir, "example.com/helper") + if err != nil { + t.Fatalf("syntheticSourceTrimPath error: %v", err) + } + want := abs(sourceDir) + "=>example.com/helper" + if !strings.Contains(got, want) { + t.Fatalf("syntheticSourceTrimPath = %q, want mapping %q", got, want) + } +} + // TestWithSyntheticTestmainModuleCacheEnvRestoresEnvironment verifies that the // synthetic helper wrapper points subprocess cache env vars at the helper // cache root during execution and restores the caller's environment after. diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/importcfg.go b/third_party/rgo/v0_61_1/base/go/tools/builders/importcfg.go index b6298e6c..7c19d36a 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/importcfg.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/importcfg.go @@ -787,7 +787,7 @@ func resolveModuleExportsForPackagesWithRoot(goenv *env, packages []string, orch if err != nil { return nil, fmt.Errorf("prepare module resolution env: %w", err) } - cmd.Env = ensureGoFlagsModMode(cmd.Env) + cmd.Env = ensureSyntheticModuleGoFlags(cmd.Env) if getEnv(cmd.Env, "HOME") == "" { homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") if err := os.MkdirAll(homePath, 0o755); err != nil { diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/module_proxy.go b/third_party/rgo/v0_61_1/base/go/tools/builders/module_proxy.go index b44c0f3b..91b0ba4b 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/module_proxy.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/module_proxy.go @@ -83,15 +83,22 @@ func normalizeGoModuleResolutionEnv(env []string) ([]string, error) { return env, nil } -func ensureGoFlagsModMode(env []string) []string { +// ensureSyntheticModuleGoFlags keeps helper module builds offline-compatible +// and removes their temporary module-cache roots from compiled archives. +func ensureSyntheticModuleGoFlags(env []string) []string { goFlags := strings.TrimSpace(getEnv(env, "GOFLAGS")) - if strings.Contains(goFlags, "-mod=") { - return env + if !strings.Contains(goFlags, "-mod=") { + goFlags = strings.TrimSpace("-mod=mod " + goFlags) } - if goFlags == "" { - goFlags = "-mod=mod" - } else { - goFlags = "-mod=mod " + goFlags + trimPathEnabled := false + for _, flag := range strings.Fields(goFlags) { + if flag == "-trimpath" || flag == "-trimpath=true" { + trimPathEnabled = true + break + } + } + if !trimPathEnabled { + goFlags += " -trimpath" } return setEnv(env, "GOFLAGS", goFlags) } diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_skip_test.go b/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_skip_test.go index 32c0cf4b..00d37c1f 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_skip_test.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/orchestrion_skip_test.go @@ -294,6 +294,14 @@ func TestSyntheticHelperManifestKeepsExternalAbsoluteDependencies(t *testing.T) func TestSyntheticTestmainManifestPublishesExternalArchives(t *testing.T) { execroot := t.TempDir() + previousDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(execroot); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(previousDir) }) externalArchive := filepath.Join(t.TempDir(), "helper.a") if err := os.WriteFile(externalArchive, []byte("helper archive"), 0o644); err != nil { t.Fatal(err) @@ -336,7 +344,8 @@ func TestSyntheticTestmainManifestPublishesExternalArchives(t *testing.T) { t.Fatalf("published helper archive count = %d, want 1", len(entries)) } publishedArchive := filepath.Join(helperOutputDir, entries[0].Name()) - if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedArchive) { + publishedRelative := execrootRelativePath(publishedArchive) + if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedRelative) { t.Fatalf("manifest does not reference published helper %q:\n%s", publishedArchive, gotManifest) } publishedData, err := os.ReadFile(publishedArchive) diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib.go b/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib.go index 65244aa7..8ea63311 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/stdlib.go @@ -270,6 +270,11 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } } installArgs := goenv.goCmd("install", "-toolexec", toolexec) + if *orchestrion != "" { + // Helper module exports use Go's reproducible build mode. Compile the + // woven stdlib with the same flag so imported package fingerprints match. + installArgs = append(installArgs, "-trimpath") + } if len(build.Default.BuildTags) > 0 { installArgs = append(installArgs, "-tags", strings.Join(build.Default.BuildTags, ",")) } diff --git a/third_party/rgo/v0_62_0/base.CHANGED_FILES.md b/third_party/rgo/v0_62_0/base.CHANGED_FILES.md index 56da2091..18b46e35 100644 --- a/third_party/rgo/v0_62_0/base.CHANGED_FILES.md +++ b/third_party/rgo/v0_62_0/base.CHANGED_FILES.md @@ -12,8 +12,8 @@ This file is generated. Do not edit by hand. ## Summary -- Total changed paths: `55` -- Modified files: `31` +- Total changed paths: `56` +- Modified files: `32` - Added files: `24` - Removed files: `0` @@ -37,6 +37,7 @@ This file is generated. Do not edit by hand. - `go/private/rules/transition.bzl` - `go/tools/builders/BUILD.bazel` - `go/tools/builders/ar.go` +- `go/tools/builders/asm.go` - `go/tools/builders/builder.go` - `go/tools/builders/cc.go` - `go/tools/builders/compilepkg.go` diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/asm.go b/third_party/rgo/v0_62_0/base/go/tools/builders/asm.go index 6704be70..5e7d3404 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/asm.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/asm.go @@ -111,7 +111,7 @@ func buildSymabisFile(goenv *env, packagePath string, sFiles, hFiles []fileInfo, return symabisName, err } -func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, outPath string) error { +func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, trimPath, outPath string) error { args := goenv.goTool("asm") args = append(args, asmFlags...) // The package path has to be specified as of Go 1.19 or the resulting @@ -121,10 +121,13 @@ func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, outPath args = append(args, "-p", packagePath) } args = append(args, ASM_DEFINES...) - args = append(args, "-trimpath", ".") + if trimPath == "" { + trimPath = abs(".") + } + args = append(args, "-trimpath", trimPath) args = append(args, "-o", outPath) args = append(args, "--", srcPath) - absArgs(args, []string{"-I", "-o", "-trimpath"}) + absArgs(args, []string{"-I", "-o"}) return goenv.runCommand(args) } diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/compilepkg.go b/third_party/rgo/v0_62_0/base/go/tools/builders/compilepkg.go index 8004cd0f..5d3ae228 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/compilepkg.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/compilepkg.go @@ -210,7 +210,7 @@ func publishSyntheticTestmainPackagefiles(data []byte, outputDir string) ([]byte } published[publishedPath] = true } - lines[index] = "packagefile " + packagePath + "=" + publishedPath + lines[index] = "packagefile " + packagePath + "=" + execrootRelativePath(publishedPath) } return []byte(strings.Join(lines, "\n")), nil } @@ -768,7 +768,7 @@ func compileArchive( } for i, sSrc := range srcs.sSrcs { obj := filepath.Join(workDir, fmt.Sprintf("s%d.o", i)) - if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, obj); err != nil { + if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, "", obj); err != nil { return err } objFiles = append(objFiles, obj) @@ -1351,7 +1351,7 @@ func modulePackageCommandEnv(goenv *env, exportRoot string) ([]string, error) { env = setEnv(env, "GIT_CONFIG_GLOBAL", os.DevNull) env = setEnv(env, "GIT_CONFIG_NOSYSTEM", "1") env = setEnv(env, "GIT_TERMINAL_PROMPT", "0") - env = ensureGoFlagsModMode(env) + env = ensureSyntheticModuleGoFlags(env) if getEnv(env, "HOME") == "" { homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -1921,13 +1921,17 @@ func writeSharedSyntheticTestmainHelperBundle(outputDir string, compiled map[str if published := publishedPaths[source]; published != "" { return published, nil } - digest, err := fullDigestFile(source) + data, err := os.ReadFile(source) if err != nil { - return "", fmt.Errorf("digest shared synthetic helper archive %s: %w", source, err) + return "", fmt.Errorf("read shared synthetic helper archive %s: %w", source, err) } + if err := normalizePublishedGoArchiveBuildID(data); err != nil { + return "", fmt.Errorf("normalize shared synthetic helper archive %s: %w", source, err) + } + digest := fmt.Sprintf("%x", sha256.Sum256(data)) destination := filepath.Join(outputDir, "archives", digest+".a") if _, err := os.Stat(destination); errors.Is(err, os.ErrNotExist) { - if err := copyArchiveFile(source, destination); err != nil { + if err := os.WriteFile(destination, data, 0o644); err != nil { return "", fmt.Errorf("publish shared synthetic helper archive %s: %w", source, err) } } else if err != nil { @@ -1968,6 +1972,49 @@ func writeSharedSyntheticTestmainHelperBundle(outputDir string, compiled map[str return writeJSONAtomically(filepath.Join(outputDir, cacheManifestFileName), manifest) } +// normalizePublishedGoArchiveBuildID removes execroot-specific action IDs from +// a copied helper archive. The Go cache keeps the original archive and build ID; +// only the declared Bazel output receives the stable content/content form. +func normalizePublishedGoArchiveBuildID(data []byte) error { + if !bytes.HasPrefix(data, []byte("!\n")) { + return nil + } + header := data + if len(header) > 1024 { + header = header[:1024] + } + const marker = "\nbuild id \"" + start := bytes.Index(header, []byte(marker)) + if start < 0 { + return nil + } + start += len(marker) + end := bytes.IndexByte(header[start:], '"') + if end < 0 { + return errors.New("unterminated Go archive build ID") + } + end += start + buildID := string(header[start:end]) + if buildID == "" { + return nil + } + parts := strings.Split(buildID, "/") + if len(parts) != 2 || len(parts[0]) != len(parts[1]) { + return fmt.Errorf("unexpected Go archive build ID %q", buildID) + } + stableBuildID := parts[1] + "/" + parts[1] + for offset := 0; ; { + index := bytes.Index(data[offset:], []byte(buildID)) + if index < 0 { + break + } + index += offset + copy(data[index:index+len(buildID)], stableBuildID) + offset = index + len(buildID) + } + return nil +} + // reusableSharedSyntheticTestmainHelperBundle returns the declared shared // bundle only when it covers the requested roots and does not overlap archives // supplied by the current Bazel target. Overlap falls back to the existing @@ -2356,7 +2403,7 @@ func compileSyntheticTestmainSourcePackage(goenv *env, pack, workDir, moduleDir, stem := fmt.Sprintf("%x", sha256.Sum256([]byte(meta.ImportPath)))[:16] outLinkobjPath := filepath.Join(archiveDir, stem+".a") outInterfacePath := filepath.Join(archiveDir, stem+".iface.a") - trimPath, err := createTrimPath() + trimPath, err := syntheticSourceTrimPath(meta.Dir, meta.ImportPath) if err != nil { return "", "", fmt.Errorf("create trimpath for %s: %w", pkg, err) } @@ -2399,7 +2446,7 @@ func compileSyntheticTestmainSourcePackage(goenv *env, pack, workDir, moduleDir, } for i, sSrc := range filteredSrcs.sSrcs { obj := filepath.Join(packageWorkDir, fmt.Sprintf("s%d.o", i)) - if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, obj); err != nil { + if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, trimPath, obj); err != nil { return "", "", fmt.Errorf("assemble synthetic helper %s: %w", pkg, err) } objFiles = append(objFiles, obj) @@ -2427,6 +2474,21 @@ func compileSyntheticTestmainSourcePackage(goenv *env, pack, workDir, moduleDir, return result.compilePath, result.linkPath, nil } +// syntheticSourceTrimPath gives module sources a stable logical prefix. The +// physical GOMODCACHE belongs to the action sandbox and must not reach archives +// that Bazel may publish to a shared cache. +func syntheticSourceTrimPath(sourceDir, importPath string) (string, error) { + trimPath, err := createTrimPath() + if err != nil { + return "", err + } + sourceDir = strings.TrimSpace(sourceDir) + if sourceDir == "" { + return trimPath, nil + } + return trimPath + ";" + abs(sourceDir) + "=>" + strings.TrimSpace(importPath), nil +} + func shouldSkipOrchestrionForImportPath(importPath string) bool { if strings.HasPrefix(importPath, "github.com/bazelbuild/rules_go/go/tools/") { return true diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/compilepkg_test.go b/third_party/rgo/v0_62_0/base/go/tools/builders/compilepkg_test.go index 320fb66e..91065d66 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/compilepkg_test.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/compilepkg_test.go @@ -140,6 +140,51 @@ func TestSharedSyntheticTestmainHelperBundleRoundTrip(t *testing.T) { } } +func TestSharedSyntheticTestmainHelperBundleNormalizesGoBuildIDs(t *testing.T) { + const contentID = "cccccccccccccccccccc" + archive := func(actionID string) []byte { + buildID := actionID + "/" + contentID + return []byte("!\narchive header\ngo object test\nbuild id \"" + buildID + "\"\npayload build id \"" + buildID + "\"\n") + } + + sourceDir := t.TempDir() + firstSource := filepath.Join(sourceDir, "first.a") + secondSource := filepath.Join(sourceDir, "second.a") + if err := os.WriteFile(firstSource, archive("aaaaaaaaaaaaaaaaaaaa"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(secondSource, archive("bbbbbbbbbbbbbbbbbbbb"), 0o644); err != nil { + t.Fatal(err) + } + + bundleDir := filepath.Join(t.TempDir(), "shared-helpers") + compiled := map[string]compiledModuleArchive{ + "example.com/first": {compilePath: firstSource, linkPath: firstSource}, + "example.com/second": {compilePath: secondSource, linkPath: secondSource}, + } + if err := writeSharedSyntheticTestmainHelperBundle(bundleDir, compiled, orchestrionModeTestOptimization); err != nil { + t.Fatal(err) + } + + manifest, err := readSyntheticTestmainHelperManifest(bundleDir) + if err != nil { + t.Fatal(err) + } + firstPath := manifest.Packages["example.com/first"].CompilePath + secondPath := manifest.Packages["example.com/second"].CompilePath + if firstPath != secondPath { + t.Fatalf("equivalent archives published at different paths: %q != %q", firstPath, secondPath) + } + published, err := os.ReadFile(filepath.Join(bundleDir, firstPath)) + if err != nil { + t.Fatal(err) + } + wantBuildID := contentID + "/" + contentID + if got := strings.Count(string(published), wantBuildID); got != 2 { + t.Fatalf("normalized build ID occurs %d times, want 2", got) + } +} + func TestSharedSyntheticTestmainHelperBundleFallsBackOnTargetOverlap(t *testing.T) { rootPackage := syntheticTestmainRootPackagesTestOptimization[0].packagePath sourceDir := t.TempDir() @@ -539,6 +584,39 @@ func TestSyntheticTestmainHelperModuleCacheRootIsStable(t *testing.T) { } } +func TestSyntheticModuleGoFlagsRequireTrimpath(t *testing.T) { + tests := []struct { + name string + flags string + want string + }{ + {name: "empty", want: "-mod=mod -trimpath"}, + {name: "preserve module mode", flags: "-mod=readonly", want: "-mod=readonly -trimpath"}, + {name: "preserve trimpath", flags: "-trimpath -mod=vendor", want: "-trimpath -mod=vendor"}, + {name: "override disabled trimpath", flags: "-trimpath=false", want: "-mod=mod -trimpath=false -trimpath"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + env := ensureSyntheticModuleGoFlags([]string{"GOFLAGS=" + test.flags}) + if got := getEnv(env, "GOFLAGS"); got != test.want { + t.Fatalf("GOFLAGS = %q, want %q", got, test.want) + } + }) + } +} + +func TestSyntheticSourceTrimPathReplacesModuleCacheRoot(t *testing.T) { + sourceDir := filepath.Join(t.TempDir(), "pkg", "mod", "example.com", "helper@v1.0.0") + got, err := syntheticSourceTrimPath(sourceDir, "example.com/helper") + if err != nil { + t.Fatalf("syntheticSourceTrimPath error: %v", err) + } + want := abs(sourceDir) + "=>example.com/helper" + if !strings.Contains(got, want) { + t.Fatalf("syntheticSourceTrimPath = %q, want mapping %q", got, want) + } +} + // TestWithSyntheticTestmainModuleCacheEnvRestoresEnvironment verifies that the // synthetic helper wrapper points subprocess cache env vars at the helper // cache root during execution and restores the caller's environment after. diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/importcfg.go b/third_party/rgo/v0_62_0/base/go/tools/builders/importcfg.go index b6298e6c..7c19d36a 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/importcfg.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/importcfg.go @@ -787,7 +787,7 @@ func resolveModuleExportsForPackagesWithRoot(goenv *env, packages []string, orch if err != nil { return nil, fmt.Errorf("prepare module resolution env: %w", err) } - cmd.Env = ensureGoFlagsModMode(cmd.Env) + cmd.Env = ensureSyntheticModuleGoFlags(cmd.Env) if getEnv(cmd.Env, "HOME") == "" { homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") if err := os.MkdirAll(homePath, 0o755); err != nil { diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/module_proxy.go b/third_party/rgo/v0_62_0/base/go/tools/builders/module_proxy.go index b44c0f3b..91b0ba4b 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/module_proxy.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/module_proxy.go @@ -83,15 +83,22 @@ func normalizeGoModuleResolutionEnv(env []string) ([]string, error) { return env, nil } -func ensureGoFlagsModMode(env []string) []string { +// ensureSyntheticModuleGoFlags keeps helper module builds offline-compatible +// and removes their temporary module-cache roots from compiled archives. +func ensureSyntheticModuleGoFlags(env []string) []string { goFlags := strings.TrimSpace(getEnv(env, "GOFLAGS")) - if strings.Contains(goFlags, "-mod=") { - return env + if !strings.Contains(goFlags, "-mod=") { + goFlags = strings.TrimSpace("-mod=mod " + goFlags) } - if goFlags == "" { - goFlags = "-mod=mod" - } else { - goFlags = "-mod=mod " + goFlags + trimPathEnabled := false + for _, flag := range strings.Fields(goFlags) { + if flag == "-trimpath" || flag == "-trimpath=true" { + trimPathEnabled = true + break + } + } + if !trimPathEnabled { + goFlags += " -trimpath" } return setEnv(env, "GOFLAGS", goFlags) } diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_skip_test.go b/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_skip_test.go index 32c0cf4b..00d37c1f 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_skip_test.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/orchestrion_skip_test.go @@ -294,6 +294,14 @@ func TestSyntheticHelperManifestKeepsExternalAbsoluteDependencies(t *testing.T) func TestSyntheticTestmainManifestPublishesExternalArchives(t *testing.T) { execroot := t.TempDir() + previousDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(execroot); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(previousDir) }) externalArchive := filepath.Join(t.TempDir(), "helper.a") if err := os.WriteFile(externalArchive, []byte("helper archive"), 0o644); err != nil { t.Fatal(err) @@ -336,7 +344,8 @@ func TestSyntheticTestmainManifestPublishesExternalArchives(t *testing.T) { t.Fatalf("published helper archive count = %d, want 1", len(entries)) } publishedArchive := filepath.Join(helperOutputDir, entries[0].Name()) - if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedArchive) { + publishedRelative := execrootRelativePath(publishedArchive) + if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedRelative) { t.Fatalf("manifest does not reference published helper %q:\n%s", publishedArchive, gotManifest) } publishedData, err := os.ReadFile(publishedArchive) diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib.go b/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib.go index 65244aa7..8ea63311 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/stdlib.go @@ -270,6 +270,11 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } } installArgs := goenv.goCmd("install", "-toolexec", toolexec) + if *orchestrion != "" { + // Helper module exports use Go's reproducible build mode. Compile the + // woven stdlib with the same flag so imported package fingerprints match. + installArgs = append(installArgs, "-trimpath") + } if len(build.Default.BuildTags) > 0 { installArgs = append(installArgs, "-tags", strings.Join(build.Default.BuildTags, ",")) } diff --git a/third_party/rgo/v0_63_0/base.CHANGED_FILES.md b/third_party/rgo/v0_63_0/base.CHANGED_FILES.md index e2874a2b..9c2bff51 100644 --- a/third_party/rgo/v0_63_0/base.CHANGED_FILES.md +++ b/third_party/rgo/v0_63_0/base.CHANGED_FILES.md @@ -12,8 +12,8 @@ This file is generated. Do not edit by hand. ## Summary -- Total changed paths: `54` -- Modified files: `30` +- Total changed paths: `55` +- Modified files: `31` - Added files: `24` - Removed files: `0` @@ -36,6 +36,7 @@ This file is generated. Do not edit by hand. - `go/private/rules/transition.bzl` - `go/tools/builders/BUILD.bazel` - `go/tools/builders/ar.go` +- `go/tools/builders/asm.go` - `go/tools/builders/builder.go` - `go/tools/builders/cc.go` - `go/tools/builders/compilepkg.go` diff --git a/third_party/rgo/v0_63_0/base/go/tools/builders/asm.go b/third_party/rgo/v0_63_0/base/go/tools/builders/asm.go index c1b79979..3c8cf894 100644 --- a/third_party/rgo/v0_63_0/base/go/tools/builders/asm.go +++ b/third_party/rgo/v0_63_0/base/go/tools/builders/asm.go @@ -87,16 +87,19 @@ func buildSymabisFile(goenv *env, packagePath string, sFiles, hFiles []fileInfo, return symabisName, err } -func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, outPath string) error { +func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, trimPath, outPath string) error { args := goenv.goTool("asm") args = append(args, asmFlags...) if packagePath != "" { args = append(args, "-p", packagePath) } args = append(args, ASM_DEFINES...) - args = append(args, "-trimpath", ".") + if trimPath == "" { + trimPath = abs(".") + } + args = append(args, "-trimpath", trimPath) args = append(args, "-o", outPath) args = append(args, "--", srcPath) - absArgs(args, []string{"-I", "-o", "-trimpath"}) + absArgs(args, []string{"-I", "-o"}) return goenv.runCommand(args) } diff --git a/third_party/rgo/v0_63_0/base/go/tools/builders/compilepkg.go b/third_party/rgo/v0_63_0/base/go/tools/builders/compilepkg.go index 3eef445b..b64ae96d 100644 --- a/third_party/rgo/v0_63_0/base/go/tools/builders/compilepkg.go +++ b/third_party/rgo/v0_63_0/base/go/tools/builders/compilepkg.go @@ -210,7 +210,7 @@ func publishSyntheticTestmainPackagefiles(data []byte, outputDir string) ([]byte } published[publishedPath] = true } - lines[index] = "packagefile " + packagePath + "=" + publishedPath + lines[index] = "packagefile " + packagePath + "=" + execrootRelativePath(publishedPath) } return []byte(strings.Join(lines, "\n")), nil } @@ -768,7 +768,7 @@ func compileArchive( } for i, sSrc := range srcs.sSrcs { obj := filepath.Join(workDir, fmt.Sprintf("s%d.o", i)) - if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, obj); err != nil { + if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, "", obj); err != nil { return err } objFiles = append(objFiles, obj) @@ -1351,7 +1351,7 @@ func modulePackageCommandEnv(goenv *env, exportRoot string) ([]string, error) { env = setEnv(env, "GIT_CONFIG_GLOBAL", os.DevNull) env = setEnv(env, "GIT_CONFIG_NOSYSTEM", "1") env = setEnv(env, "GIT_TERMINAL_PROMPT", "0") - env = ensureGoFlagsModMode(env) + env = ensureSyntheticModuleGoFlags(env) if getEnv(env, "HOME") == "" { homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -1921,13 +1921,17 @@ func writeSharedSyntheticTestmainHelperBundle(outputDir string, compiled map[str if published := publishedPaths[source]; published != "" { return published, nil } - digest, err := fullDigestFile(source) + data, err := os.ReadFile(source) if err != nil { - return "", fmt.Errorf("digest shared synthetic helper archive %s: %w", source, err) + return "", fmt.Errorf("read shared synthetic helper archive %s: %w", source, err) } + if err := normalizePublishedGoArchiveBuildID(data); err != nil { + return "", fmt.Errorf("normalize shared synthetic helper archive %s: %w", source, err) + } + digest := fmt.Sprintf("%x", sha256.Sum256(data)) destination := filepath.Join(outputDir, "archives", digest+".a") if _, err := os.Stat(destination); errors.Is(err, os.ErrNotExist) { - if err := copyArchiveFile(source, destination); err != nil { + if err := os.WriteFile(destination, data, 0o644); err != nil { return "", fmt.Errorf("publish shared synthetic helper archive %s: %w", source, err) } } else if err != nil { @@ -1968,6 +1972,49 @@ func writeSharedSyntheticTestmainHelperBundle(outputDir string, compiled map[str return writeJSONAtomically(filepath.Join(outputDir, cacheManifestFileName), manifest) } +// normalizePublishedGoArchiveBuildID removes execroot-specific action IDs from +// a copied helper archive. The Go cache keeps the original archive and build ID; +// only the declared Bazel output receives the stable content/content form. +func normalizePublishedGoArchiveBuildID(data []byte) error { + if !bytes.HasPrefix(data, []byte("!\n")) { + return nil + } + header := data + if len(header) > 1024 { + header = header[:1024] + } + const marker = "\nbuild id \"" + start := bytes.Index(header, []byte(marker)) + if start < 0 { + return nil + } + start += len(marker) + end := bytes.IndexByte(header[start:], '"') + if end < 0 { + return errors.New("unterminated Go archive build ID") + } + end += start + buildID := string(header[start:end]) + if buildID == "" { + return nil + } + parts := strings.Split(buildID, "/") + if len(parts) != 2 || len(parts[0]) != len(parts[1]) { + return fmt.Errorf("unexpected Go archive build ID %q", buildID) + } + stableBuildID := parts[1] + "/" + parts[1] + for offset := 0; ; { + index := bytes.Index(data[offset:], []byte(buildID)) + if index < 0 { + break + } + index += offset + copy(data[index:index+len(buildID)], stableBuildID) + offset = index + len(buildID) + } + return nil +} + // reusableSharedSyntheticTestmainHelperBundle returns the declared shared // bundle only when it covers the requested roots and does not overlap archives // supplied by the current Bazel target. Overlap falls back to the existing @@ -2356,7 +2403,7 @@ func compileSyntheticTestmainSourcePackage(goenv *env, pack, workDir, moduleDir, stem := fmt.Sprintf("%x", sha256.Sum256([]byte(meta.ImportPath)))[:16] outLinkobjPath := filepath.Join(archiveDir, stem+".a") outInterfacePath := filepath.Join(archiveDir, stem+".iface.a") - trimPath, err := createTrimPath() + trimPath, err := syntheticSourceTrimPath(meta.Dir, meta.ImportPath) if err != nil { return "", "", fmt.Errorf("create trimpath for %s: %w", pkg, err) } @@ -2399,7 +2446,7 @@ func compileSyntheticTestmainSourcePackage(goenv *env, pack, workDir, moduleDir, } for i, sSrc := range filteredSrcs.sSrcs { obj := filepath.Join(packageWorkDir, fmt.Sprintf("s%d.o", i)) - if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, obj); err != nil { + if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, trimPath, obj); err != nil { return "", "", fmt.Errorf("assemble synthetic helper %s: %w", pkg, err) } objFiles = append(objFiles, obj) @@ -2427,6 +2474,21 @@ func compileSyntheticTestmainSourcePackage(goenv *env, pack, workDir, moduleDir, return result.compilePath, result.linkPath, nil } +// syntheticSourceTrimPath gives module sources a stable logical prefix. The +// physical GOMODCACHE belongs to the action sandbox and must not reach archives +// that Bazel may publish to a shared cache. +func syntheticSourceTrimPath(sourceDir, importPath string) (string, error) { + trimPath, err := createTrimPath() + if err != nil { + return "", err + } + sourceDir = strings.TrimSpace(sourceDir) + if sourceDir == "" { + return trimPath, nil + } + return trimPath + ";" + abs(sourceDir) + "=>" + strings.TrimSpace(importPath), nil +} + func shouldSkipOrchestrionForImportPath(importPath string) bool { if strings.HasPrefix(importPath, "github.com/bazelbuild/rules_go/go/tools/") { return true diff --git a/third_party/rgo/v0_63_0/base/go/tools/builders/compilepkg_test.go b/third_party/rgo/v0_63_0/base/go/tools/builders/compilepkg_test.go index 320fb66e..91065d66 100644 --- a/third_party/rgo/v0_63_0/base/go/tools/builders/compilepkg_test.go +++ b/third_party/rgo/v0_63_0/base/go/tools/builders/compilepkg_test.go @@ -140,6 +140,51 @@ func TestSharedSyntheticTestmainHelperBundleRoundTrip(t *testing.T) { } } +func TestSharedSyntheticTestmainHelperBundleNormalizesGoBuildIDs(t *testing.T) { + const contentID = "cccccccccccccccccccc" + archive := func(actionID string) []byte { + buildID := actionID + "/" + contentID + return []byte("!\narchive header\ngo object test\nbuild id \"" + buildID + "\"\npayload build id \"" + buildID + "\"\n") + } + + sourceDir := t.TempDir() + firstSource := filepath.Join(sourceDir, "first.a") + secondSource := filepath.Join(sourceDir, "second.a") + if err := os.WriteFile(firstSource, archive("aaaaaaaaaaaaaaaaaaaa"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(secondSource, archive("bbbbbbbbbbbbbbbbbbbb"), 0o644); err != nil { + t.Fatal(err) + } + + bundleDir := filepath.Join(t.TempDir(), "shared-helpers") + compiled := map[string]compiledModuleArchive{ + "example.com/first": {compilePath: firstSource, linkPath: firstSource}, + "example.com/second": {compilePath: secondSource, linkPath: secondSource}, + } + if err := writeSharedSyntheticTestmainHelperBundle(bundleDir, compiled, orchestrionModeTestOptimization); err != nil { + t.Fatal(err) + } + + manifest, err := readSyntheticTestmainHelperManifest(bundleDir) + if err != nil { + t.Fatal(err) + } + firstPath := manifest.Packages["example.com/first"].CompilePath + secondPath := manifest.Packages["example.com/second"].CompilePath + if firstPath != secondPath { + t.Fatalf("equivalent archives published at different paths: %q != %q", firstPath, secondPath) + } + published, err := os.ReadFile(filepath.Join(bundleDir, firstPath)) + if err != nil { + t.Fatal(err) + } + wantBuildID := contentID + "/" + contentID + if got := strings.Count(string(published), wantBuildID); got != 2 { + t.Fatalf("normalized build ID occurs %d times, want 2", got) + } +} + func TestSharedSyntheticTestmainHelperBundleFallsBackOnTargetOverlap(t *testing.T) { rootPackage := syntheticTestmainRootPackagesTestOptimization[0].packagePath sourceDir := t.TempDir() @@ -539,6 +584,39 @@ func TestSyntheticTestmainHelperModuleCacheRootIsStable(t *testing.T) { } } +func TestSyntheticModuleGoFlagsRequireTrimpath(t *testing.T) { + tests := []struct { + name string + flags string + want string + }{ + {name: "empty", want: "-mod=mod -trimpath"}, + {name: "preserve module mode", flags: "-mod=readonly", want: "-mod=readonly -trimpath"}, + {name: "preserve trimpath", flags: "-trimpath -mod=vendor", want: "-trimpath -mod=vendor"}, + {name: "override disabled trimpath", flags: "-trimpath=false", want: "-mod=mod -trimpath=false -trimpath"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + env := ensureSyntheticModuleGoFlags([]string{"GOFLAGS=" + test.flags}) + if got := getEnv(env, "GOFLAGS"); got != test.want { + t.Fatalf("GOFLAGS = %q, want %q", got, test.want) + } + }) + } +} + +func TestSyntheticSourceTrimPathReplacesModuleCacheRoot(t *testing.T) { + sourceDir := filepath.Join(t.TempDir(), "pkg", "mod", "example.com", "helper@v1.0.0") + got, err := syntheticSourceTrimPath(sourceDir, "example.com/helper") + if err != nil { + t.Fatalf("syntheticSourceTrimPath error: %v", err) + } + want := abs(sourceDir) + "=>example.com/helper" + if !strings.Contains(got, want) { + t.Fatalf("syntheticSourceTrimPath = %q, want mapping %q", got, want) + } +} + // TestWithSyntheticTestmainModuleCacheEnvRestoresEnvironment verifies that the // synthetic helper wrapper points subprocess cache env vars at the helper // cache root during execution and restores the caller's environment after. diff --git a/third_party/rgo/v0_63_0/base/go/tools/builders/importcfg.go b/third_party/rgo/v0_63_0/base/go/tools/builders/importcfg.go index 4b0fa3fe..2fc994ce 100644 --- a/third_party/rgo/v0_63_0/base/go/tools/builders/importcfg.go +++ b/third_party/rgo/v0_63_0/base/go/tools/builders/importcfg.go @@ -789,7 +789,7 @@ func resolveModuleExportsForPackagesWithRoot(goenv *env, packages []string, orch if err != nil { return nil, fmt.Errorf("prepare module resolution env: %w", err) } - cmd.Env = ensureGoFlagsModMode(cmd.Env) + cmd.Env = ensureSyntheticModuleGoFlags(cmd.Env) if getEnv(cmd.Env, "HOME") == "" { homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") if err := os.MkdirAll(homePath, 0o755); err != nil { diff --git a/third_party/rgo/v0_63_0/base/go/tools/builders/module_proxy.go b/third_party/rgo/v0_63_0/base/go/tools/builders/module_proxy.go index b44c0f3b..91b0ba4b 100644 --- a/third_party/rgo/v0_63_0/base/go/tools/builders/module_proxy.go +++ b/third_party/rgo/v0_63_0/base/go/tools/builders/module_proxy.go @@ -83,15 +83,22 @@ func normalizeGoModuleResolutionEnv(env []string) ([]string, error) { return env, nil } -func ensureGoFlagsModMode(env []string) []string { +// ensureSyntheticModuleGoFlags keeps helper module builds offline-compatible +// and removes their temporary module-cache roots from compiled archives. +func ensureSyntheticModuleGoFlags(env []string) []string { goFlags := strings.TrimSpace(getEnv(env, "GOFLAGS")) - if strings.Contains(goFlags, "-mod=") { - return env + if !strings.Contains(goFlags, "-mod=") { + goFlags = strings.TrimSpace("-mod=mod " + goFlags) } - if goFlags == "" { - goFlags = "-mod=mod" - } else { - goFlags = "-mod=mod " + goFlags + trimPathEnabled := false + for _, flag := range strings.Fields(goFlags) { + if flag == "-trimpath" || flag == "-trimpath=true" { + trimPathEnabled = true + break + } + } + if !trimPathEnabled { + goFlags += " -trimpath" } return setEnv(env, "GOFLAGS", goFlags) } diff --git a/third_party/rgo/v0_63_0/base/go/tools/builders/orchestrion_skip_test.go b/third_party/rgo/v0_63_0/base/go/tools/builders/orchestrion_skip_test.go index 32c0cf4b..00d37c1f 100644 --- a/third_party/rgo/v0_63_0/base/go/tools/builders/orchestrion_skip_test.go +++ b/third_party/rgo/v0_63_0/base/go/tools/builders/orchestrion_skip_test.go @@ -294,6 +294,14 @@ func TestSyntheticHelperManifestKeepsExternalAbsoluteDependencies(t *testing.T) func TestSyntheticTestmainManifestPublishesExternalArchives(t *testing.T) { execroot := t.TempDir() + previousDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(execroot); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(previousDir) }) externalArchive := filepath.Join(t.TempDir(), "helper.a") if err := os.WriteFile(externalArchive, []byte("helper archive"), 0o644); err != nil { t.Fatal(err) @@ -336,7 +344,8 @@ func TestSyntheticTestmainManifestPublishesExternalArchives(t *testing.T) { t.Fatalf("published helper archive count = %d, want 1", len(entries)) } publishedArchive := filepath.Join(helperOutputDir, entries[0].Name()) - if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedArchive) { + publishedRelative := execrootRelativePath(publishedArchive) + if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedRelative) { t.Fatalf("manifest does not reference published helper %q:\n%s", publishedArchive, gotManifest) } publishedData, err := os.ReadFile(publishedArchive) diff --git a/third_party/rgo/v0_63_0/base/go/tools/builders/stdlib.go b/third_party/rgo/v0_63_0/base/go/tools/builders/stdlib.go index 65791a42..cad3e0b5 100644 --- a/third_party/rgo/v0_63_0/base/go/tools/builders/stdlib.go +++ b/third_party/rgo/v0_63_0/base/go/tools/builders/stdlib.go @@ -270,6 +270,11 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } } installArgs := goenv.goCmd("install", "-toolexec", toolexec) + if *orchestrion != "" { + // Helper module exports use Go's reproducible build mode. Compile the + // woven stdlib with the same flag so imported package fingerprints match. + installArgs = append(installArgs, "-trimpath") + } if len(build.Default.BuildTags) > 0 { installArgs = append(installArgs, "-tags", strings.Join(build.Default.BuildTags, ",")) } diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index 1d7bf4e4..b59117ae 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -4644,6 +4644,35 @@ index d2de6b9..ffbf778 100644 + } + return "." +} +diff --git a/go/tools/builders/asm.go b/go/tools/builders/asm.go +index 6704be7..5e7d340 100644 +--- a/go/tools/builders/asm.go ++++ b/go/tools/builders/asm.go +@@ -111,7 +111,7 @@ func buildSymabisFile(goenv *env, packagePath string, sFiles, hFiles []fileInfo, + return symabisName, err + } + +-func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, outPath string) error { ++func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, trimPath, outPath string) error { + args := goenv.goTool("asm") + args = append(args, asmFlags...) + // The package path has to be specified as of Go 1.19 or the resulting +@@ -121,10 +121,13 @@ func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, outPath + args = append(args, "-p", packagePath) + } + args = append(args, ASM_DEFINES...) +- args = append(args, "-trimpath", ".") ++ if trimPath == "" { ++ trimPath = abs(".") ++ } ++ args = append(args, "-trimpath", trimPath) + args = append(args, "-o", outPath) + args = append(args, "--", srcPath) +- absArgs(args, []string{"-I", "-o", "-trimpath"}) ++ absArgs(args, []string{"-I", "-o"}) + return goenv.runCommand(args) + } + diff --git a/go/tools/builders/builder.go b/go/tools/builders/builder.go index fdeda2d..76d69b6 100644 --- a/go/tools/builders/builder.go @@ -4795,7 +4824,7 @@ index 6f67914..5451c67 100644 transformArgs(normalized, cgoAbsEnvFlags, func(s string) string { if strings.HasPrefix(s, cgoAbsPlaceholder) { diff --git a/go/tools/builders/compilepkg.go b/go/tools/builders/compilepkg.go -index e2b0be5..cb5b10a 100644 +index e2b0be5..5356b08 100644 --- a/go/tools/builders/compilepkg.go +++ b/go/tools/builders/compilepkg.go @@ -17,17 +17,252 @@ @@ -4995,7 +5024,7 @@ index e2b0be5..cb5b10a 100644 + } + published[publishedPath] = true + } -+ lines[index] = "packagefile " + packagePath + "=" + publishedPath ++ lines[index] = "packagefile " + packagePath + "=" + execrootRelativePath(publishedPath) + } + return []byte(strings.Join(lines, "\n")), nil +} @@ -5247,7 +5276,16 @@ index e2b0be5..cb5b10a 100644 // Compile the .s files with Go's assembler, if this is not a cgo package. // Cgo is assembled by cc above. -@@ -489,100 +793,1895 @@ func compileArchive( +@@ -466,7 +770,7 @@ func compileArchive( + } + for i, sSrc := range srcs.sSrcs { + obj := filepath.Join(workDir, fmt.Sprintf("s%d.o", i)) +- if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, obj); err != nil { ++ if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, "", obj); err != nil { + return err + } + objFiles = append(objFiles, obj) +@@ -489,100 +793,1957 @@ func compileArchive( return nil } @@ -5879,7 +5917,7 @@ index e2b0be5..cb5b10a 100644 + env = setEnv(env, "GIT_CONFIG_GLOBAL", os.DevNull) + env = setEnv(env, "GIT_CONFIG_NOSYSTEM", "1") + env = setEnv(env, "GIT_TERMINAL_PROMPT", "0") -+ env = ensureGoFlagsModMode(env) ++ env = ensureSyntheticModuleGoFlags(env) + if getEnv(env, "HOME") == "" { + homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") + if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -6449,13 +6487,17 @@ index e2b0be5..cb5b10a 100644 + if published := publishedPaths[source]; published != "" { + return published, nil + } -+ digest, err := fullDigestFile(source) ++ data, err := os.ReadFile(source) + if err != nil { -+ return "", fmt.Errorf("digest shared synthetic helper archive %s: %w", source, err) ++ return "", fmt.Errorf("read shared synthetic helper archive %s: %w", source, err) ++ } ++ if err := normalizePublishedGoArchiveBuildID(data); err != nil { ++ return "", fmt.Errorf("normalize shared synthetic helper archive %s: %w", source, err) + } ++ digest := fmt.Sprintf("%x", sha256.Sum256(data)) + destination := filepath.Join(outputDir, "archives", digest+".a") + if _, err := os.Stat(destination); errors.Is(err, os.ErrNotExist) { -+ if err := copyArchiveFile(source, destination); err != nil { ++ if err := os.WriteFile(destination, data, 0o644); err != nil { + return "", fmt.Errorf("publish shared synthetic helper archive %s: %w", source, err) + } + } else if err != nil { @@ -6496,6 +6538,49 @@ index e2b0be5..cb5b10a 100644 + return writeJSONAtomically(filepath.Join(outputDir, cacheManifestFileName), manifest) +} + ++// normalizePublishedGoArchiveBuildID removes execroot-specific action IDs from ++// a copied helper archive. The Go cache keeps the original archive and build ID; ++// only the declared Bazel output receives the stable content/content form. ++func normalizePublishedGoArchiveBuildID(data []byte) error { ++ if !bytes.HasPrefix(data, []byte("!\n")) { ++ return nil ++ } ++ header := data ++ if len(header) > 1024 { ++ header = header[:1024] ++ } ++ const marker = "\nbuild id \"" ++ start := bytes.Index(header, []byte(marker)) ++ if start < 0 { ++ return nil ++ } ++ start += len(marker) ++ end := bytes.IndexByte(header[start:], '"') ++ if end < 0 { ++ return errors.New("unterminated Go archive build ID") ++ } ++ end += start ++ buildID := string(header[start:end]) ++ if buildID == "" { ++ return nil ++ } ++ parts := strings.Split(buildID, "/") ++ if len(parts) != 2 || len(parts[0]) != len(parts[1]) { ++ return fmt.Errorf("unexpected Go archive build ID %q", buildID) ++ } ++ stableBuildID := parts[1] + "/" + parts[1] ++ for offset := 0; ; { ++ index := bytes.Index(data[offset:], []byte(buildID)) ++ if index < 0 { ++ break ++ } ++ index += offset ++ copy(data[index:index+len(buildID)], stableBuildID) ++ offset = index + len(buildID) ++ } ++ return nil ++} ++ +// reusableSharedSyntheticTestmainHelperBundle returns the declared shared +// bundle only when it covers the requested roots and does not overlap archives +// supplied by the current Bazel target. Overlap falls back to the existing @@ -6884,7 +6969,7 @@ index e2b0be5..cb5b10a 100644 + stem := fmt.Sprintf("%x", sha256.Sum256([]byte(meta.ImportPath)))[:16] + outLinkobjPath := filepath.Join(archiveDir, stem+".a") + outInterfacePath := filepath.Join(archiveDir, stem+".iface.a") -+ trimPath, err := createTrimPath() ++ trimPath, err := syntheticSourceTrimPath(meta.Dir, meta.ImportPath) + if err != nil { + return "", "", fmt.Errorf("create trimpath for %s: %w", pkg, err) + } @@ -6927,7 +7012,7 @@ index e2b0be5..cb5b10a 100644 + } + for i, sSrc := range filteredSrcs.sSrcs { + obj := filepath.Join(packageWorkDir, fmt.Sprintf("s%d.o", i)) -+ if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, obj); err != nil { ++ if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, trimPath, obj); err != nil { + return "", "", fmt.Errorf("assemble synthetic helper %s: %w", pkg, err) + } + objFiles = append(objFiles, obj) @@ -6955,6 +7040,21 @@ index e2b0be5..cb5b10a 100644 + return result.compilePath, result.linkPath, nil +} + ++// syntheticSourceTrimPath gives module sources a stable logical prefix. The ++// physical GOMODCACHE belongs to the action sandbox and must not reach archives ++// that Bazel may publish to a shared cache. ++func syntheticSourceTrimPath(sourceDir, importPath string) (string, error) { ++ trimPath, err := createTrimPath() ++ if err != nil { ++ return "", err ++ } ++ sourceDir = strings.TrimSpace(sourceDir) ++ if sourceDir == "" { ++ return trimPath, nil ++ } ++ return trimPath + ";" + abs(sourceDir) + "=>" + strings.TrimSpace(importPath), nil ++} ++ +func shouldSkipOrchestrionForImportPath(importPath string) bool { + if strings.HasPrefix(importPath, "github.com/bazelbuild/rules_go/go/tools/") { + return true @@ -7213,10 +7313,10 @@ index e2b0be5..cb5b10a 100644 +} diff --git a/go/tools/builders/compilepkg_test.go b/go/tools/builders/compilepkg_test.go new file mode 100644 -index 0000000..320fb66 +index 0000000..91065d6 --- /dev/null +++ b/go/tools/builders/compilepkg_test.go -@@ -0,0 +1,837 @@ +@@ -0,0 +1,915 @@ +package main + +import ( @@ -7359,6 +7459,51 @@ index 0000000..320fb66 + } +} + ++func TestSharedSyntheticTestmainHelperBundleNormalizesGoBuildIDs(t *testing.T) { ++ const contentID = "cccccccccccccccccccc" ++ archive := func(actionID string) []byte { ++ buildID := actionID + "/" + contentID ++ return []byte("!\narchive header\ngo object test\nbuild id \"" + buildID + "\"\npayload build id \"" + buildID + "\"\n") ++ } ++ ++ sourceDir := t.TempDir() ++ firstSource := filepath.Join(sourceDir, "first.a") ++ secondSource := filepath.Join(sourceDir, "second.a") ++ if err := os.WriteFile(firstSource, archive("aaaaaaaaaaaaaaaaaaaa"), 0o644); err != nil { ++ t.Fatal(err) ++ } ++ if err := os.WriteFile(secondSource, archive("bbbbbbbbbbbbbbbbbbbb"), 0o644); err != nil { ++ t.Fatal(err) ++ } ++ ++ bundleDir := filepath.Join(t.TempDir(), "shared-helpers") ++ compiled := map[string]compiledModuleArchive{ ++ "example.com/first": {compilePath: firstSource, linkPath: firstSource}, ++ "example.com/second": {compilePath: secondSource, linkPath: secondSource}, ++ } ++ if err := writeSharedSyntheticTestmainHelperBundle(bundleDir, compiled, orchestrionModeTestOptimization); err != nil { ++ t.Fatal(err) ++ } ++ ++ manifest, err := readSyntheticTestmainHelperManifest(bundleDir) ++ if err != nil { ++ t.Fatal(err) ++ } ++ firstPath := manifest.Packages["example.com/first"].CompilePath ++ secondPath := manifest.Packages["example.com/second"].CompilePath ++ if firstPath != secondPath { ++ t.Fatalf("equivalent archives published at different paths: %q != %q", firstPath, secondPath) ++ } ++ published, err := os.ReadFile(filepath.Join(bundleDir, firstPath)) ++ if err != nil { ++ t.Fatal(err) ++ } ++ wantBuildID := contentID + "/" + contentID ++ if got := strings.Count(string(published), wantBuildID); got != 2 { ++ t.Fatalf("normalized build ID occurs %d times, want 2", got) ++ } ++} ++ +func TestSharedSyntheticTestmainHelperBundleFallsBackOnTargetOverlap(t *testing.T) { + rootPackage := syntheticTestmainRootPackagesTestOptimization[0].packagePath + sourceDir := t.TempDir() @@ -7758,6 +7903,39 @@ index 0000000..320fb66 + } +} + ++func TestSyntheticModuleGoFlagsRequireTrimpath(t *testing.T) { ++ tests := []struct { ++ name string ++ flags string ++ want string ++ }{ ++ {name: "empty", want: "-mod=mod -trimpath"}, ++ {name: "preserve module mode", flags: "-mod=readonly", want: "-mod=readonly -trimpath"}, ++ {name: "preserve trimpath", flags: "-trimpath -mod=vendor", want: "-trimpath -mod=vendor"}, ++ {name: "override disabled trimpath", flags: "-trimpath=false", want: "-mod=mod -trimpath=false -trimpath"}, ++ } ++ for _, test := range tests { ++ t.Run(test.name, func(t *testing.T) { ++ env := ensureSyntheticModuleGoFlags([]string{"GOFLAGS=" + test.flags}) ++ if got := getEnv(env, "GOFLAGS"); got != test.want { ++ t.Fatalf("GOFLAGS = %q, want %q", got, test.want) ++ } ++ }) ++ } ++} ++ ++func TestSyntheticSourceTrimPathReplacesModuleCacheRoot(t *testing.T) { ++ sourceDir := filepath.Join(t.TempDir(), "pkg", "mod", "example.com", "helper@v1.0.0") ++ got, err := syntheticSourceTrimPath(sourceDir, "example.com/helper") ++ if err != nil { ++ t.Fatalf("syntheticSourceTrimPath error: %v", err) ++ } ++ want := abs(sourceDir) + "=>example.com/helper" ++ if !strings.Contains(got, want) { ++ t.Fatalf("syntheticSourceTrimPath = %q, want mapping %q", got, want) ++ } ++} ++ +// TestWithSyntheticTestmainModuleCacheEnvRestoresEnvironment verifies that the +// synthetic helper wrapper points subprocess cache env vars at the helper +// cache root during execution and restores the caller's environment after. @@ -8748,7 +8926,7 @@ index 893a0f6..16955a7 100644 + return "" +} diff --git a/go/tools/builders/importcfg.go b/go/tools/builders/importcfg.go -index c25763a..b6298e6 100644 +index c25763a..7c19d36 100644 --- a/go/tools/builders/importcfg.go +++ b/go/tools/builders/importcfg.go @@ -17,21 +17,94 @@ package main @@ -9351,7 +9529,7 @@ index c25763a..b6298e6 100644 + if err != nil { + return nil, fmt.Errorf("prepare module resolution env: %w", err) + } -+ cmd.Env = ensureGoFlagsModMode(cmd.Env) ++ cmd.Env = ensureSyntheticModuleGoFlags(cmd.Env) + if getEnv(cmd.Env, "HOME") == "" { + homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") + if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -12026,10 +12204,10 @@ index 11dc0ab..610af53 100644 if *buildmode == "c-archive" { diff --git a/go/tools/builders/module_proxy.go b/go/tools/builders/module_proxy.go new file mode 100644 -index 0000000..b44c0f3 +index 0000000..91b0ba4 --- /dev/null +++ b/go/tools/builders/module_proxy.go -@@ -0,0 +1,291 @@ +@@ -0,0 +1,298 @@ +package main + +import ( @@ -12115,15 +12293,22 @@ index 0000000..b44c0f3 + return env, nil +} + -+func ensureGoFlagsModMode(env []string) []string { ++// ensureSyntheticModuleGoFlags keeps helper module builds offline-compatible ++// and removes their temporary module-cache roots from compiled archives. ++func ensureSyntheticModuleGoFlags(env []string) []string { + goFlags := strings.TrimSpace(getEnv(env, "GOFLAGS")) -+ if strings.Contains(goFlags, "-mod=") { -+ return env ++ if !strings.Contains(goFlags, "-mod=") { ++ goFlags = strings.TrimSpace("-mod=mod " + goFlags) + } -+ if goFlags == "" { -+ goFlags = "-mod=mod" -+ } else { -+ goFlags = "-mod=mod " + goFlags ++ trimPathEnabled := false ++ for _, flag := range strings.Fields(goFlags) { ++ if flag == "-trimpath" || flag == "-trimpath=true" { ++ trimPathEnabled = true ++ break ++ } ++ } ++ if !trimPathEnabled { ++ goFlags += " -trimpath" + } + return setEnv(env, "GOFLAGS", goFlags) +} @@ -14294,10 +14479,10 @@ index 0000000..fbb5712 +} diff --git a/go/tools/builders/orchestrion_skip_test.go b/go/tools/builders/orchestrion_skip_test.go new file mode 100644 -index 0000000..32c0cf4 +index 0000000..00d37c1 --- /dev/null +++ b/go/tools/builders/orchestrion_skip_test.go -@@ -0,0 +1,383 @@ +@@ -0,0 +1,392 @@ +package main + +import ( @@ -14594,6 +14779,14 @@ index 0000000..32c0cf4 + +func TestSyntheticTestmainManifestPublishesExternalArchives(t *testing.T) { + execroot := t.TempDir() ++ previousDir, err := os.Getwd() ++ if err != nil { ++ t.Fatal(err) ++ } ++ if err := os.Chdir(execroot); err != nil { ++ t.Fatal(err) ++ } ++ t.Cleanup(func() { _ = os.Chdir(previousDir) }) + externalArchive := filepath.Join(t.TempDir(), "helper.a") + if err := os.WriteFile(externalArchive, []byte("helper archive"), 0o644); err != nil { + t.Fatal(err) @@ -14636,7 +14829,8 @@ index 0000000..32c0cf4 + t.Fatalf("published helper archive count = %d, want 1", len(entries)) + } + publishedArchive := filepath.Join(helperOutputDir, entries[0].Name()) -+ if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedArchive) { ++ publishedRelative := execrootRelativePath(publishedArchive) ++ if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedRelative) { + t.Fatalf("manifest does not reference published helper %q:\n%s", publishedArchive, gotManifest) + } + publishedData, err := os.ReadFile(publishedArchive) @@ -16565,7 +16759,7 @@ index 0000000..9edae16 + } +} diff --git a/go/tools/builders/stdlib.go b/go/tools/builders/stdlib.go -index cfcb991..65244aa 100644 +index cfcb991..8ea6331 100644 --- a/go/tools/builders/stdlib.go +++ b/go/tools/builders/stdlib.go @@ -21,15 +21,29 @@ import ( @@ -16687,7 +16881,7 @@ index cfcb991..65244aa 100644 // Build the commands needed to build the std library in the right mode // NOTE: the go command stamps compiled .a files with build ids, which are -@@ -121,7 +164,112 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -121,7 +164,117 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) // creating reproducible builds because the build ids are hashed from // CGO_CFLAGS, which frequently contains absolute paths. As a workaround, // we strip the build ids, since they won't be used after this. @@ -16798,10 +16992,15 @@ index cfcb991..65244aa 100644 + } + } + installArgs := goenv.goCmd("install", "-toolexec", toolexec) ++ if *orchestrion != "" { ++ // Helper module exports use Go's reproducible build mode. Compile the ++ // woven stdlib with the same flag so imported package fingerprints match. ++ installArgs = append(installArgs, "-trimpath") ++ } if len(build.Default.BuildTags) > 0 { installArgs = append(installArgs, "-tags", strings.Join(build.Default.BuildTags, ",")) } -@@ -135,7 +283,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -135,7 +288,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) installArgs = append(installArgs, "-msan") } if *pgoprofile != "" { @@ -16810,7 +17009,7 @@ index cfcb991..65244aa 100644 } if *shared { gcflags = append(gcflags, "-shared") -@@ -167,8 +315,393 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -167,8 +320,393 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } installArgs = append(installArgs, packages...) diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index ca479c8c..d02c738d 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -4640,6 +4640,35 @@ index d2de6b9..ffbf778 100644 + } + return "." +} +diff --git a/go/tools/builders/asm.go b/go/tools/builders/asm.go +index 6704be7..5e7d340 100644 +--- a/go/tools/builders/asm.go ++++ b/go/tools/builders/asm.go +@@ -111,7 +111,7 @@ func buildSymabisFile(goenv *env, packagePath string, sFiles, hFiles []fileInfo, + return symabisName, err + } + +-func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, outPath string) error { ++func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, trimPath, outPath string) error { + args := goenv.goTool("asm") + args = append(args, asmFlags...) + // The package path has to be specified as of Go 1.19 or the resulting +@@ -121,10 +121,13 @@ func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, outPath + args = append(args, "-p", packagePath) + } + args = append(args, ASM_DEFINES...) +- args = append(args, "-trimpath", ".") ++ if trimPath == "" { ++ trimPath = abs(".") ++ } ++ args = append(args, "-trimpath", trimPath) + args = append(args, "-o", outPath) + args = append(args, "--", srcPath) +- absArgs(args, []string{"-I", "-o", "-trimpath"}) ++ absArgs(args, []string{"-I", "-o"}) + return goenv.runCommand(args) + } + diff --git a/go/tools/builders/builder.go b/go/tools/builders/builder.go index fdeda2d..76d69b6 100644 --- a/go/tools/builders/builder.go @@ -4791,7 +4820,7 @@ index 6f67914..5451c67 100644 transformArgs(normalized, cgoAbsEnvFlags, func(s string) string { if strings.HasPrefix(s, cgoAbsPlaceholder) { diff --git a/go/tools/builders/compilepkg.go b/go/tools/builders/compilepkg.go -index e2b0be5..cb5b10a 100644 +index e2b0be5..5356b08 100644 --- a/go/tools/builders/compilepkg.go +++ b/go/tools/builders/compilepkg.go @@ -17,17 +17,252 @@ @@ -4991,7 +5020,7 @@ index e2b0be5..cb5b10a 100644 + } + published[publishedPath] = true + } -+ lines[index] = "packagefile " + packagePath + "=" + publishedPath ++ lines[index] = "packagefile " + packagePath + "=" + execrootRelativePath(publishedPath) + } + return []byte(strings.Join(lines, "\n")), nil +} @@ -5243,7 +5272,16 @@ index e2b0be5..cb5b10a 100644 // Compile the .s files with Go's assembler, if this is not a cgo package. // Cgo is assembled by cc above. -@@ -489,100 +793,1895 @@ func compileArchive( +@@ -466,7 +770,7 @@ func compileArchive( + } + for i, sSrc := range srcs.sSrcs { + obj := filepath.Join(workDir, fmt.Sprintf("s%d.o", i)) +- if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, obj); err != nil { ++ if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, "", obj); err != nil { + return err + } + objFiles = append(objFiles, obj) +@@ -489,100 +793,1957 @@ func compileArchive( return nil } @@ -5875,7 +5913,7 @@ index e2b0be5..cb5b10a 100644 + env = setEnv(env, "GIT_CONFIG_GLOBAL", os.DevNull) + env = setEnv(env, "GIT_CONFIG_NOSYSTEM", "1") + env = setEnv(env, "GIT_TERMINAL_PROMPT", "0") -+ env = ensureGoFlagsModMode(env) ++ env = ensureSyntheticModuleGoFlags(env) + if getEnv(env, "HOME") == "" { + homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") + if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -6445,13 +6483,17 @@ index e2b0be5..cb5b10a 100644 + if published := publishedPaths[source]; published != "" { + return published, nil + } -+ digest, err := fullDigestFile(source) ++ data, err := os.ReadFile(source) + if err != nil { -+ return "", fmt.Errorf("digest shared synthetic helper archive %s: %w", source, err) ++ return "", fmt.Errorf("read shared synthetic helper archive %s: %w", source, err) ++ } ++ if err := normalizePublishedGoArchiveBuildID(data); err != nil { ++ return "", fmt.Errorf("normalize shared synthetic helper archive %s: %w", source, err) + } ++ digest := fmt.Sprintf("%x", sha256.Sum256(data)) + destination := filepath.Join(outputDir, "archives", digest+".a") + if _, err := os.Stat(destination); errors.Is(err, os.ErrNotExist) { -+ if err := copyArchiveFile(source, destination); err != nil { ++ if err := os.WriteFile(destination, data, 0o644); err != nil { + return "", fmt.Errorf("publish shared synthetic helper archive %s: %w", source, err) + } + } else if err != nil { @@ -6492,6 +6534,49 @@ index e2b0be5..cb5b10a 100644 + return writeJSONAtomically(filepath.Join(outputDir, cacheManifestFileName), manifest) +} + ++// normalizePublishedGoArchiveBuildID removes execroot-specific action IDs from ++// a copied helper archive. The Go cache keeps the original archive and build ID; ++// only the declared Bazel output receives the stable content/content form. ++func normalizePublishedGoArchiveBuildID(data []byte) error { ++ if !bytes.HasPrefix(data, []byte("!\n")) { ++ return nil ++ } ++ header := data ++ if len(header) > 1024 { ++ header = header[:1024] ++ } ++ const marker = "\nbuild id \"" ++ start := bytes.Index(header, []byte(marker)) ++ if start < 0 { ++ return nil ++ } ++ start += len(marker) ++ end := bytes.IndexByte(header[start:], '"') ++ if end < 0 { ++ return errors.New("unterminated Go archive build ID") ++ } ++ end += start ++ buildID := string(header[start:end]) ++ if buildID == "" { ++ return nil ++ } ++ parts := strings.Split(buildID, "/") ++ if len(parts) != 2 || len(parts[0]) != len(parts[1]) { ++ return fmt.Errorf("unexpected Go archive build ID %q", buildID) ++ } ++ stableBuildID := parts[1] + "/" + parts[1] ++ for offset := 0; ; { ++ index := bytes.Index(data[offset:], []byte(buildID)) ++ if index < 0 { ++ break ++ } ++ index += offset ++ copy(data[index:index+len(buildID)], stableBuildID) ++ offset = index + len(buildID) ++ } ++ return nil ++} ++ +// reusableSharedSyntheticTestmainHelperBundle returns the declared shared +// bundle only when it covers the requested roots and does not overlap archives +// supplied by the current Bazel target. Overlap falls back to the existing @@ -6880,7 +6965,7 @@ index e2b0be5..cb5b10a 100644 + stem := fmt.Sprintf("%x", sha256.Sum256([]byte(meta.ImportPath)))[:16] + outLinkobjPath := filepath.Join(archiveDir, stem+".a") + outInterfacePath := filepath.Join(archiveDir, stem+".iface.a") -+ trimPath, err := createTrimPath() ++ trimPath, err := syntheticSourceTrimPath(meta.Dir, meta.ImportPath) + if err != nil { + return "", "", fmt.Errorf("create trimpath for %s: %w", pkg, err) + } @@ -6923,7 +7008,7 @@ index e2b0be5..cb5b10a 100644 + } + for i, sSrc := range filteredSrcs.sSrcs { + obj := filepath.Join(packageWorkDir, fmt.Sprintf("s%d.o", i)) -+ if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, obj); err != nil { ++ if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, trimPath, obj); err != nil { + return "", "", fmt.Errorf("assemble synthetic helper %s: %w", pkg, err) + } + objFiles = append(objFiles, obj) @@ -6951,6 +7036,21 @@ index e2b0be5..cb5b10a 100644 + return result.compilePath, result.linkPath, nil +} + ++// syntheticSourceTrimPath gives module sources a stable logical prefix. The ++// physical GOMODCACHE belongs to the action sandbox and must not reach archives ++// that Bazel may publish to a shared cache. ++func syntheticSourceTrimPath(sourceDir, importPath string) (string, error) { ++ trimPath, err := createTrimPath() ++ if err != nil { ++ return "", err ++ } ++ sourceDir = strings.TrimSpace(sourceDir) ++ if sourceDir == "" { ++ return trimPath, nil ++ } ++ return trimPath + ";" + abs(sourceDir) + "=>" + strings.TrimSpace(importPath), nil ++} ++ +func shouldSkipOrchestrionForImportPath(importPath string) bool { + if strings.HasPrefix(importPath, "github.com/bazelbuild/rules_go/go/tools/") { + return true @@ -7209,10 +7309,10 @@ index e2b0be5..cb5b10a 100644 +} diff --git a/go/tools/builders/compilepkg_test.go b/go/tools/builders/compilepkg_test.go new file mode 100644 -index 0000000..320fb66 +index 0000000..91065d6 --- /dev/null +++ b/go/tools/builders/compilepkg_test.go -@@ -0,0 +1,837 @@ +@@ -0,0 +1,915 @@ +package main + +import ( @@ -7355,6 +7455,51 @@ index 0000000..320fb66 + } +} + ++func TestSharedSyntheticTestmainHelperBundleNormalizesGoBuildIDs(t *testing.T) { ++ const contentID = "cccccccccccccccccccc" ++ archive := func(actionID string) []byte { ++ buildID := actionID + "/" + contentID ++ return []byte("!\narchive header\ngo object test\nbuild id \"" + buildID + "\"\npayload build id \"" + buildID + "\"\n") ++ } ++ ++ sourceDir := t.TempDir() ++ firstSource := filepath.Join(sourceDir, "first.a") ++ secondSource := filepath.Join(sourceDir, "second.a") ++ if err := os.WriteFile(firstSource, archive("aaaaaaaaaaaaaaaaaaaa"), 0o644); err != nil { ++ t.Fatal(err) ++ } ++ if err := os.WriteFile(secondSource, archive("bbbbbbbbbbbbbbbbbbbb"), 0o644); err != nil { ++ t.Fatal(err) ++ } ++ ++ bundleDir := filepath.Join(t.TempDir(), "shared-helpers") ++ compiled := map[string]compiledModuleArchive{ ++ "example.com/first": {compilePath: firstSource, linkPath: firstSource}, ++ "example.com/second": {compilePath: secondSource, linkPath: secondSource}, ++ } ++ if err := writeSharedSyntheticTestmainHelperBundle(bundleDir, compiled, orchestrionModeTestOptimization); err != nil { ++ t.Fatal(err) ++ } ++ ++ manifest, err := readSyntheticTestmainHelperManifest(bundleDir) ++ if err != nil { ++ t.Fatal(err) ++ } ++ firstPath := manifest.Packages["example.com/first"].CompilePath ++ secondPath := manifest.Packages["example.com/second"].CompilePath ++ if firstPath != secondPath { ++ t.Fatalf("equivalent archives published at different paths: %q != %q", firstPath, secondPath) ++ } ++ published, err := os.ReadFile(filepath.Join(bundleDir, firstPath)) ++ if err != nil { ++ t.Fatal(err) ++ } ++ wantBuildID := contentID + "/" + contentID ++ if got := strings.Count(string(published), wantBuildID); got != 2 { ++ t.Fatalf("normalized build ID occurs %d times, want 2", got) ++ } ++} ++ +func TestSharedSyntheticTestmainHelperBundleFallsBackOnTargetOverlap(t *testing.T) { + rootPackage := syntheticTestmainRootPackagesTestOptimization[0].packagePath + sourceDir := t.TempDir() @@ -7754,6 +7899,39 @@ index 0000000..320fb66 + } +} + ++func TestSyntheticModuleGoFlagsRequireTrimpath(t *testing.T) { ++ tests := []struct { ++ name string ++ flags string ++ want string ++ }{ ++ {name: "empty", want: "-mod=mod -trimpath"}, ++ {name: "preserve module mode", flags: "-mod=readonly", want: "-mod=readonly -trimpath"}, ++ {name: "preserve trimpath", flags: "-trimpath -mod=vendor", want: "-trimpath -mod=vendor"}, ++ {name: "override disabled trimpath", flags: "-trimpath=false", want: "-mod=mod -trimpath=false -trimpath"}, ++ } ++ for _, test := range tests { ++ t.Run(test.name, func(t *testing.T) { ++ env := ensureSyntheticModuleGoFlags([]string{"GOFLAGS=" + test.flags}) ++ if got := getEnv(env, "GOFLAGS"); got != test.want { ++ t.Fatalf("GOFLAGS = %q, want %q", got, test.want) ++ } ++ }) ++ } ++} ++ ++func TestSyntheticSourceTrimPathReplacesModuleCacheRoot(t *testing.T) { ++ sourceDir := filepath.Join(t.TempDir(), "pkg", "mod", "example.com", "helper@v1.0.0") ++ got, err := syntheticSourceTrimPath(sourceDir, "example.com/helper") ++ if err != nil { ++ t.Fatalf("syntheticSourceTrimPath error: %v", err) ++ } ++ want := abs(sourceDir) + "=>example.com/helper" ++ if !strings.Contains(got, want) { ++ t.Fatalf("syntheticSourceTrimPath = %q, want mapping %q", got, want) ++ } ++} ++ +// TestWithSyntheticTestmainModuleCacheEnvRestoresEnvironment verifies that the +// synthetic helper wrapper points subprocess cache env vars at the helper +// cache root during execution and restores the caller's environment after. @@ -8738,7 +8916,7 @@ index 893a0f6..16955a7 100644 + return "" +} diff --git a/go/tools/builders/importcfg.go b/go/tools/builders/importcfg.go -index c25763a..b6298e6 100644 +index c25763a..7c19d36 100644 --- a/go/tools/builders/importcfg.go +++ b/go/tools/builders/importcfg.go @@ -17,21 +17,94 @@ package main @@ -9341,7 +9519,7 @@ index c25763a..b6298e6 100644 + if err != nil { + return nil, fmt.Errorf("prepare module resolution env: %w", err) + } -+ cmd.Env = ensureGoFlagsModMode(cmd.Env) ++ cmd.Env = ensureSyntheticModuleGoFlags(cmd.Env) + if getEnv(cmd.Env, "HOME") == "" { + homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") + if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -12016,10 +12194,10 @@ index 11dc0ab..610af53 100644 if *buildmode == "c-archive" { diff --git a/go/tools/builders/module_proxy.go b/go/tools/builders/module_proxy.go new file mode 100644 -index 0000000..b44c0f3 +index 0000000..91b0ba4 --- /dev/null +++ b/go/tools/builders/module_proxy.go -@@ -0,0 +1,291 @@ +@@ -0,0 +1,298 @@ +package main + +import ( @@ -12105,15 +12283,22 @@ index 0000000..b44c0f3 + return env, nil +} + -+func ensureGoFlagsModMode(env []string) []string { ++// ensureSyntheticModuleGoFlags keeps helper module builds offline-compatible ++// and removes their temporary module-cache roots from compiled archives. ++func ensureSyntheticModuleGoFlags(env []string) []string { + goFlags := strings.TrimSpace(getEnv(env, "GOFLAGS")) -+ if strings.Contains(goFlags, "-mod=") { -+ return env ++ if !strings.Contains(goFlags, "-mod=") { ++ goFlags = strings.TrimSpace("-mod=mod " + goFlags) + } -+ if goFlags == "" { -+ goFlags = "-mod=mod" -+ } else { -+ goFlags = "-mod=mod " + goFlags ++ trimPathEnabled := false ++ for _, flag := range strings.Fields(goFlags) { ++ if flag == "-trimpath" || flag == "-trimpath=true" { ++ trimPathEnabled = true ++ break ++ } ++ } ++ if !trimPathEnabled { ++ goFlags += " -trimpath" + } + return setEnv(env, "GOFLAGS", goFlags) +} @@ -14284,10 +14469,10 @@ index 0000000..fbb5712 +} diff --git a/go/tools/builders/orchestrion_skip_test.go b/go/tools/builders/orchestrion_skip_test.go new file mode 100644 -index 0000000..32c0cf4 +index 0000000..00d37c1 --- /dev/null +++ b/go/tools/builders/orchestrion_skip_test.go -@@ -0,0 +1,383 @@ +@@ -0,0 +1,392 @@ +package main + +import ( @@ -14584,6 +14769,14 @@ index 0000000..32c0cf4 + +func TestSyntheticTestmainManifestPublishesExternalArchives(t *testing.T) { + execroot := t.TempDir() ++ previousDir, err := os.Getwd() ++ if err != nil { ++ t.Fatal(err) ++ } ++ if err := os.Chdir(execroot); err != nil { ++ t.Fatal(err) ++ } ++ t.Cleanup(func() { _ = os.Chdir(previousDir) }) + externalArchive := filepath.Join(t.TempDir(), "helper.a") + if err := os.WriteFile(externalArchive, []byte("helper archive"), 0o644); err != nil { + t.Fatal(err) @@ -14626,7 +14819,8 @@ index 0000000..32c0cf4 + t.Fatalf("published helper archive count = %d, want 1", len(entries)) + } + publishedArchive := filepath.Join(helperOutputDir, entries[0].Name()) -+ if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedArchive) { ++ publishedRelative := execrootRelativePath(publishedArchive) ++ if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedRelative) { + t.Fatalf("manifest does not reference published helper %q:\n%s", publishedArchive, gotManifest) + } + publishedData, err := os.ReadFile(publishedArchive) @@ -16555,7 +16749,7 @@ index 0000000..9edae16 + } +} diff --git a/go/tools/builders/stdlib.go b/go/tools/builders/stdlib.go -index cfcb991..65244aa 100644 +index cfcb991..8ea6331 100644 --- a/go/tools/builders/stdlib.go +++ b/go/tools/builders/stdlib.go @@ -21,15 +21,29 @@ import ( @@ -16677,7 +16871,7 @@ index cfcb991..65244aa 100644 // Build the commands needed to build the std library in the right mode // NOTE: the go command stamps compiled .a files with build ids, which are -@@ -121,7 +164,112 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -121,7 +164,117 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) // creating reproducible builds because the build ids are hashed from // CGO_CFLAGS, which frequently contains absolute paths. As a workaround, // we strip the build ids, since they won't be used after this. @@ -16788,10 +16982,15 @@ index cfcb991..65244aa 100644 + } + } + installArgs := goenv.goCmd("install", "-toolexec", toolexec) ++ if *orchestrion != "" { ++ // Helper module exports use Go's reproducible build mode. Compile the ++ // woven stdlib with the same flag so imported package fingerprints match. ++ installArgs = append(installArgs, "-trimpath") ++ } if len(build.Default.BuildTags) > 0 { installArgs = append(installArgs, "-tags", strings.Join(build.Default.BuildTags, ",")) } -@@ -135,7 +283,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -135,7 +288,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) installArgs = append(installArgs, "-msan") } if *pgoprofile != "" { @@ -16800,7 +16999,7 @@ index cfcb991..65244aa 100644 } if *shared { gcflags = append(gcflags, "-shared") -@@ -167,8 +315,393 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -167,8 +320,393 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } installArgs = append(installArgs, packages...) diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index 78af0fc0..d5526007 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -4644,6 +4644,35 @@ index d2de6b9..ffbf778 100644 + } + return "." +} +diff --git a/go/tools/builders/asm.go b/go/tools/builders/asm.go +index 6704be7..5e7d340 100644 +--- a/go/tools/builders/asm.go ++++ b/go/tools/builders/asm.go +@@ -111,7 +111,7 @@ func buildSymabisFile(goenv *env, packagePath string, sFiles, hFiles []fileInfo, + return symabisName, err + } + +-func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, outPath string) error { ++func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, trimPath, outPath string) error { + args := goenv.goTool("asm") + args = append(args, asmFlags...) + // The package path has to be specified as of Go 1.19 or the resulting +@@ -121,10 +121,13 @@ func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, outPath + args = append(args, "-p", packagePath) + } + args = append(args, ASM_DEFINES...) +- args = append(args, "-trimpath", ".") ++ if trimPath == "" { ++ trimPath = abs(".") ++ } ++ args = append(args, "-trimpath", trimPath) + args = append(args, "-o", outPath) + args = append(args, "--", srcPath) +- absArgs(args, []string{"-I", "-o", "-trimpath"}) ++ absArgs(args, []string{"-I", "-o"}) + return goenv.runCommand(args) + } + diff --git a/go/tools/builders/builder.go b/go/tools/builders/builder.go index fdeda2d..76d69b6 100644 --- a/go/tools/builders/builder.go @@ -4795,7 +4824,7 @@ index 6f67914..5451c67 100644 transformArgs(normalized, cgoAbsEnvFlags, func(s string) string { if strings.HasPrefix(s, cgoAbsPlaceholder) { diff --git a/go/tools/builders/compilepkg.go b/go/tools/builders/compilepkg.go -index a1782d4..8004cd0 100644 +index a1782d4..5d3ae22 100644 --- a/go/tools/builders/compilepkg.go +++ b/go/tools/builders/compilepkg.go @@ -17,17 +17,252 @@ @@ -4995,7 +5024,7 @@ index a1782d4..8004cd0 100644 + } + published[publishedPath] = true + } -+ lines[index] = "packagefile " + packagePath + "=" + publishedPath ++ lines[index] = "packagefile " + packagePath + "=" + execrootRelativePath(publishedPath) + } + return []byte(strings.Join(lines, "\n")), nil +} @@ -5247,7 +5276,16 @@ index a1782d4..8004cd0 100644 // Compile the .s files with Go's assembler, if this is not a cgo package. // Cgo is assembled by cc above. -@@ -487,100 +791,1895 @@ func compileArchive( +@@ -464,7 +768,7 @@ func compileArchive( + } + for i, sSrc := range srcs.sSrcs { + obj := filepath.Join(workDir, fmt.Sprintf("s%d.o", i)) +- if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, obj); err != nil { ++ if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, "", obj); err != nil { + return err + } + objFiles = append(objFiles, obj) +@@ -487,100 +791,1957 @@ func compileArchive( return nil } @@ -5879,7 +5917,7 @@ index a1782d4..8004cd0 100644 + env = setEnv(env, "GIT_CONFIG_GLOBAL", os.DevNull) + env = setEnv(env, "GIT_CONFIG_NOSYSTEM", "1") + env = setEnv(env, "GIT_TERMINAL_PROMPT", "0") -+ env = ensureGoFlagsModMode(env) ++ env = ensureSyntheticModuleGoFlags(env) + if getEnv(env, "HOME") == "" { + homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") + if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -6449,13 +6487,17 @@ index a1782d4..8004cd0 100644 + if published := publishedPaths[source]; published != "" { + return published, nil + } -+ digest, err := fullDigestFile(source) ++ data, err := os.ReadFile(source) + if err != nil { -+ return "", fmt.Errorf("digest shared synthetic helper archive %s: %w", source, err) ++ return "", fmt.Errorf("read shared synthetic helper archive %s: %w", source, err) ++ } ++ if err := normalizePublishedGoArchiveBuildID(data); err != nil { ++ return "", fmt.Errorf("normalize shared synthetic helper archive %s: %w", source, err) + } ++ digest := fmt.Sprintf("%x", sha256.Sum256(data)) + destination := filepath.Join(outputDir, "archives", digest+".a") + if _, err := os.Stat(destination); errors.Is(err, os.ErrNotExist) { -+ if err := copyArchiveFile(source, destination); err != nil { ++ if err := os.WriteFile(destination, data, 0o644); err != nil { + return "", fmt.Errorf("publish shared synthetic helper archive %s: %w", source, err) + } + } else if err != nil { @@ -6496,6 +6538,49 @@ index a1782d4..8004cd0 100644 + return writeJSONAtomically(filepath.Join(outputDir, cacheManifestFileName), manifest) +} + ++// normalizePublishedGoArchiveBuildID removes execroot-specific action IDs from ++// a copied helper archive. The Go cache keeps the original archive and build ID; ++// only the declared Bazel output receives the stable content/content form. ++func normalizePublishedGoArchiveBuildID(data []byte) error { ++ if !bytes.HasPrefix(data, []byte("!\n")) { ++ return nil ++ } ++ header := data ++ if len(header) > 1024 { ++ header = header[:1024] ++ } ++ const marker = "\nbuild id \"" ++ start := bytes.Index(header, []byte(marker)) ++ if start < 0 { ++ return nil ++ } ++ start += len(marker) ++ end := bytes.IndexByte(header[start:], '"') ++ if end < 0 { ++ return errors.New("unterminated Go archive build ID") ++ } ++ end += start ++ buildID := string(header[start:end]) ++ if buildID == "" { ++ return nil ++ } ++ parts := strings.Split(buildID, "/") ++ if len(parts) != 2 || len(parts[0]) != len(parts[1]) { ++ return fmt.Errorf("unexpected Go archive build ID %q", buildID) ++ } ++ stableBuildID := parts[1] + "/" + parts[1] ++ for offset := 0; ; { ++ index := bytes.Index(data[offset:], []byte(buildID)) ++ if index < 0 { ++ break ++ } ++ index += offset ++ copy(data[index:index+len(buildID)], stableBuildID) ++ offset = index + len(buildID) ++ } ++ return nil ++} ++ +// reusableSharedSyntheticTestmainHelperBundle returns the declared shared +// bundle only when it covers the requested roots and does not overlap archives +// supplied by the current Bazel target. Overlap falls back to the existing @@ -6884,7 +6969,7 @@ index a1782d4..8004cd0 100644 + stem := fmt.Sprintf("%x", sha256.Sum256([]byte(meta.ImportPath)))[:16] + outLinkobjPath := filepath.Join(archiveDir, stem+".a") + outInterfacePath := filepath.Join(archiveDir, stem+".iface.a") -+ trimPath, err := createTrimPath() ++ trimPath, err := syntheticSourceTrimPath(meta.Dir, meta.ImportPath) + if err != nil { + return "", "", fmt.Errorf("create trimpath for %s: %w", pkg, err) + } @@ -6927,7 +7012,7 @@ index a1782d4..8004cd0 100644 + } + for i, sSrc := range filteredSrcs.sSrcs { + obj := filepath.Join(packageWorkDir, fmt.Sprintf("s%d.o", i)) -+ if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, obj); err != nil { ++ if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, trimPath, obj); err != nil { + return "", "", fmt.Errorf("assemble synthetic helper %s: %w", pkg, err) + } + objFiles = append(objFiles, obj) @@ -6955,6 +7040,21 @@ index a1782d4..8004cd0 100644 + return result.compilePath, result.linkPath, nil +} + ++// syntheticSourceTrimPath gives module sources a stable logical prefix. The ++// physical GOMODCACHE belongs to the action sandbox and must not reach archives ++// that Bazel may publish to a shared cache. ++func syntheticSourceTrimPath(sourceDir, importPath string) (string, error) { ++ trimPath, err := createTrimPath() ++ if err != nil { ++ return "", err ++ } ++ sourceDir = strings.TrimSpace(sourceDir) ++ if sourceDir == "" { ++ return trimPath, nil ++ } ++ return trimPath + ";" + abs(sourceDir) + "=>" + strings.TrimSpace(importPath), nil ++} ++ +func shouldSkipOrchestrionForImportPath(importPath string) bool { + if strings.HasPrefix(importPath, "github.com/bazelbuild/rules_go/go/tools/") { + return true @@ -7213,10 +7313,10 @@ index a1782d4..8004cd0 100644 +} diff --git a/go/tools/builders/compilepkg_test.go b/go/tools/builders/compilepkg_test.go new file mode 100644 -index 0000000..320fb66 +index 0000000..91065d6 --- /dev/null +++ b/go/tools/builders/compilepkg_test.go -@@ -0,0 +1,837 @@ +@@ -0,0 +1,915 @@ +package main + +import ( @@ -7359,6 +7459,51 @@ index 0000000..320fb66 + } +} + ++func TestSharedSyntheticTestmainHelperBundleNormalizesGoBuildIDs(t *testing.T) { ++ const contentID = "cccccccccccccccccccc" ++ archive := func(actionID string) []byte { ++ buildID := actionID + "/" + contentID ++ return []byte("!\narchive header\ngo object test\nbuild id \"" + buildID + "\"\npayload build id \"" + buildID + "\"\n") ++ } ++ ++ sourceDir := t.TempDir() ++ firstSource := filepath.Join(sourceDir, "first.a") ++ secondSource := filepath.Join(sourceDir, "second.a") ++ if err := os.WriteFile(firstSource, archive("aaaaaaaaaaaaaaaaaaaa"), 0o644); err != nil { ++ t.Fatal(err) ++ } ++ if err := os.WriteFile(secondSource, archive("bbbbbbbbbbbbbbbbbbbb"), 0o644); err != nil { ++ t.Fatal(err) ++ } ++ ++ bundleDir := filepath.Join(t.TempDir(), "shared-helpers") ++ compiled := map[string]compiledModuleArchive{ ++ "example.com/first": {compilePath: firstSource, linkPath: firstSource}, ++ "example.com/second": {compilePath: secondSource, linkPath: secondSource}, ++ } ++ if err := writeSharedSyntheticTestmainHelperBundle(bundleDir, compiled, orchestrionModeTestOptimization); err != nil { ++ t.Fatal(err) ++ } ++ ++ manifest, err := readSyntheticTestmainHelperManifest(bundleDir) ++ if err != nil { ++ t.Fatal(err) ++ } ++ firstPath := manifest.Packages["example.com/first"].CompilePath ++ secondPath := manifest.Packages["example.com/second"].CompilePath ++ if firstPath != secondPath { ++ t.Fatalf("equivalent archives published at different paths: %q != %q", firstPath, secondPath) ++ } ++ published, err := os.ReadFile(filepath.Join(bundleDir, firstPath)) ++ if err != nil { ++ t.Fatal(err) ++ } ++ wantBuildID := contentID + "/" + contentID ++ if got := strings.Count(string(published), wantBuildID); got != 2 { ++ t.Fatalf("normalized build ID occurs %d times, want 2", got) ++ } ++} ++ +func TestSharedSyntheticTestmainHelperBundleFallsBackOnTargetOverlap(t *testing.T) { + rootPackage := syntheticTestmainRootPackagesTestOptimization[0].packagePath + sourceDir := t.TempDir() @@ -7758,6 +7903,39 @@ index 0000000..320fb66 + } +} + ++func TestSyntheticModuleGoFlagsRequireTrimpath(t *testing.T) { ++ tests := []struct { ++ name string ++ flags string ++ want string ++ }{ ++ {name: "empty", want: "-mod=mod -trimpath"}, ++ {name: "preserve module mode", flags: "-mod=readonly", want: "-mod=readonly -trimpath"}, ++ {name: "preserve trimpath", flags: "-trimpath -mod=vendor", want: "-trimpath -mod=vendor"}, ++ {name: "override disabled trimpath", flags: "-trimpath=false", want: "-mod=mod -trimpath=false -trimpath"}, ++ } ++ for _, test := range tests { ++ t.Run(test.name, func(t *testing.T) { ++ env := ensureSyntheticModuleGoFlags([]string{"GOFLAGS=" + test.flags}) ++ if got := getEnv(env, "GOFLAGS"); got != test.want { ++ t.Fatalf("GOFLAGS = %q, want %q", got, test.want) ++ } ++ }) ++ } ++} ++ ++func TestSyntheticSourceTrimPathReplacesModuleCacheRoot(t *testing.T) { ++ sourceDir := filepath.Join(t.TempDir(), "pkg", "mod", "example.com", "helper@v1.0.0") ++ got, err := syntheticSourceTrimPath(sourceDir, "example.com/helper") ++ if err != nil { ++ t.Fatalf("syntheticSourceTrimPath error: %v", err) ++ } ++ want := abs(sourceDir) + "=>example.com/helper" ++ if !strings.Contains(got, want) { ++ t.Fatalf("syntheticSourceTrimPath = %q, want mapping %q", got, want) ++ } ++} ++ +// TestWithSyntheticTestmainModuleCacheEnvRestoresEnvironment verifies that the +// synthetic helper wrapper points subprocess cache env vars at the helper +// cache root during execution and restores the caller's environment after. @@ -8742,7 +8920,7 @@ index 893a0f6..16955a7 100644 + return "" +} diff --git a/go/tools/builders/importcfg.go b/go/tools/builders/importcfg.go -index c25763a..b6298e6 100644 +index c25763a..7c19d36 100644 --- a/go/tools/builders/importcfg.go +++ b/go/tools/builders/importcfg.go @@ -17,21 +17,94 @@ package main @@ -9345,7 +9523,7 @@ index c25763a..b6298e6 100644 + if err != nil { + return nil, fmt.Errorf("prepare module resolution env: %w", err) + } -+ cmd.Env = ensureGoFlagsModMode(cmd.Env) ++ cmd.Env = ensureSyntheticModuleGoFlags(cmd.Env) + if getEnv(cmd.Env, "HOME") == "" { + homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") + if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -12020,10 +12198,10 @@ index 11dc0ab..610af53 100644 if *buildmode == "c-archive" { diff --git a/go/tools/builders/module_proxy.go b/go/tools/builders/module_proxy.go new file mode 100644 -index 0000000..b44c0f3 +index 0000000..91b0ba4 --- /dev/null +++ b/go/tools/builders/module_proxy.go -@@ -0,0 +1,291 @@ +@@ -0,0 +1,298 @@ +package main + +import ( @@ -12109,15 +12287,22 @@ index 0000000..b44c0f3 + return env, nil +} + -+func ensureGoFlagsModMode(env []string) []string { ++// ensureSyntheticModuleGoFlags keeps helper module builds offline-compatible ++// and removes their temporary module-cache roots from compiled archives. ++func ensureSyntheticModuleGoFlags(env []string) []string { + goFlags := strings.TrimSpace(getEnv(env, "GOFLAGS")) -+ if strings.Contains(goFlags, "-mod=") { -+ return env ++ if !strings.Contains(goFlags, "-mod=") { ++ goFlags = strings.TrimSpace("-mod=mod " + goFlags) + } -+ if goFlags == "" { -+ goFlags = "-mod=mod" -+ } else { -+ goFlags = "-mod=mod " + goFlags ++ trimPathEnabled := false ++ for _, flag := range strings.Fields(goFlags) { ++ if flag == "-trimpath" || flag == "-trimpath=true" { ++ trimPathEnabled = true ++ break ++ } ++ } ++ if !trimPathEnabled { ++ goFlags += " -trimpath" + } + return setEnv(env, "GOFLAGS", goFlags) +} @@ -14288,10 +14473,10 @@ index 0000000..fbb5712 +} diff --git a/go/tools/builders/orchestrion_skip_test.go b/go/tools/builders/orchestrion_skip_test.go new file mode 100644 -index 0000000..32c0cf4 +index 0000000..00d37c1 --- /dev/null +++ b/go/tools/builders/orchestrion_skip_test.go -@@ -0,0 +1,383 @@ +@@ -0,0 +1,392 @@ +package main + +import ( @@ -14588,6 +14773,14 @@ index 0000000..32c0cf4 + +func TestSyntheticTestmainManifestPublishesExternalArchives(t *testing.T) { + execroot := t.TempDir() ++ previousDir, err := os.Getwd() ++ if err != nil { ++ t.Fatal(err) ++ } ++ if err := os.Chdir(execroot); err != nil { ++ t.Fatal(err) ++ } ++ t.Cleanup(func() { _ = os.Chdir(previousDir) }) + externalArchive := filepath.Join(t.TempDir(), "helper.a") + if err := os.WriteFile(externalArchive, []byte("helper archive"), 0o644); err != nil { + t.Fatal(err) @@ -14630,7 +14823,8 @@ index 0000000..32c0cf4 + t.Fatalf("published helper archive count = %d, want 1", len(entries)) + } + publishedArchive := filepath.Join(helperOutputDir, entries[0].Name()) -+ if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedArchive) { ++ publishedRelative := execrootRelativePath(publishedArchive) ++ if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedRelative) { + t.Fatalf("manifest does not reference published helper %q:\n%s", publishedArchive, gotManifest) + } + publishedData, err := os.ReadFile(publishedArchive) @@ -16559,7 +16753,7 @@ index 0000000..9edae16 + } +} diff --git a/go/tools/builders/stdlib.go b/go/tools/builders/stdlib.go -index cfcb991..65244aa 100644 +index cfcb991..8ea6331 100644 --- a/go/tools/builders/stdlib.go +++ b/go/tools/builders/stdlib.go @@ -21,15 +21,29 @@ import ( @@ -16681,7 +16875,7 @@ index cfcb991..65244aa 100644 // Build the commands needed to build the std library in the right mode // NOTE: the go command stamps compiled .a files with build ids, which are -@@ -121,7 +164,112 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -121,7 +164,117 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) // creating reproducible builds because the build ids are hashed from // CGO_CFLAGS, which frequently contains absolute paths. As a workaround, // we strip the build ids, since they won't be used after this. @@ -16792,10 +16986,15 @@ index cfcb991..65244aa 100644 + } + } + installArgs := goenv.goCmd("install", "-toolexec", toolexec) ++ if *orchestrion != "" { ++ // Helper module exports use Go's reproducible build mode. Compile the ++ // woven stdlib with the same flag so imported package fingerprints match. ++ installArgs = append(installArgs, "-trimpath") ++ } if len(build.Default.BuildTags) > 0 { installArgs = append(installArgs, "-tags", strings.Join(build.Default.BuildTags, ",")) } -@@ -135,7 +283,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -135,7 +288,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) installArgs = append(installArgs, "-msan") } if *pgoprofile != "" { @@ -16804,7 +17003,7 @@ index cfcb991..65244aa 100644 } if *shared { gcflags = append(gcflags, "-shared") -@@ -167,8 +315,393 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -167,8 +320,393 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } installArgs = append(installArgs, packages...) diff --git a/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch index d9079f2e..d11f3cb9 100644 --- a/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch @@ -3625,6 +3625,33 @@ index d2de6b9..ffbf778 100644 + } + return "." +} +diff --git a/go/tools/builders/asm.go b/go/tools/builders/asm.go +index c1b7997..3c8cf89 100644 +--- a/go/tools/builders/asm.go ++++ b/go/tools/builders/asm.go +@@ -87,16 +87,19 @@ func buildSymabisFile(goenv *env, packagePath string, sFiles, hFiles []fileInfo, + return symabisName, err + } + +-func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, outPath string) error { ++func asmFile(goenv *env, srcPath, packagePath string, asmFlags []string, trimPath, outPath string) error { + args := goenv.goTool("asm") + args = append(args, asmFlags...) + if packagePath != "" { + args = append(args, "-p", packagePath) + } + args = append(args, ASM_DEFINES...) +- args = append(args, "-trimpath", ".") ++ if trimPath == "" { ++ trimPath = abs(".") ++ } ++ args = append(args, "-trimpath", trimPath) + args = append(args, "-o", outPath) + args = append(args, "--", srcPath) +- absArgs(args, []string{"-I", "-o", "-trimpath"}) ++ absArgs(args, []string{"-I", "-o"}) + return goenv.runCommand(args) + } diff --git a/go/tools/builders/builder.go b/go/tools/builders/builder.go index fdeda2d..76d69b6 100644 --- a/go/tools/builders/builder.go @@ -3776,7 +3803,7 @@ index 6f67914..5451c67 100644 transformArgs(normalized, cgoAbsEnvFlags, func(s string) string { if strings.HasPrefix(s, cgoAbsPlaceholder) { diff --git a/go/tools/builders/compilepkg.go b/go/tools/builders/compilepkg.go -index 98963c6..3eef445 100644 +index 98963c6..b64ae96 100644 --- a/go/tools/builders/compilepkg.go +++ b/go/tools/builders/compilepkg.go @@ -17,17 +17,252 @@ @@ -3976,7 +4003,7 @@ index 98963c6..3eef445 100644 + } + published[publishedPath] = true + } -+ lines[index] = "packagefile " + packagePath + "=" + publishedPath ++ lines[index] = "packagefile " + packagePath + "=" + execrootRelativePath(publishedPath) + } + return []byte(strings.Join(lines, "\n")), nil +} @@ -4228,7 +4255,16 @@ index 98963c6..3eef445 100644 // Compile the .s files with Go's assembler, if this is not a cgo package. // Cgo is assembled by cc above. -@@ -487,100 +791,1895 @@ func compileArchive( +@@ -464,7 +768,7 @@ func compileArchive( + } + for i, sSrc := range srcs.sSrcs { + obj := filepath.Join(workDir, fmt.Sprintf("s%d.o", i)) +- if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, obj); err != nil { ++ if err := asmFile(goenv, sSrc.filename, packagePath, asmFlags, "", obj); err != nil { + return err + } + objFiles = append(objFiles, obj) +@@ -487,100 +791,1957 @@ func compileArchive( return nil } @@ -4860,7 +4896,7 @@ index 98963c6..3eef445 100644 + env = setEnv(env, "GIT_CONFIG_GLOBAL", os.DevNull) + env = setEnv(env, "GIT_CONFIG_NOSYSTEM", "1") + env = setEnv(env, "GIT_TERMINAL_PROMPT", "0") -+ env = ensureGoFlagsModMode(env) ++ env = ensureSyntheticModuleGoFlags(env) + if getEnv(env, "HOME") == "" { + homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") + if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -5430,13 +5466,17 @@ index 98963c6..3eef445 100644 + if published := publishedPaths[source]; published != "" { + return published, nil + } -+ digest, err := fullDigestFile(source) ++ data, err := os.ReadFile(source) + if err != nil { -+ return "", fmt.Errorf("digest shared synthetic helper archive %s: %w", source, err) ++ return "", fmt.Errorf("read shared synthetic helper archive %s: %w", source, err) ++ } ++ if err := normalizePublishedGoArchiveBuildID(data); err != nil { ++ return "", fmt.Errorf("normalize shared synthetic helper archive %s: %w", source, err) + } ++ digest := fmt.Sprintf("%x", sha256.Sum256(data)) + destination := filepath.Join(outputDir, "archives", digest+".a") + if _, err := os.Stat(destination); errors.Is(err, os.ErrNotExist) { -+ if err := copyArchiveFile(source, destination); err != nil { ++ if err := os.WriteFile(destination, data, 0o644); err != nil { + return "", fmt.Errorf("publish shared synthetic helper archive %s: %w", source, err) + } + } else if err != nil { @@ -5477,6 +5517,49 @@ index 98963c6..3eef445 100644 + return writeJSONAtomically(filepath.Join(outputDir, cacheManifestFileName), manifest) +} + ++// normalizePublishedGoArchiveBuildID removes execroot-specific action IDs from ++// a copied helper archive. The Go cache keeps the original archive and build ID; ++// only the declared Bazel output receives the stable content/content form. ++func normalizePublishedGoArchiveBuildID(data []byte) error { ++ if !bytes.HasPrefix(data, []byte("!\n")) { ++ return nil ++ } ++ header := data ++ if len(header) > 1024 { ++ header = header[:1024] ++ } ++ const marker = "\nbuild id \"" ++ start := bytes.Index(header, []byte(marker)) ++ if start < 0 { ++ return nil ++ } ++ start += len(marker) ++ end := bytes.IndexByte(header[start:], '"') ++ if end < 0 { ++ return errors.New("unterminated Go archive build ID") ++ } ++ end += start ++ buildID := string(header[start:end]) ++ if buildID == "" { ++ return nil ++ } ++ parts := strings.Split(buildID, "/") ++ if len(parts) != 2 || len(parts[0]) != len(parts[1]) { ++ return fmt.Errorf("unexpected Go archive build ID %q", buildID) ++ } ++ stableBuildID := parts[1] + "/" + parts[1] ++ for offset := 0; ; { ++ index := bytes.Index(data[offset:], []byte(buildID)) ++ if index < 0 { ++ break ++ } ++ index += offset ++ copy(data[index:index+len(buildID)], stableBuildID) ++ offset = index + len(buildID) ++ } ++ return nil ++} ++ +// reusableSharedSyntheticTestmainHelperBundle returns the declared shared +// bundle only when it covers the requested roots and does not overlap archives +// supplied by the current Bazel target. Overlap falls back to the existing @@ -5865,7 +5948,7 @@ index 98963c6..3eef445 100644 + stem := fmt.Sprintf("%x", sha256.Sum256([]byte(meta.ImportPath)))[:16] + outLinkobjPath := filepath.Join(archiveDir, stem+".a") + outInterfacePath := filepath.Join(archiveDir, stem+".iface.a") -+ trimPath, err := createTrimPath() ++ trimPath, err := syntheticSourceTrimPath(meta.Dir, meta.ImportPath) + if err != nil { + return "", "", fmt.Errorf("create trimpath for %s: %w", pkg, err) + } @@ -5908,7 +5991,7 @@ index 98963c6..3eef445 100644 + } + for i, sSrc := range filteredSrcs.sSrcs { + obj := filepath.Join(packageWorkDir, fmt.Sprintf("s%d.o", i)) -+ if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, obj); err != nil { ++ if err := asmFile(goenv, sSrc.filename, meta.ImportPath, asmFlags, trimPath, obj); err != nil { + return "", "", fmt.Errorf("assemble synthetic helper %s: %w", pkg, err) + } + objFiles = append(objFiles, obj) @@ -5936,6 +6019,21 @@ index 98963c6..3eef445 100644 + return result.compilePath, result.linkPath, nil +} + ++// syntheticSourceTrimPath gives module sources a stable logical prefix. The ++// physical GOMODCACHE belongs to the action sandbox and must not reach archives ++// that Bazel may publish to a shared cache. ++func syntheticSourceTrimPath(sourceDir, importPath string) (string, error) { ++ trimPath, err := createTrimPath() ++ if err != nil { ++ return "", err ++ } ++ sourceDir = strings.TrimSpace(sourceDir) ++ if sourceDir == "" { ++ return trimPath, nil ++ } ++ return trimPath + ";" + abs(sourceDir) + "=>" + strings.TrimSpace(importPath), nil ++} ++ +func shouldSkipOrchestrionForImportPath(importPath string) bool { + if strings.HasPrefix(importPath, "github.com/bazelbuild/rules_go/go/tools/") { + return true @@ -6194,10 +6292,10 @@ index 98963c6..3eef445 100644 +} diff --git a/go/tools/builders/compilepkg_test.go b/go/tools/builders/compilepkg_test.go new file mode 100644 -index 0000000..320fb66 +index 0000000..91065d6 --- /dev/null +++ b/go/tools/builders/compilepkg_test.go -@@ -0,0 +1,837 @@ +@@ -0,0 +1,915 @@ +package main + +import ( @@ -6340,6 +6438,51 @@ index 0000000..320fb66 + } +} + ++func TestSharedSyntheticTestmainHelperBundleNormalizesGoBuildIDs(t *testing.T) { ++ const contentID = "cccccccccccccccccccc" ++ archive := func(actionID string) []byte { ++ buildID := actionID + "/" + contentID ++ return []byte("!\narchive header\ngo object test\nbuild id \"" + buildID + "\"\npayload build id \"" + buildID + "\"\n") ++ } ++ ++ sourceDir := t.TempDir() ++ firstSource := filepath.Join(sourceDir, "first.a") ++ secondSource := filepath.Join(sourceDir, "second.a") ++ if err := os.WriteFile(firstSource, archive("aaaaaaaaaaaaaaaaaaaa"), 0o644); err != nil { ++ t.Fatal(err) ++ } ++ if err := os.WriteFile(secondSource, archive("bbbbbbbbbbbbbbbbbbbb"), 0o644); err != nil { ++ t.Fatal(err) ++ } ++ ++ bundleDir := filepath.Join(t.TempDir(), "shared-helpers") ++ compiled := map[string]compiledModuleArchive{ ++ "example.com/first": {compilePath: firstSource, linkPath: firstSource}, ++ "example.com/second": {compilePath: secondSource, linkPath: secondSource}, ++ } ++ if err := writeSharedSyntheticTestmainHelperBundle(bundleDir, compiled, orchestrionModeTestOptimization); err != nil { ++ t.Fatal(err) ++ } ++ ++ manifest, err := readSyntheticTestmainHelperManifest(bundleDir) ++ if err != nil { ++ t.Fatal(err) ++ } ++ firstPath := manifest.Packages["example.com/first"].CompilePath ++ secondPath := manifest.Packages["example.com/second"].CompilePath ++ if firstPath != secondPath { ++ t.Fatalf("equivalent archives published at different paths: %q != %q", firstPath, secondPath) ++ } ++ published, err := os.ReadFile(filepath.Join(bundleDir, firstPath)) ++ if err != nil { ++ t.Fatal(err) ++ } ++ wantBuildID := contentID + "/" + contentID ++ if got := strings.Count(string(published), wantBuildID); got != 2 { ++ t.Fatalf("normalized build ID occurs %d times, want 2", got) ++ } ++} ++ +func TestSharedSyntheticTestmainHelperBundleFallsBackOnTargetOverlap(t *testing.T) { + rootPackage := syntheticTestmainRootPackagesTestOptimization[0].packagePath + sourceDir := t.TempDir() @@ -6739,6 +6882,39 @@ index 0000000..320fb66 + } +} + ++func TestSyntheticModuleGoFlagsRequireTrimpath(t *testing.T) { ++ tests := []struct { ++ name string ++ flags string ++ want string ++ }{ ++ {name: "empty", want: "-mod=mod -trimpath"}, ++ {name: "preserve module mode", flags: "-mod=readonly", want: "-mod=readonly -trimpath"}, ++ {name: "preserve trimpath", flags: "-trimpath -mod=vendor", want: "-trimpath -mod=vendor"}, ++ {name: "override disabled trimpath", flags: "-trimpath=false", want: "-mod=mod -trimpath=false -trimpath"}, ++ } ++ for _, test := range tests { ++ t.Run(test.name, func(t *testing.T) { ++ env := ensureSyntheticModuleGoFlags([]string{"GOFLAGS=" + test.flags}) ++ if got := getEnv(env, "GOFLAGS"); got != test.want { ++ t.Fatalf("GOFLAGS = %q, want %q", got, test.want) ++ } ++ }) ++ } ++} ++ ++func TestSyntheticSourceTrimPathReplacesModuleCacheRoot(t *testing.T) { ++ sourceDir := filepath.Join(t.TempDir(), "pkg", "mod", "example.com", "helper@v1.0.0") ++ got, err := syntheticSourceTrimPath(sourceDir, "example.com/helper") ++ if err != nil { ++ t.Fatalf("syntheticSourceTrimPath error: %v", err) ++ } ++ want := abs(sourceDir) + "=>example.com/helper" ++ if !strings.Contains(got, want) { ++ t.Fatalf("syntheticSourceTrimPath = %q, want mapping %q", got, want) ++ } ++} ++ +// TestWithSyntheticTestmainModuleCacheEnvRestoresEnvironment verifies that the +// synthetic helper wrapper points subprocess cache env vars at the helper +// cache root during execution and restores the caller's environment after. @@ -7723,7 +7899,7 @@ index 893a0f6..16955a7 100644 + return "" +} diff --git a/go/tools/builders/importcfg.go b/go/tools/builders/importcfg.go -index 36c2329..4b0fa3f 100644 +index 36c2329..2fc994c 100644 --- a/go/tools/builders/importcfg.go +++ b/go/tools/builders/importcfg.go @@ -17,20 +17,93 @@ package main @@ -8325,7 +8501,7 @@ index 36c2329..4b0fa3f 100644 + if err != nil { + return nil, fmt.Errorf("prepare module resolution env: %w", err) + } -+ cmd.Env = ensureGoFlagsModMode(cmd.Env) ++ cmd.Env = ensureSyntheticModuleGoFlags(cmd.Env) + if getEnv(cmd.Env, "HOME") == "" { + homePath := filepath.Join(os.TempDir(), "datadog-orchestrion-home") + if err := os.MkdirAll(homePath, 0o755); err != nil { @@ -11013,10 +11189,10 @@ index 6564629..fab985a 100644 if *buildmode == "c-archive" { diff --git a/go/tools/builders/module_proxy.go b/go/tools/builders/module_proxy.go new file mode 100644 -index 0000000..b44c0f3 +index 0000000..91b0ba4 --- /dev/null +++ b/go/tools/builders/module_proxy.go -@@ -0,0 +1,291 @@ +@@ -0,0 +1,298 @@ +package main + +import ( @@ -11102,15 +11278,22 @@ index 0000000..b44c0f3 + return env, nil +} + -+func ensureGoFlagsModMode(env []string) []string { ++// ensureSyntheticModuleGoFlags keeps helper module builds offline-compatible ++// and removes their temporary module-cache roots from compiled archives. ++func ensureSyntheticModuleGoFlags(env []string) []string { + goFlags := strings.TrimSpace(getEnv(env, "GOFLAGS")) -+ if strings.Contains(goFlags, "-mod=") { -+ return env ++ if !strings.Contains(goFlags, "-mod=") { ++ goFlags = strings.TrimSpace("-mod=mod " + goFlags) + } -+ if goFlags == "" { -+ goFlags = "-mod=mod" -+ } else { -+ goFlags = "-mod=mod " + goFlags ++ trimPathEnabled := false ++ for _, flag := range strings.Fields(goFlags) { ++ if flag == "-trimpath" || flag == "-trimpath=true" { ++ trimPathEnabled = true ++ break ++ } ++ } ++ if !trimPathEnabled { ++ goFlags += " -trimpath" + } + return setEnv(env, "GOFLAGS", goFlags) +} @@ -13281,10 +13464,10 @@ index 0000000..fbb5712 +} diff --git a/go/tools/builders/orchestrion_skip_test.go b/go/tools/builders/orchestrion_skip_test.go new file mode 100644 -index 0000000..32c0cf4 +index 0000000..00d37c1 --- /dev/null +++ b/go/tools/builders/orchestrion_skip_test.go -@@ -0,0 +1,383 @@ +@@ -0,0 +1,392 @@ +package main + +import ( @@ -13581,6 +13764,14 @@ index 0000000..32c0cf4 + +func TestSyntheticTestmainManifestPublishesExternalArchives(t *testing.T) { + execroot := t.TempDir() ++ previousDir, err := os.Getwd() ++ if err != nil { ++ t.Fatal(err) ++ } ++ if err := os.Chdir(execroot); err != nil { ++ t.Fatal(err) ++ } ++ t.Cleanup(func() { _ = os.Chdir(previousDir) }) + externalArchive := filepath.Join(t.TempDir(), "helper.a") + if err := os.WriteFile(externalArchive, []byte("helper archive"), 0o644); err != nil { + t.Fatal(err) @@ -13623,7 +13814,8 @@ index 0000000..32c0cf4 + t.Fatalf("published helper archive count = %d, want 1", len(entries)) + } + publishedArchive := filepath.Join(helperOutputDir, entries[0].Name()) -+ if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedArchive) { ++ publishedRelative := execrootRelativePath(publishedArchive) ++ if !strings.Contains(gotManifest, "packagefile example.com/helper="+publishedRelative) { + t.Fatalf("manifest does not reference published helper %q:\n%s", publishedArchive, gotManifest) + } + publishedData, err := os.ReadFile(publishedArchive) @@ -15552,7 +15744,7 @@ index 0000000..9edae16 + } +} diff --git a/go/tools/builders/stdlib.go b/go/tools/builders/stdlib.go -index fa9c01c..65791a4 100644 +index fa9c01c..cad3e0b 100644 --- a/go/tools/builders/stdlib.go +++ b/go/tools/builders/stdlib.go @@ -21,15 +21,29 @@ import ( @@ -15673,7 +15865,7 @@ index fa9c01c..65791a4 100644 // Build the commands needed to build the std library in the right mode // NOTE: the go command stamps compiled .a files with build ids, which are -@@ -120,7 +164,112 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -120,7 +164,117 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) // creating reproducible builds because the build ids are hashed from // CGO_CFLAGS, which frequently contains absolute paths. As a workaround, // we strip the build ids, since they won't be used after this. @@ -15784,10 +15976,15 @@ index fa9c01c..65791a4 100644 + } + } + installArgs := goenv.goCmd("install", "-toolexec", toolexec) ++ if *orchestrion != "" { ++ // Helper module exports use Go's reproducible build mode. Compile the ++ // woven stdlib with the same flag so imported package fingerprints match. ++ installArgs = append(installArgs, "-trimpath") ++ } if len(build.Default.BuildTags) > 0 { installArgs = append(installArgs, "-tags", strings.Join(build.Default.BuildTags, ",")) } -@@ -134,7 +283,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -134,7 +288,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) installArgs = append(installArgs, "-msan") } if *pgoprofile != "" { @@ -15796,7 +15993,7 @@ index fa9c01c..65791a4 100644 } if *shared { gcflags = append(gcflags, "-shared") -@@ -156,8 +305,393 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) +@@ -156,8 +310,393 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) } installArgs = append(installArgs, packages...) diff --git a/third_party/rules_go_orchestrion/profiles/workspace_runtime.json b/third_party/rules_go_orchestrion/profiles/workspace_runtime.json index 36cb04f1..14133099 100644 --- a/third_party/rules_go_orchestrion/profiles/workspace_runtime.json +++ b/third_party/rules_go_orchestrion/profiles/workspace_runtime.json @@ -21,6 +21,7 @@ "go/private/rules/transition.bzl", "go/tools/builders/BUILD.bazel", "go/tools/builders/ar.go", + "go/tools/builders/asm.go", "go/tools/builders/builder.go", "go/tools/builders/cc.go", "go/tools/builders/compilepkg.go", diff --git a/tools/dev/verify_rules_go_profiles.py b/tools/dev/verify_rules_go_profiles.py index ce89db92..f2ff0dfb 100644 --- a/tools/dev/verify_rules_go_profiles.py +++ b/tools/dev/verify_rules_go_profiles.py @@ -194,7 +194,7 @@ def verify_workspace_runtime_functional_smoke( target="//app:hello_test", private_safe_patterns=private_safe_patterns, ) - first_orchestrion = run_stdlib_inventory( + first_orchestrion = run_orchestrion_reproducibility_snapshot( bazel, bazel_output_user_root, workspace, @@ -222,7 +222,7 @@ def verify_workspace_runtime_functional_smoke( target="//app:hello_test", private_safe_patterns=private_safe_patterns, ) - second_orchestrion = run_stdlib_inventory( + second_orchestrion = run_orchestrion_reproducibility_snapshot( bazel, replay_output_user_root, workspace, @@ -233,19 +233,44 @@ def verify_workspace_runtime_functional_smoke( ) assert_plain_stdlib_cache(first_plain, patch) assert_plain_stdlib_cache(second_plain, patch) - assert_orchestrion_stdlib_cache(first_orchestrion, patch) - assert_orchestrion_stdlib_cache(second_orchestrion, patch) + assert_orchestrion_stdlib_cache(first_orchestrion.stdlib_cache, patch) + assert_orchestrion_stdlib_cache(second_orchestrion.stdlib_cache, patch) if first_plain != second_plain: raise ValueError( "plain stdlib cache inventories differ for %s: %s" % (patch, describe_snapshot_difference(first_plain, second_plain)) ) - if first_orchestrion != second_orchestrion: + if first_orchestrion.stdlib_cache != second_orchestrion.stdlib_cache: raise ValueError( "Test Optimization stdlib cache inventories differ for %s: %s" % ( patch, - describe_snapshot_difference(first_orchestrion, second_orchestrion), + describe_snapshot_difference( + first_orchestrion.stdlib_cache, + second_orchestrion.stdlib_cache, + ), + ) + ) + if first_orchestrion.action_keys != second_orchestrion.action_keys: + raise ValueError( + "Test Optimization action keys differ for %s: %s" + % ( + patch, + describe_mapping_difference( + first_orchestrion.action_keys, + second_orchestrion.action_keys, + ), + ) + ) + if first_orchestrion.outputs != second_orchestrion.outputs: + raise ValueError( + "Test Optimization action outputs differ for %s: %s" + % ( + patch, + describe_mapping_difference( + first_orchestrion.outputs, + second_orchestrion.outputs, + ), ) ) @@ -265,6 +290,19 @@ def describe_snapshot_difference( return ", ".join(differences) or "snapshot metadata differs" +def describe_mapping_difference(first: dict[str, str], second: dict[str, str]) -> str: + """Describe a bounded set of differing action keys or output digests.""" + differences = [] + for key in sorted(set(first) | set(second)): + first_value = first.get(key, "missing") + second_value = second.get(key, "missing") + if first_value != second_value: + differences.append("%s=(%s != %s)" % (key, first_value, second_value)) + if len(differences) > 20: + differences = [*differences[:20], "... and %d more" % (len(differences) - 20)] + return ", ".join(differences) or "snapshot metadata differs" + + def write_smoke_workspace( *, workspace: Path, @@ -392,6 +430,232 @@ class StdlibCacheSnapshot: manifest: str | None +@dataclass(frozen=True) +class ReproducibilitySnapshot: + """Cache contents, action keys, and outputs from one instrumented build.""" + + stdlib_cache: StdlibCacheSnapshot + action_keys: dict[str, str] + outputs: dict[str, str] + + +def run_orchestrion_reproducibility_snapshot( + bazel: Path, + output_user_root: Path, + workspace: Path, + *, + command: str, + mode_flags: list[str], + target: str, + private_safe_patterns: list[str] | None = None, +) -> ReproducibilitySnapshot: + """Build once and capture the cache-critical Test Optimization actions.""" + stdlib_cache = run_stdlib_inventory( + bazel, + output_user_root, + workspace, + command=command, + mode_flags=mode_flags, + target=target, + private_safe_patterns=private_safe_patterns, + ) + aquery = run_bazel( + bazel, + output_user_root, + workspace, + [ + "aquery", + *mode_flags, + "--output=jsonproto", + 'mnemonic("(GoStdlib|GoSyntheticTestmainHelpers|GoCompilePkg|GoLink)", deps(%s))' + % target, + ], + private_safe_patterns=private_safe_patterns, + ) + action_keys, outputs = action_snapshot_from_aquery( + json.loads(aquery.stdout), + workspace=workspace, + target_label=target, + ) + required = { + "GoCompilePkg", + "GoLink", + "GoStdlib", + "GoSyntheticTestmainHelpers", + } + found = {identity.split(" ", 1)[0] for identity in action_keys} + missing = sorted(required - found) + if missing: + raise ValueError( + "reproducibility aquery for %s is missing actions: %s" + % (target, ", ".join(missing)) + ) + return ReproducibilitySnapshot( + stdlib_cache=stdlib_cache, + action_keys=action_keys, + outputs=outputs, + ) + + +def action_snapshot_from_aquery( + data: dict[str, object], + *, + workspace: Path, + target_label: str, +) -> tuple[dict[str, str], dict[str, str]]: + """Return stable identities, action keys, and output digests from aquery JSON.""" + path_fragments = { + int(fragment["id"]): fragment + for fragment in data.get("pathFragments", []) + } + artifacts = { + int(artifact["id"]): resolve_path_fragment( + int(artifact["pathFragmentId"]), path_fragments + ) + for artifact in data.get("artifacts", []) + } + targets = { + int(target["id"]): str(target.get("label", "")) + for target in data.get("targets", []) + } + configurations = { + int(configuration["id"]): str(configuration.get("mnemonic", "")) + for configuration in data.get("configuration", []) + } + + action_keys: dict[str, str] = {} + outputs: dict[str, str] = {} + for action in data.get("actions", []): + mnemonic = str(action.get("mnemonic", "")) + output_paths = sorted( + artifacts[int(output_id)] for output_id in action.get("outputIds", []) + ) + action_target = targets.get(int(action.get("targetId", 0)), "") + if not is_reproducibility_action( + mnemonic=mnemonic, + target=action_target, + outputs=output_paths, + requested_target=target_label, + ): + continue + + configuration = configurations.get(int(action.get("configurationId", 0)), "") + identity = "%s %s [%s] -> %s" % ( + mnemonic, + action_target, + configuration, + ",".join(output_paths), + ) + if identity in action_keys: + raise ValueError("duplicate reproducibility action identity: %s" % identity) + action_keys[identity] = str(action.get("actionKey", "")) + if not action_keys[identity]: + raise ValueError("reproducibility action has no action key: %s" % identity) + for output_path in output_paths: + for relative, digest in canonical_artifact_inventory( + workspace / output_path + ).items(): + identity = output_path if not relative else output_path + "/" + relative + if identity in outputs: + raise ValueError("duplicate reproducibility output: %s" % identity) + outputs[identity] = digest + return action_keys, outputs + + +def resolve_path_fragment( + fragment_id: int, fragments: dict[int, dict[str, object]] +) -> str: + """Resolve one aquery path-fragment chain without host path assumptions.""" + labels = [] + seen = set() + while fragment_id: + if fragment_id in seen: + raise ValueError("cycle in aquery path fragments at id %d" % fragment_id) + seen.add(fragment_id) + fragment = fragments.get(fragment_id) + if fragment is None: + raise ValueError("unknown aquery path fragment id %d" % fragment_id) + labels.append(str(fragment.get("label", ""))) + fragment_id = int(fragment.get("parentId", 0)) + return PurePosixPath(*reversed(labels)).as_posix() + + +def is_reproducibility_action( + *, + mnemonic: str, + target: str, + outputs: list[str], + requested_target: str, +) -> bool: + """Select the four actions whose stability controls instrumented test caching.""" + if mnemonic in {"GoStdlib", "GoSyntheticTestmainHelpers"}: + return True + if target != requested_target: + return False + if mnemonic == "GoCompilePkg": + return any("~testmain.a" in output for output in outputs) + return mnemonic == "GoLink" + + +def canonical_artifact_digest(path: Path) -> str: + """Hash a declared file, symlink, or TreeArtifact including logical paths.""" + digest = hashlib.sha256() + if path.is_symlink(): + digest.update(b"symlink\0") + digest.update(os.readlink(path).encode("utf-8")) + return digest.hexdigest() + if path.is_file(): + digest.update(b"file\0") + digest.update(b"executable\0" if path.stat().st_mode & 0o111 else b"regular\0") + update_digest_from_file(digest, path) + return digest.hexdigest() + if not path.is_dir(): + raise ValueError("declared action output does not exist: %s" % path) + + digest.update(b"tree\0") + for child in sorted(path.rglob("*")): + relative = child.relative_to(path).as_posix().encode("utf-8") + digest.update(relative) + digest.update(b"\0") + if child.is_symlink(): + digest.update(b"symlink\0") + digest.update(os.readlink(child).encode("utf-8")) + elif child.is_dir(): + digest.update(b"directory\0") + elif child.is_file(): + digest.update(b"file\0") + digest.update( + b"executable\0" if child.stat().st_mode & 0o111 else b"regular\0" + ) + update_digest_from_file(digest, child) + else: + raise ValueError("unsupported action output entry: %s" % child) + digest.update(b"\0") + return digest.hexdigest() + + +def canonical_artifact_inventory(path: Path) -> dict[str, str]: + """Expand a declared tree so failures identify the exact unstable entry.""" + if path.is_symlink() or not path.is_dir(): + return {"": canonical_artifact_digest(path)} + + inventory = {"": "tree"} + for child in sorted(path.rglob("*")): + relative = child.relative_to(path).as_posix() + if child.is_dir() and not child.is_symlink(): + inventory[relative] = "directory" + else: + inventory[relative] = canonical_artifact_digest(child) + return inventory + + +def update_digest_from_file(digest, path: Path) -> None: + """Hash a file without retaining large archives in memory.""" + with path.open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + + def run_stdlib_inventory( bazel: Path, output_user_root: Path, diff --git a/tools/tests/python/test_rules_go_profile_tools.py b/tools/tests/python/test_rules_go_profile_tools.py index 0e6a41b1..f1c64ce5 100644 --- a/tools/tests/python/test_rules_go_profile_tools.py +++ b/tools/tests/python/test_rules_go_profile_tools.py @@ -450,6 +450,113 @@ def test_stdlib_cache_snapshot_accepts_manifested_data_entries(self) -> None: "runtime/internal/sys=cc/runtime-internal-d\n", ) + def test_action_snapshot_covers_cache_critical_instrumented_actions(self) -> None: + """The replay snapshot covers stdlib, helpers, testmain compile, and link.""" + with tempfile.TemporaryDirectory() as raw_tmp: + workspace = Path(raw_tmp) + paths = { + 1: {"id": 1, "label": "bazel-out"}, + 2: {"id": 2, "label": "arm64-fastbuild", "parentId": 1}, + 3: {"id": 3, "label": "bin", "parentId": 2}, + 4: {"id": 4, "label": "external", "parentId": 3}, + 5: {"id": 5, "label": "io_bazel_rules_go", "parentId": 4}, + 6: {"id": 6, "label": "stdlib_", "parentId": 5}, + 7: {"id": 7, "label": "synthetic_helpers", "parentId": 5}, + 8: {"id": 8, "label": "app", "parentId": 3}, + 9: {"id": 9, "label": "hello_test~testmain.a", "parentId": 8}, + 10: {"id": 10, "label": "hello_test", "parentId": 8}, + 11: {"id": 11, "label": "library.a", "parentId": 8}, + } + artifacts = [ + {"id": 101, "pathFragmentId": 6}, + {"id": 102, "pathFragmentId": 7}, + {"id": 103, "pathFragmentId": 9}, + {"id": 104, "pathFragmentId": 10}, + {"id": 105, "pathFragmentId": 11}, + ] + for relative, content in { + "bazel-out/arm64-fastbuild/bin/external/io_bazel_rules_go/stdlib_/fmt.a": "stdlib", + "bazel-out/arm64-fastbuild/bin/external/io_bazel_rules_go/synthetic_helpers/testing.a": "helpers", + "bazel-out/arm64-fastbuild/bin/app/hello_test~testmain.a": "testmain", + "bazel-out/arm64-fastbuild/bin/app/hello_test": "binary", + "bazel-out/arm64-fastbuild/bin/app/library.a": "library", + }.items(): + _write(workspace / relative, content) + + action_keys, outputs = self.mod.action_snapshot_from_aquery( + { + "pathFragments": list(paths.values()), + "artifacts": artifacts, + "targets": [ + {"id": 1, "label": "@@io_bazel_rules_go//:stdlib"}, + {"id": 2, "label": "//app:hello_test"}, + ], + "configuration": [{"id": 1, "mnemonic": "arm64-fastbuild"}], + "actions": [ + { + "mnemonic": "GoStdlib", + "targetId": 1, + "configurationId": 1, + "outputIds": [101], + "actionKey": "stdlib-key", + }, + { + "mnemonic": "GoSyntheticTestmainHelpers", + "targetId": 1, + "configurationId": 1, + "outputIds": [102], + "actionKey": "helpers-key", + }, + { + "mnemonic": "GoCompilePkg", + "targetId": 2, + "configurationId": 1, + "outputIds": [103], + "actionKey": "testmain-key", + }, + { + "mnemonic": "GoCompilePkg", + "targetId": 2, + "configurationId": 1, + "outputIds": [105], + "actionKey": "ordinary-library-key", + }, + { + "mnemonic": "GoLink", + "targetId": 2, + "configurationId": 1, + "outputIds": [104], + "actionKey": "link-key", + }, + ], + }, + workspace=workspace, + target_label="//app:hello_test", + ) + + self.assertEqual(4, len(action_keys)) + self.assertEqual( + {"GoCompilePkg", "GoLink", "GoStdlib", "GoSyntheticTestmainHelpers"}, + {identity.split(" ", 1)[0] for identity in action_keys}, + ) + self.assertEqual(6, len(outputs)) + self.assertFalse(any("library.a" in path for path in outputs)) + + def test_action_output_digest_is_independent_of_its_root(self) -> None: + """Logical paths, modes, and bytes determine a declared output digest.""" + with tempfile.TemporaryDirectory() as raw_tmp: + root = Path(raw_tmp) + first = root / "first" + second = root / "second" + _write(first / "nested" / "archive.a", "same bytes") + _write(second / "nested" / "archive.a", "same bytes") + + first_digest = self.mod.canonical_artifact_digest(first) + self.assertEqual(first_digest, self.mod.canonical_artifact_digest(second)) + + _write(second / "nested" / "archive.a", "different bytes") + self.assertNotEqual(first_digest, self.mod.canonical_artifact_digest(second)) + def test_stdlib_cache_snapshot_rejects_unmanifested_entries(self) -> None: """Action indexes and other unmanifested files fail verification.""" with tempfile.TemporaryDirectory() as raw_tmp: From 16bfd879e179797fa3eebd847d3b4901b18c5190 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 9 Sep 2026 12:54:28 +0200 Subject: [PATCH 2/6] rules_go: normalize CGO random seeds --- third_party/rgo/v0_60_0/base.CHANGED_FILES.md | 5 +- .../base/go/tools/builders/BUILD.bazel | 5 +- .../rgo/v0_60_0/base/go/tools/builders/cc.go | 65 ++++++ .../v0_60_0/base/go/tools/builders/cc_test.go | 50 +++++ third_party/rgo/v0_61_1/base.CHANGED_FILES.md | 5 +- .../base/go/tools/builders/BUILD.bazel | 5 +- .../rgo/v0_61_1/base/go/tools/builders/cc.go | 65 ++++++ .../v0_61_1/base/go/tools/builders/cc_test.go | 50 +++++ third_party/rgo/v0_62_0/base.CHANGED_FILES.md | 5 +- .../base/go/tools/builders/BUILD.bazel | 5 +- .../rgo/v0_62_0/base/go/tools/builders/cc.go | 65 ++++++ .../v0_62_0/base/go/tools/builders/cc_test.go | 50 +++++ third_party/rgo/v0_63_0/base.CHANGED_FILES.md | 5 +- .../base/go/tools/builders/BUILD.bazel | 5 +- .../rgo/v0_63_0/base/go/tools/builders/cc.go | 65 ++++++ .../v0_63_0/base/go/tools/builders/cc_test.go | 50 +++++ .../v0_60_0/base/0001-full-delta.patch | 167 +++++++++++++-- .../v0_61_1/base/0001-full-delta.patch | 169 ++++++++++++++-- .../v0_62_0/base/0001-full-delta.patch | 169 ++++++++++++++-- .../v0_63_0/base/0001-full-delta.patch | 169 ++++++++++++++-- tools/dev/verify_rules_go_profiles.py | 191 ++++++++++++++++-- .../python/test_rules_go_profile_tools.py | 47 +++++ 22 files changed, 1327 insertions(+), 85 deletions(-) create mode 100644 third_party/rgo/v0_60_0/base/go/tools/builders/cc_test.go create mode 100644 third_party/rgo/v0_61_1/base/go/tools/builders/cc_test.go create mode 100644 third_party/rgo/v0_62_0/base/go/tools/builders/cc_test.go create mode 100644 third_party/rgo/v0_63_0/base/go/tools/builders/cc_test.go diff --git a/third_party/rgo/v0_60_0/base.CHANGED_FILES.md b/third_party/rgo/v0_60_0/base.CHANGED_FILES.md index bb1f80df..a6c49e71 100644 --- a/third_party/rgo/v0_60_0/base.CHANGED_FILES.md +++ b/third_party/rgo/v0_60_0/base.CHANGED_FILES.md @@ -12,9 +12,9 @@ This file is generated. Do not edit by hand. ## Summary -- Total changed paths: `56` +- Total changed paths: `57` - Modified files: `32` -- Added files: `24` +- Added files: `25` - Removed files: `0` ## Modified files @@ -58,6 +58,7 @@ This file is generated. Do not edit by hand. - `go/private/orchestrion/BUILD` - `go/private/orchestrion/extensions.bzl` - `go/private/orchestrion/pin_files.bzl` +- `go/tools/builders/cc_test.go` - `go/tools/builders/compilepkg_test.go` - `go/tools/builders/env_orchestrion.go` - `go/tools/builders/importcfg_test.go` diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/BUILD.bazel b/third_party/rgo/v0_60_0/base/go/tools/builders/BUILD.bazel index 6f57dc4d..f09b3364 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/BUILD.bazel +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/BUILD.bazel @@ -119,11 +119,12 @@ go_test( name = "orchestrion_test", size = "small", srcs = [ + "cc_test.go", "orchestrion_cache_test.go", "orchestrion_mode_test.go", "orchestrion_skip_test.go", - "orchestrion_test_helpers_test.go", "orchestrion_test.go", + "orchestrion_test_helpers_test.go", "stdlib_test.go", ":builder_srcs", ], @@ -190,12 +191,12 @@ go_source( "constants.go", "env.go", "flags.go", + "module_proxy.go", "nogo_fix.go", "nogo_main.go", "nogo_typeparams_go117.go", "nogo_typeparams_go118.go", "nolint.go", - "module_proxy.go", "orchestrion.go", "orchestrion_cache.go", "orchestrion_mode.go", diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/cc.go b/third_party/rgo/v0_60_0/base/go/tools/builders/cc.go index 5451c671..f7ba8b1c 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/cc.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/cc.go @@ -1,8 +1,11 @@ package main import ( + "crypto/sha256" + "encoding/hex" "errors" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -108,6 +111,11 @@ func cc(args []string) error { } return s }) + // The go command derives this seed from its action ID. That ID contains the + // absolute -toolexec path, so otherwise identical stdlib actions running in + // different Bazel sandboxes make the CGO archive outputs differ. Recompute + // the seed from the compiler command after replacing ephemeral roots. + normalizeCgoRandomSeed(normalized, ccroot) if runtime.GOOS == "windows" { cmd := exec.Command(normalized[0], normalized[1:]...) cmd.Stdout = os.Stdout @@ -117,3 +125,60 @@ func cc(args []string) error { return syscall.Exec(normalized[0], normalized, os.Environ()) } } + +const cgoRandomSeedPrefix = "-frandom-seed=" + +func normalizeCgoRandomSeed(args []string, execRoot string) { + hasRandomSeed := false + for _, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { + hasRandomSeed = true + break + } + } + if !hasRandomSeed { + return + } + + workRoot := goBuildWorkRoot(args) + digest := sha256.New() + for _, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { + continue + } + normalized := arg + if workRoot != "" { + normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") + } + if execRoot != "" { + normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") + } + _, _ = io.WriteString(digest, normalized) + _, _ = digest.Write([]byte{0}) + } + seed := cgoRandomSeedPrefix + hex.EncodeToString(digest.Sum(nil)) + for i, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { + args[i] = seed + } + } +} + +func goBuildWorkRoot(args []string) string { + for i := 0; i+1 < len(args); i++ { + if args[i] != "-o" && args[i] != "-c" { + continue + } + for dir := filepath.Dir(args[i+1]); ; dir = filepath.Dir(dir) { + base := filepath.Base(dir) + if strings.HasPrefix(base, "go-build") { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + } + } + return "" +} diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/cc_test.go b/third_party/rgo/v0_60_0/base/go/tools/builders/cc_test.go new file mode 100644 index 00000000..f570843c --- /dev/null +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/cc_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { + firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") + secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") + firstWorkRoot := filepath.Join("tmp", "go-build111") + secondWorkRoot := filepath.Join("tmp", "go-build222") + + first := []string{ + "clang", + "-I" + filepath.Join(firstExecRoot, "external", "sysroot", "include"), + "-frandom-seed=volatile-first", + "-o", filepath.Join(firstWorkRoot, "b001", "_x001.o"), + "-c", filepath.Join(firstWorkRoot, "b001", "_cgo_export.c"), + } + second := []string{ + "clang", + "-I" + filepath.Join(secondExecRoot, "external", "sysroot", "include"), + "-frandom-seed=volatile-second", + "-o", filepath.Join(secondWorkRoot, "b001", "_x001.o"), + "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), + } + + normalizeCgoRandomSeed(first, firstExecRoot) + normalizeCgoRandomSeed(second, secondExecRoot) + if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { + t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) + } + + second = append(second, "-DPROFILE=changed") + normalizeCgoRandomSeed(second, secondExecRoot) + if randomSeedArg(first) == randomSeedArg(second) { + t.Fatal("meaningful compiler arguments must affect the normalized seed") + } +} + +func randomSeedArg(args []string) string { + for _, arg := range args { + if strings.HasPrefix(arg, "-frandom-seed=") { + return arg + } + } + return "" +} diff --git a/third_party/rgo/v0_61_1/base.CHANGED_FILES.md b/third_party/rgo/v0_61_1/base.CHANGED_FILES.md index d5c85f66..bd6e9d62 100644 --- a/third_party/rgo/v0_61_1/base.CHANGED_FILES.md +++ b/third_party/rgo/v0_61_1/base.CHANGED_FILES.md @@ -12,9 +12,9 @@ This file is generated. Do not edit by hand. ## Summary -- Total changed paths: `56` +- Total changed paths: `57` - Modified files: `32` -- Added files: `24` +- Added files: `25` - Removed files: `0` ## Modified files @@ -58,6 +58,7 @@ This file is generated. Do not edit by hand. - `go/private/orchestrion/BUILD` - `go/private/orchestrion/extensions.bzl` - `go/private/orchestrion/pin_files.bzl` +- `go/tools/builders/cc_test.go` - `go/tools/builders/compilepkg_test.go` - `go/tools/builders/env_orchestrion.go` - `go/tools/builders/importcfg_test.go` diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/BUILD.bazel b/third_party/rgo/v0_61_1/base/go/tools/builders/BUILD.bazel index 9cb71185..07999310 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/BUILD.bazel +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/BUILD.bazel @@ -119,11 +119,12 @@ go_test( name = "orchestrion_test", size = "small", srcs = [ + "cc_test.go", "orchestrion_cache_test.go", "orchestrion_mode_test.go", "orchestrion_skip_test.go", - "orchestrion_test_helpers_test.go", "orchestrion_test.go", + "orchestrion_test_helpers_test.go", "stdlib_test.go", ":builder_srcs", ], @@ -205,6 +206,7 @@ go_source( "constants.go", "env.go", "flags.go", + "module_proxy.go", "nogo_fix.go", "nogo_goversions_go117.go", "nogo_goversions_go118.go", @@ -215,7 +217,6 @@ go_source( "nogo_typeparams_go118.go", "nogo_version.go", "nolint.go", - "module_proxy.go", "orchestrion.go", "orchestrion_cache.go", "orchestrion_mode.go", diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/cc.go b/third_party/rgo/v0_61_1/base/go/tools/builders/cc.go index 5451c671..f7ba8b1c 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/cc.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/cc.go @@ -1,8 +1,11 @@ package main import ( + "crypto/sha256" + "encoding/hex" "errors" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -108,6 +111,11 @@ func cc(args []string) error { } return s }) + // The go command derives this seed from its action ID. That ID contains the + // absolute -toolexec path, so otherwise identical stdlib actions running in + // different Bazel sandboxes make the CGO archive outputs differ. Recompute + // the seed from the compiler command after replacing ephemeral roots. + normalizeCgoRandomSeed(normalized, ccroot) if runtime.GOOS == "windows" { cmd := exec.Command(normalized[0], normalized[1:]...) cmd.Stdout = os.Stdout @@ -117,3 +125,60 @@ func cc(args []string) error { return syscall.Exec(normalized[0], normalized, os.Environ()) } } + +const cgoRandomSeedPrefix = "-frandom-seed=" + +func normalizeCgoRandomSeed(args []string, execRoot string) { + hasRandomSeed := false + for _, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { + hasRandomSeed = true + break + } + } + if !hasRandomSeed { + return + } + + workRoot := goBuildWorkRoot(args) + digest := sha256.New() + for _, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { + continue + } + normalized := arg + if workRoot != "" { + normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") + } + if execRoot != "" { + normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") + } + _, _ = io.WriteString(digest, normalized) + _, _ = digest.Write([]byte{0}) + } + seed := cgoRandomSeedPrefix + hex.EncodeToString(digest.Sum(nil)) + for i, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { + args[i] = seed + } + } +} + +func goBuildWorkRoot(args []string) string { + for i := 0; i+1 < len(args); i++ { + if args[i] != "-o" && args[i] != "-c" { + continue + } + for dir := filepath.Dir(args[i+1]); ; dir = filepath.Dir(dir) { + base := filepath.Base(dir) + if strings.HasPrefix(base, "go-build") { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + } + } + return "" +} diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/cc_test.go b/third_party/rgo/v0_61_1/base/go/tools/builders/cc_test.go new file mode 100644 index 00000000..f570843c --- /dev/null +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/cc_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { + firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") + secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") + firstWorkRoot := filepath.Join("tmp", "go-build111") + secondWorkRoot := filepath.Join("tmp", "go-build222") + + first := []string{ + "clang", + "-I" + filepath.Join(firstExecRoot, "external", "sysroot", "include"), + "-frandom-seed=volatile-first", + "-o", filepath.Join(firstWorkRoot, "b001", "_x001.o"), + "-c", filepath.Join(firstWorkRoot, "b001", "_cgo_export.c"), + } + second := []string{ + "clang", + "-I" + filepath.Join(secondExecRoot, "external", "sysroot", "include"), + "-frandom-seed=volatile-second", + "-o", filepath.Join(secondWorkRoot, "b001", "_x001.o"), + "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), + } + + normalizeCgoRandomSeed(first, firstExecRoot) + normalizeCgoRandomSeed(second, secondExecRoot) + if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { + t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) + } + + second = append(second, "-DPROFILE=changed") + normalizeCgoRandomSeed(second, secondExecRoot) + if randomSeedArg(first) == randomSeedArg(second) { + t.Fatal("meaningful compiler arguments must affect the normalized seed") + } +} + +func randomSeedArg(args []string) string { + for _, arg := range args { + if strings.HasPrefix(arg, "-frandom-seed=") { + return arg + } + } + return "" +} diff --git a/third_party/rgo/v0_62_0/base.CHANGED_FILES.md b/third_party/rgo/v0_62_0/base.CHANGED_FILES.md index 18b46e35..b690a1ce 100644 --- a/third_party/rgo/v0_62_0/base.CHANGED_FILES.md +++ b/third_party/rgo/v0_62_0/base.CHANGED_FILES.md @@ -12,9 +12,9 @@ This file is generated. Do not edit by hand. ## Summary -- Total changed paths: `56` +- Total changed paths: `57` - Modified files: `32` -- Added files: `24` +- Added files: `25` - Removed files: `0` ## Modified files @@ -58,6 +58,7 @@ This file is generated. Do not edit by hand. - `go/private/orchestrion/BUILD` - `go/private/orchestrion/extensions.bzl` - `go/private/orchestrion/pin_files.bzl` +- `go/tools/builders/cc_test.go` - `go/tools/builders/compilepkg_test.go` - `go/tools/builders/env_orchestrion.go` - `go/tools/builders/importcfg_test.go` diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/BUILD.bazel b/third_party/rgo/v0_62_0/base/go/tools/builders/BUILD.bazel index 64996d3f..2a060d57 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/BUILD.bazel +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/BUILD.bazel @@ -128,11 +128,12 @@ go_test( name = "orchestrion_test", size = "small", srcs = [ + "cc_test.go", "orchestrion_cache_test.go", "orchestrion_mode_test.go", "orchestrion_skip_test.go", - "orchestrion_test_helpers_test.go", "orchestrion_test.go", + "orchestrion_test_helpers_test.go", "stdlib_test.go", ":builder_srcs", ], @@ -215,6 +216,7 @@ go_source( "constants.go", "env.go", "flags.go", + "module_proxy.go", "nogo_fix.go", "nogo_goversions_go117.go", "nogo_goversions_go118.go", @@ -225,7 +227,6 @@ go_source( "nogo_typeparams_go118.go", "nogo_version.go", "nolint.go", - "module_proxy.go", "orchestrion.go", "orchestrion_cache.go", "orchestrion_mode.go", diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/cc.go b/third_party/rgo/v0_62_0/base/go/tools/builders/cc.go index 5451c671..f7ba8b1c 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/cc.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/cc.go @@ -1,8 +1,11 @@ package main import ( + "crypto/sha256" + "encoding/hex" "errors" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -108,6 +111,11 @@ func cc(args []string) error { } return s }) + // The go command derives this seed from its action ID. That ID contains the + // absolute -toolexec path, so otherwise identical stdlib actions running in + // different Bazel sandboxes make the CGO archive outputs differ. Recompute + // the seed from the compiler command after replacing ephemeral roots. + normalizeCgoRandomSeed(normalized, ccroot) if runtime.GOOS == "windows" { cmd := exec.Command(normalized[0], normalized[1:]...) cmd.Stdout = os.Stdout @@ -117,3 +125,60 @@ func cc(args []string) error { return syscall.Exec(normalized[0], normalized, os.Environ()) } } + +const cgoRandomSeedPrefix = "-frandom-seed=" + +func normalizeCgoRandomSeed(args []string, execRoot string) { + hasRandomSeed := false + for _, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { + hasRandomSeed = true + break + } + } + if !hasRandomSeed { + return + } + + workRoot := goBuildWorkRoot(args) + digest := sha256.New() + for _, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { + continue + } + normalized := arg + if workRoot != "" { + normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") + } + if execRoot != "" { + normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") + } + _, _ = io.WriteString(digest, normalized) + _, _ = digest.Write([]byte{0}) + } + seed := cgoRandomSeedPrefix + hex.EncodeToString(digest.Sum(nil)) + for i, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { + args[i] = seed + } + } +} + +func goBuildWorkRoot(args []string) string { + for i := 0; i+1 < len(args); i++ { + if args[i] != "-o" && args[i] != "-c" { + continue + } + for dir := filepath.Dir(args[i+1]); ; dir = filepath.Dir(dir) { + base := filepath.Base(dir) + if strings.HasPrefix(base, "go-build") { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + } + } + return "" +} diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/cc_test.go b/third_party/rgo/v0_62_0/base/go/tools/builders/cc_test.go new file mode 100644 index 00000000..f570843c --- /dev/null +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/cc_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { + firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") + secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") + firstWorkRoot := filepath.Join("tmp", "go-build111") + secondWorkRoot := filepath.Join("tmp", "go-build222") + + first := []string{ + "clang", + "-I" + filepath.Join(firstExecRoot, "external", "sysroot", "include"), + "-frandom-seed=volatile-first", + "-o", filepath.Join(firstWorkRoot, "b001", "_x001.o"), + "-c", filepath.Join(firstWorkRoot, "b001", "_cgo_export.c"), + } + second := []string{ + "clang", + "-I" + filepath.Join(secondExecRoot, "external", "sysroot", "include"), + "-frandom-seed=volatile-second", + "-o", filepath.Join(secondWorkRoot, "b001", "_x001.o"), + "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), + } + + normalizeCgoRandomSeed(first, firstExecRoot) + normalizeCgoRandomSeed(second, secondExecRoot) + if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { + t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) + } + + second = append(second, "-DPROFILE=changed") + normalizeCgoRandomSeed(second, secondExecRoot) + if randomSeedArg(first) == randomSeedArg(second) { + t.Fatal("meaningful compiler arguments must affect the normalized seed") + } +} + +func randomSeedArg(args []string) string { + for _, arg := range args { + if strings.HasPrefix(arg, "-frandom-seed=") { + return arg + } + } + return "" +} diff --git a/third_party/rgo/v0_63_0/base.CHANGED_FILES.md b/third_party/rgo/v0_63_0/base.CHANGED_FILES.md index 9c2bff51..8a1f8151 100644 --- a/third_party/rgo/v0_63_0/base.CHANGED_FILES.md +++ b/third_party/rgo/v0_63_0/base.CHANGED_FILES.md @@ -12,9 +12,9 @@ This file is generated. Do not edit by hand. ## Summary -- Total changed paths: `55` +- Total changed paths: `56` - Modified files: `31` -- Added files: `24` +- Added files: `25` - Removed files: `0` ## Modified files @@ -57,6 +57,7 @@ This file is generated. Do not edit by hand. - `go/private/orchestrion/BUILD` - `go/private/orchestrion/extensions.bzl` - `go/private/orchestrion/pin_files.bzl` +- `go/tools/builders/cc_test.go` - `go/tools/builders/compilepkg_test.go` - `go/tools/builders/env_orchestrion.go` - `go/tools/builders/importcfg_test.go` diff --git a/third_party/rgo/v0_63_0/base/go/tools/builders/BUILD.bazel b/third_party/rgo/v0_63_0/base/go/tools/builders/BUILD.bazel index c2fb8ed5..3a8d7a75 100644 --- a/third_party/rgo/v0_63_0/base/go/tools/builders/BUILD.bazel +++ b/third_party/rgo/v0_63_0/base/go/tools/builders/BUILD.bazel @@ -134,11 +134,12 @@ go_test( name = "orchestrion_test", size = "small", srcs = [ + "cc_test.go", "orchestrion_cache_test.go", "orchestrion_mode_test.go", "orchestrion_skip_test.go", - "orchestrion_test_helpers_test.go", "orchestrion_test.go", + "orchestrion_test_helpers_test.go", "stdlib_test.go", ":builder_srcs", ], @@ -230,6 +231,7 @@ go_source( "constants.go", "env.go", "flags.go", + "module_proxy.go", "nogo_fix.go", "nogo_goversions_go118.go", "nogo_goversions_go121.go", @@ -237,7 +239,6 @@ go_source( "nogo_main.go", "nogo_version.go", "nolint.go", - "module_proxy.go", "orchestrion.go", "orchestrion_cache.go", "orchestrion_mode.go", diff --git a/third_party/rgo/v0_63_0/base/go/tools/builders/cc.go b/third_party/rgo/v0_63_0/base/go/tools/builders/cc.go index 5451c671..f7ba8b1c 100644 --- a/third_party/rgo/v0_63_0/base/go/tools/builders/cc.go +++ b/third_party/rgo/v0_63_0/base/go/tools/builders/cc.go @@ -1,8 +1,11 @@ package main import ( + "crypto/sha256" + "encoding/hex" "errors" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -108,6 +111,11 @@ func cc(args []string) error { } return s }) + // The go command derives this seed from its action ID. That ID contains the + // absolute -toolexec path, so otherwise identical stdlib actions running in + // different Bazel sandboxes make the CGO archive outputs differ. Recompute + // the seed from the compiler command after replacing ephemeral roots. + normalizeCgoRandomSeed(normalized, ccroot) if runtime.GOOS == "windows" { cmd := exec.Command(normalized[0], normalized[1:]...) cmd.Stdout = os.Stdout @@ -117,3 +125,60 @@ func cc(args []string) error { return syscall.Exec(normalized[0], normalized, os.Environ()) } } + +const cgoRandomSeedPrefix = "-frandom-seed=" + +func normalizeCgoRandomSeed(args []string, execRoot string) { + hasRandomSeed := false + for _, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { + hasRandomSeed = true + break + } + } + if !hasRandomSeed { + return + } + + workRoot := goBuildWorkRoot(args) + digest := sha256.New() + for _, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { + continue + } + normalized := arg + if workRoot != "" { + normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") + } + if execRoot != "" { + normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") + } + _, _ = io.WriteString(digest, normalized) + _, _ = digest.Write([]byte{0}) + } + seed := cgoRandomSeedPrefix + hex.EncodeToString(digest.Sum(nil)) + for i, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { + args[i] = seed + } + } +} + +func goBuildWorkRoot(args []string) string { + for i := 0; i+1 < len(args); i++ { + if args[i] != "-o" && args[i] != "-c" { + continue + } + for dir := filepath.Dir(args[i+1]); ; dir = filepath.Dir(dir) { + base := filepath.Base(dir) + if strings.HasPrefix(base, "go-build") { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + } + } + return "" +} diff --git a/third_party/rgo/v0_63_0/base/go/tools/builders/cc_test.go b/third_party/rgo/v0_63_0/base/go/tools/builders/cc_test.go new file mode 100644 index 00000000..f570843c --- /dev/null +++ b/third_party/rgo/v0_63_0/base/go/tools/builders/cc_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { + firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") + secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") + firstWorkRoot := filepath.Join("tmp", "go-build111") + secondWorkRoot := filepath.Join("tmp", "go-build222") + + first := []string{ + "clang", + "-I" + filepath.Join(firstExecRoot, "external", "sysroot", "include"), + "-frandom-seed=volatile-first", + "-o", filepath.Join(firstWorkRoot, "b001", "_x001.o"), + "-c", filepath.Join(firstWorkRoot, "b001", "_cgo_export.c"), + } + second := []string{ + "clang", + "-I" + filepath.Join(secondExecRoot, "external", "sysroot", "include"), + "-frandom-seed=volatile-second", + "-o", filepath.Join(secondWorkRoot, "b001", "_x001.o"), + "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), + } + + normalizeCgoRandomSeed(first, firstExecRoot) + normalizeCgoRandomSeed(second, secondExecRoot) + if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { + t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) + } + + second = append(second, "-DPROFILE=changed") + normalizeCgoRandomSeed(second, secondExecRoot) + if randomSeedArg(first) == randomSeedArg(second) { + t.Fatal("meaningful compiler arguments must affect the normalized seed") + } +} + +func randomSeedArg(args []string) string { + for _, arg := range args { + if strings.HasPrefix(arg, "-frandom-seed=") { + return arg + } + } + return "" +} diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index b59117ae..44d3f669 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -4317,7 +4317,7 @@ index b31a29a..071746f 100644 def _check_ternary(name, value): diff --git a/go/tools/builders/BUILD.bazel b/go/tools/builders/BUILD.bazel -index 42b26a2..6f57dc4 100644 +index 42b26a2..f09b336 100644 --- a/go/tools/builders/BUILD.bazel +++ b/go/tools/builders/BUILD.bazel @@ -23,11 +23,8 @@ go_test( @@ -4346,7 +4346,7 @@ index 42b26a2..6f57dc4 100644 ], data = ["@go_sdk//:files"], rundir = ".", -@@ -74,9 +68,76 @@ go_test( +@@ -74,9 +68,77 @@ go_test( name = "env_test", size = "small", srcs = [ @@ -4403,11 +4403,12 @@ index 42b26a2..6f57dc4 100644 + name = "orchestrion_test", + size = "small", + srcs = [ ++ "cc_test.go", + "orchestrion_cache_test.go", + "orchestrion_mode_test.go", + "orchestrion_skip_test.go", -+ "orchestrion_test_helpers_test.go", + "orchestrion_test.go", ++ "orchestrion_test_helpers_test.go", + "stdlib_test.go", + ":builder_srcs", + ], @@ -4425,7 +4426,7 @@ index 42b26a2..6f57dc4 100644 ], ) -@@ -94,6 +155,7 @@ filegroup( +@@ -94,6 +156,7 @@ filegroup( "edit.go", "embedcfg.go", "env.go", @@ -4433,7 +4434,7 @@ index 42b26a2..6f57dc4 100644 "filter.go", "filter_buildid.go", "flags.go", -@@ -101,12 +163,20 @@ filegroup( +@@ -101,12 +164,20 @@ filegroup( "generate_test_main.go", "importcfg.go", "link.go", @@ -4454,11 +4455,16 @@ index 42b26a2..6f57dc4 100644 ] + select({ "@bazel_tools//src/conditions:windows": ["path_windows.go"], "//conditions:default": ["path.go"], -@@ -125,6 +195,14 @@ go_source( +@@ -120,11 +191,19 @@ go_source( + "constants.go", + "env.go", + "flags.go", ++ "module_proxy.go", + "nogo_fix.go", + "nogo_main.go", "nogo_typeparams_go117.go", "nogo_typeparams_go118.go", "nolint.go", -+ "module_proxy.go", + "orchestrion.go", + "orchestrion_cache.go", + "orchestrion_mode.go", @@ -4469,7 +4475,7 @@ index 42b26a2..6f57dc4 100644 ], # //go/tools/builders:nogo_srcs is considered a different target by # Bazel's visibility check than -@@ -146,6 +224,14 @@ go_binary( +@@ -146,6 +225,14 @@ go_binary( "env.go", "flags.go", "go_path.go", @@ -4484,7 +4490,7 @@ index 42b26a2..6f57dc4 100644 ], visibility = ["//visibility:public"], ) -@@ -162,6 +248,7 @@ go_binary( +@@ -162,6 +249,7 @@ go_binary( "env.go", "flags.go", "info.go", @@ -4492,7 +4498,7 @@ index 42b26a2..6f57dc4 100644 ], visibility = ["//visibility:public"], ) -@@ -179,6 +266,7 @@ go_binary( +@@ -179,6 +267,7 @@ go_binary( srcs = [ "env.go", "flags.go", @@ -4726,18 +4732,22 @@ index fdeda2d..76d69b6 100644 } } diff --git a/go/tools/builders/cc.go b/go/tools/builders/cc.go -index 6f67914..5451c67 100644 +index 6f67914..f7ba8b1 100644 --- a/go/tools/builders/cc.go +++ b/go/tools/builders/cc.go -@@ -2,6 +2,7 @@ package main +@@ -1,7 +1,11 @@ + package main import ( ++ "crypto/sha256" ++ "encoding/hex" "errors" + "fmt" ++ "io" "os" "os/exec" "path/filepath" -@@ -10,6 +11,70 @@ import ( +@@ -10,6 +14,70 @@ import ( "syscall" ) @@ -4808,7 +4818,7 @@ index 6f67914..5451c67 100644 func cc(args []string) error { cc := os.Getenv("GO_CC") if cc == "" { -@@ -20,7 +85,13 @@ func cc(args []string) error { +@@ -20,7 +88,13 @@ func cc(args []string) error { return errors.New("GO_CC_ROOT environment variable not set") } @@ -4823,6 +4833,135 @@ index 6f67914..5451c67 100644 normalized = append(normalized, args...) transformArgs(normalized, cgoAbsEnvFlags, func(s string) string { if strings.HasPrefix(s, cgoAbsPlaceholder) { +@@ -37,6 +111,11 @@ func cc(args []string) error { + } + return s + }) ++ // The go command derives this seed from its action ID. That ID contains the ++ // absolute -toolexec path, so otherwise identical stdlib actions running in ++ // different Bazel sandboxes make the CGO archive outputs differ. Recompute ++ // the seed from the compiler command after replacing ephemeral roots. ++ normalizeCgoRandomSeed(normalized, ccroot) + if runtime.GOOS == "windows" { + cmd := exec.Command(normalized[0], normalized[1:]...) + cmd.Stdout = os.Stdout +@@ -46,3 +125,60 @@ func cc(args []string) error { + return syscall.Exec(normalized[0], normalized, os.Environ()) + } + } ++ ++const cgoRandomSeedPrefix = "-frandom-seed=" ++ ++func normalizeCgoRandomSeed(args []string, execRoot string) { ++ hasRandomSeed := false ++ for _, arg := range args { ++ if strings.HasPrefix(arg, cgoRandomSeedPrefix) { ++ hasRandomSeed = true ++ break ++ } ++ } ++ if !hasRandomSeed { ++ return ++ } ++ ++ workRoot := goBuildWorkRoot(args) ++ digest := sha256.New() ++ for _, arg := range args { ++ if strings.HasPrefix(arg, cgoRandomSeedPrefix) { ++ continue ++ } ++ normalized := arg ++ if workRoot != "" { ++ normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") ++ } ++ if execRoot != "" { ++ normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") ++ } ++ _, _ = io.WriteString(digest, normalized) ++ _, _ = digest.Write([]byte{0}) ++ } ++ seed := cgoRandomSeedPrefix + hex.EncodeToString(digest.Sum(nil)) ++ for i, arg := range args { ++ if strings.HasPrefix(arg, cgoRandomSeedPrefix) { ++ args[i] = seed ++ } ++ } ++} ++ ++func goBuildWorkRoot(args []string) string { ++ for i := 0; i+1 < len(args); i++ { ++ if args[i] != "-o" && args[i] != "-c" { ++ continue ++ } ++ for dir := filepath.Dir(args[i+1]); ; dir = filepath.Dir(dir) { ++ base := filepath.Base(dir) ++ if strings.HasPrefix(base, "go-build") { ++ return dir ++ } ++ parent := filepath.Dir(dir) ++ if parent == dir { ++ break ++ } ++ } ++ } ++ return "" ++} +diff --git a/go/tools/builders/cc_test.go b/go/tools/builders/cc_test.go +new file mode 100644 +index 0000000..f570843 +--- /dev/null ++++ b/go/tools/builders/cc_test.go +@@ -0,0 +1,50 @@ ++package main ++ ++import ( ++ "path/filepath" ++ "strings" ++ "testing" ++) ++ ++func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { ++ firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") ++ secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") ++ firstWorkRoot := filepath.Join("tmp", "go-build111") ++ secondWorkRoot := filepath.Join("tmp", "go-build222") ++ ++ first := []string{ ++ "clang", ++ "-I" + filepath.Join(firstExecRoot, "external", "sysroot", "include"), ++ "-frandom-seed=volatile-first", ++ "-o", filepath.Join(firstWorkRoot, "b001", "_x001.o"), ++ "-c", filepath.Join(firstWorkRoot, "b001", "_cgo_export.c"), ++ } ++ second := []string{ ++ "clang", ++ "-I" + filepath.Join(secondExecRoot, "external", "sysroot", "include"), ++ "-frandom-seed=volatile-second", ++ "-o", filepath.Join(secondWorkRoot, "b001", "_x001.o"), ++ "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), ++ } ++ ++ normalizeCgoRandomSeed(first, firstExecRoot) ++ normalizeCgoRandomSeed(second, secondExecRoot) ++ if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { ++ t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) ++ } ++ ++ second = append(second, "-DPROFILE=changed") ++ normalizeCgoRandomSeed(second, secondExecRoot) ++ if randomSeedArg(first) == randomSeedArg(second) { ++ t.Fatal("meaningful compiler arguments must affect the normalized seed") ++ } ++} ++ ++func randomSeedArg(args []string) string { ++ for _, arg := range args { ++ if strings.HasPrefix(arg, "-frandom-seed=") { ++ return arg ++ } ++ } ++ return "" ++} diff --git a/go/tools/builders/compilepkg.go b/go/tools/builders/compilepkg.go index e2b0be5..5356b08 100644 --- a/go/tools/builders/compilepkg.go diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index d02c738d..e65dd054 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -4313,7 +4313,7 @@ index 809ede6..6751455 100644 def _check_ternary(name, value): diff --git a/go/tools/builders/BUILD.bazel b/go/tools/builders/BUILD.bazel -index ae68ab5..9cb7118 100644 +index ae68ab5..0799931 100644 --- a/go/tools/builders/BUILD.bazel +++ b/go/tools/builders/BUILD.bazel @@ -23,11 +23,8 @@ go_test( @@ -4342,7 +4342,7 @@ index ae68ab5..9cb7118 100644 ], data = ["@go_sdk//:files"], rundir = ".", -@@ -74,9 +68,76 @@ go_test( +@@ -74,9 +68,77 @@ go_test( name = "env_test", size = "small", srcs = [ @@ -4399,11 +4399,12 @@ index ae68ab5..9cb7118 100644 + name = "orchestrion_test", + size = "small", + srcs = [ ++ "cc_test.go", + "orchestrion_cache_test.go", + "orchestrion_mode_test.go", + "orchestrion_skip_test.go", -+ "orchestrion_test_helpers_test.go", + "orchestrion_test.go", ++ "orchestrion_test_helpers_test.go", + "stdlib_test.go", + ":builder_srcs", + ], @@ -4421,7 +4422,7 @@ index ae68ab5..9cb7118 100644 ], ) -@@ -109,6 +170,7 @@ filegroup( +@@ -109,6 +171,7 @@ filegroup( "edit.go", "embedcfg.go", "env.go", @@ -4429,7 +4430,7 @@ index ae68ab5..9cb7118 100644 "filter.go", "filter_buildid.go", "flags.go", -@@ -116,12 +178,20 @@ filegroup( +@@ -116,12 +179,20 @@ filegroup( "generate_test_main.go", "importcfg.go", "link.go", @@ -4450,11 +4451,18 @@ index ae68ab5..9cb7118 100644 ] + select({ "@bazel_tools//src/conditions:windows": ["path_windows.go"], "//conditions:default": ["path.go"], -@@ -145,6 +215,14 @@ go_source( +@@ -135,6 +206,7 @@ go_source( + "constants.go", + "env.go", + "flags.go", ++ "module_proxy.go", + "nogo_fix.go", + "nogo_goversions_go117.go", + "nogo_goversions_go118.go", +@@ -145,6 +217,13 @@ go_source( "nogo_typeparams_go118.go", "nogo_version.go", "nolint.go", -+ "module_proxy.go", + "orchestrion.go", + "orchestrion_cache.go", + "orchestrion_mode.go", @@ -4465,7 +4473,7 @@ index ae68ab5..9cb7118 100644 ], # //go/tools/builders:nogo_srcs is considered a different target by # Bazel's visibility check than -@@ -166,6 +244,14 @@ go_binary( +@@ -166,6 +245,14 @@ go_binary( "env.go", "flags.go", "go_path.go", @@ -4480,7 +4488,7 @@ index ae68ab5..9cb7118 100644 ], visibility = ["//visibility:public"], ) -@@ -182,6 +268,7 @@ go_binary( +@@ -182,6 +269,7 @@ go_binary( "env.go", "flags.go", "info.go", @@ -4488,7 +4496,7 @@ index ae68ab5..9cb7118 100644 ], visibility = ["//visibility:public"], ) -@@ -199,6 +286,7 @@ go_binary( +@@ -199,6 +287,7 @@ go_binary( srcs = [ "env.go", "flags.go", @@ -4722,18 +4730,22 @@ index fdeda2d..76d69b6 100644 } } diff --git a/go/tools/builders/cc.go b/go/tools/builders/cc.go -index 6f67914..5451c67 100644 +index 6f67914..f7ba8b1 100644 --- a/go/tools/builders/cc.go +++ b/go/tools/builders/cc.go -@@ -2,6 +2,7 @@ package main +@@ -1,7 +1,11 @@ + package main import ( ++ "crypto/sha256" ++ "encoding/hex" "errors" + "fmt" ++ "io" "os" "os/exec" "path/filepath" -@@ -10,6 +11,70 @@ import ( +@@ -10,6 +14,70 @@ import ( "syscall" ) @@ -4804,7 +4816,7 @@ index 6f67914..5451c67 100644 func cc(args []string) error { cc := os.Getenv("GO_CC") if cc == "" { -@@ -20,7 +85,13 @@ func cc(args []string) error { +@@ -20,7 +88,13 @@ func cc(args []string) error { return errors.New("GO_CC_ROOT environment variable not set") } @@ -4819,6 +4831,135 @@ index 6f67914..5451c67 100644 normalized = append(normalized, args...) transformArgs(normalized, cgoAbsEnvFlags, func(s string) string { if strings.HasPrefix(s, cgoAbsPlaceholder) { +@@ -37,6 +111,11 @@ func cc(args []string) error { + } + return s + }) ++ // The go command derives this seed from its action ID. That ID contains the ++ // absolute -toolexec path, so otherwise identical stdlib actions running in ++ // different Bazel sandboxes make the CGO archive outputs differ. Recompute ++ // the seed from the compiler command after replacing ephemeral roots. ++ normalizeCgoRandomSeed(normalized, ccroot) + if runtime.GOOS == "windows" { + cmd := exec.Command(normalized[0], normalized[1:]...) + cmd.Stdout = os.Stdout +@@ -46,3 +125,60 @@ func cc(args []string) error { + return syscall.Exec(normalized[0], normalized, os.Environ()) + } + } ++ ++const cgoRandomSeedPrefix = "-frandom-seed=" ++ ++func normalizeCgoRandomSeed(args []string, execRoot string) { ++ hasRandomSeed := false ++ for _, arg := range args { ++ if strings.HasPrefix(arg, cgoRandomSeedPrefix) { ++ hasRandomSeed = true ++ break ++ } ++ } ++ if !hasRandomSeed { ++ return ++ } ++ ++ workRoot := goBuildWorkRoot(args) ++ digest := sha256.New() ++ for _, arg := range args { ++ if strings.HasPrefix(arg, cgoRandomSeedPrefix) { ++ continue ++ } ++ normalized := arg ++ if workRoot != "" { ++ normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") ++ } ++ if execRoot != "" { ++ normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") ++ } ++ _, _ = io.WriteString(digest, normalized) ++ _, _ = digest.Write([]byte{0}) ++ } ++ seed := cgoRandomSeedPrefix + hex.EncodeToString(digest.Sum(nil)) ++ for i, arg := range args { ++ if strings.HasPrefix(arg, cgoRandomSeedPrefix) { ++ args[i] = seed ++ } ++ } ++} ++ ++func goBuildWorkRoot(args []string) string { ++ for i := 0; i+1 < len(args); i++ { ++ if args[i] != "-o" && args[i] != "-c" { ++ continue ++ } ++ for dir := filepath.Dir(args[i+1]); ; dir = filepath.Dir(dir) { ++ base := filepath.Base(dir) ++ if strings.HasPrefix(base, "go-build") { ++ return dir ++ } ++ parent := filepath.Dir(dir) ++ if parent == dir { ++ break ++ } ++ } ++ } ++ return "" ++} +diff --git a/go/tools/builders/cc_test.go b/go/tools/builders/cc_test.go +new file mode 100644 +index 0000000..f570843 +--- /dev/null ++++ b/go/tools/builders/cc_test.go +@@ -0,0 +1,50 @@ ++package main ++ ++import ( ++ "path/filepath" ++ "strings" ++ "testing" ++) ++ ++func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { ++ firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") ++ secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") ++ firstWorkRoot := filepath.Join("tmp", "go-build111") ++ secondWorkRoot := filepath.Join("tmp", "go-build222") ++ ++ first := []string{ ++ "clang", ++ "-I" + filepath.Join(firstExecRoot, "external", "sysroot", "include"), ++ "-frandom-seed=volatile-first", ++ "-o", filepath.Join(firstWorkRoot, "b001", "_x001.o"), ++ "-c", filepath.Join(firstWorkRoot, "b001", "_cgo_export.c"), ++ } ++ second := []string{ ++ "clang", ++ "-I" + filepath.Join(secondExecRoot, "external", "sysroot", "include"), ++ "-frandom-seed=volatile-second", ++ "-o", filepath.Join(secondWorkRoot, "b001", "_x001.o"), ++ "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), ++ } ++ ++ normalizeCgoRandomSeed(first, firstExecRoot) ++ normalizeCgoRandomSeed(second, secondExecRoot) ++ if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { ++ t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) ++ } ++ ++ second = append(second, "-DPROFILE=changed") ++ normalizeCgoRandomSeed(second, secondExecRoot) ++ if randomSeedArg(first) == randomSeedArg(second) { ++ t.Fatal("meaningful compiler arguments must affect the normalized seed") ++ } ++} ++ ++func randomSeedArg(args []string) string { ++ for _, arg := range args { ++ if strings.HasPrefix(arg, "-frandom-seed=") { ++ return arg ++ } ++ } ++ return "" ++} diff --git a/go/tools/builders/compilepkg.go b/go/tools/builders/compilepkg.go index e2b0be5..5356b08 100644 --- a/go/tools/builders/compilepkg.go diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index d5526007..a1eaeb35 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -4317,7 +4317,7 @@ index 809ede6..6751455 100644 def _check_ternary(name, value): diff --git a/go/tools/builders/BUILD.bazel b/go/tools/builders/BUILD.bazel -index a8834a1..64996d3 100644 +index a8834a1..2a060d5 100644 --- a/go/tools/builders/BUILD.bazel +++ b/go/tools/builders/BUILD.bazel @@ -32,11 +32,8 @@ go_test( @@ -4346,7 +4346,7 @@ index a8834a1..64996d3 100644 ], data = ["@go_sdk//:files"], rundir = ".", -@@ -83,9 +77,76 @@ go_test( +@@ -83,9 +77,77 @@ go_test( name = "env_test", size = "small", srcs = [ @@ -4403,11 +4403,12 @@ index a8834a1..64996d3 100644 + name = "orchestrion_test", + size = "small", + srcs = [ ++ "cc_test.go", + "orchestrion_cache_test.go", + "orchestrion_mode_test.go", + "orchestrion_skip_test.go", -+ "orchestrion_test_helpers_test.go", + "orchestrion_test.go", ++ "orchestrion_test_helpers_test.go", + "stdlib_test.go", + ":builder_srcs", + ], @@ -4425,7 +4426,7 @@ index a8834a1..64996d3 100644 ], ) -@@ -119,6 +180,7 @@ filegroup( +@@ -119,6 +181,7 @@ filegroup( "edit.go", "embedcfg.go", "env.go", @@ -4433,7 +4434,7 @@ index a8834a1..64996d3 100644 "filter.go", "filter_buildid.go", "flags.go", -@@ -126,12 +188,20 @@ filegroup( +@@ -126,12 +189,20 @@ filegroup( "generate_test_main.go", "importcfg.go", "link.go", @@ -4454,11 +4455,18 @@ index a8834a1..64996d3 100644 ] + select({ "@bazel_tools//src/conditions:windows": ["path_windows.go"], "//conditions:default": ["path.go"], -@@ -155,6 +225,14 @@ go_source( +@@ -145,6 +216,7 @@ go_source( + "constants.go", + "env.go", + "flags.go", ++ "module_proxy.go", + "nogo_fix.go", + "nogo_goversions_go117.go", + "nogo_goversions_go118.go", +@@ -155,6 +227,13 @@ go_source( "nogo_typeparams_go118.go", "nogo_version.go", "nolint.go", -+ "module_proxy.go", + "orchestrion.go", + "orchestrion_cache.go", + "orchestrion_mode.go", @@ -4469,7 +4477,7 @@ index a8834a1..64996d3 100644 ], # //go/tools/builders:nogo_srcs is considered a different target by # Bazel's visibility check than -@@ -176,6 +254,14 @@ go_binary( +@@ -176,6 +255,14 @@ go_binary( "env.go", "flags.go", "go_path.go", @@ -4484,7 +4492,7 @@ index a8834a1..64996d3 100644 ], visibility = ["//visibility:public"], ) -@@ -192,6 +278,7 @@ go_binary( +@@ -192,6 +279,7 @@ go_binary( "env.go", "flags.go", "info.go", @@ -4492,7 +4500,7 @@ index a8834a1..64996d3 100644 ], visibility = ["//visibility:public"], ) -@@ -209,6 +296,7 @@ go_binary( +@@ -209,6 +297,7 @@ go_binary( srcs = [ "env.go", "flags.go", @@ -4726,18 +4734,22 @@ index fdeda2d..76d69b6 100644 } } diff --git a/go/tools/builders/cc.go b/go/tools/builders/cc.go -index 6f67914..5451c67 100644 +index 6f67914..f7ba8b1 100644 --- a/go/tools/builders/cc.go +++ b/go/tools/builders/cc.go -@@ -2,6 +2,7 @@ package main +@@ -1,7 +1,11 @@ + package main import ( ++ "crypto/sha256" ++ "encoding/hex" "errors" + "fmt" ++ "io" "os" "os/exec" "path/filepath" -@@ -10,6 +11,70 @@ import ( +@@ -10,6 +14,70 @@ import ( "syscall" ) @@ -4808,7 +4820,7 @@ index 6f67914..5451c67 100644 func cc(args []string) error { cc := os.Getenv("GO_CC") if cc == "" { -@@ -20,7 +85,13 @@ func cc(args []string) error { +@@ -20,7 +88,13 @@ func cc(args []string) error { return errors.New("GO_CC_ROOT environment variable not set") } @@ -4823,6 +4835,135 @@ index 6f67914..5451c67 100644 normalized = append(normalized, args...) transformArgs(normalized, cgoAbsEnvFlags, func(s string) string { if strings.HasPrefix(s, cgoAbsPlaceholder) { +@@ -37,6 +111,11 @@ func cc(args []string) error { + } + return s + }) ++ // The go command derives this seed from its action ID. That ID contains the ++ // absolute -toolexec path, so otherwise identical stdlib actions running in ++ // different Bazel sandboxes make the CGO archive outputs differ. Recompute ++ // the seed from the compiler command after replacing ephemeral roots. ++ normalizeCgoRandomSeed(normalized, ccroot) + if runtime.GOOS == "windows" { + cmd := exec.Command(normalized[0], normalized[1:]...) + cmd.Stdout = os.Stdout +@@ -46,3 +125,60 @@ func cc(args []string) error { + return syscall.Exec(normalized[0], normalized, os.Environ()) + } + } ++ ++const cgoRandomSeedPrefix = "-frandom-seed=" ++ ++func normalizeCgoRandomSeed(args []string, execRoot string) { ++ hasRandomSeed := false ++ for _, arg := range args { ++ if strings.HasPrefix(arg, cgoRandomSeedPrefix) { ++ hasRandomSeed = true ++ break ++ } ++ } ++ if !hasRandomSeed { ++ return ++ } ++ ++ workRoot := goBuildWorkRoot(args) ++ digest := sha256.New() ++ for _, arg := range args { ++ if strings.HasPrefix(arg, cgoRandomSeedPrefix) { ++ continue ++ } ++ normalized := arg ++ if workRoot != "" { ++ normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") ++ } ++ if execRoot != "" { ++ normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") ++ } ++ _, _ = io.WriteString(digest, normalized) ++ _, _ = digest.Write([]byte{0}) ++ } ++ seed := cgoRandomSeedPrefix + hex.EncodeToString(digest.Sum(nil)) ++ for i, arg := range args { ++ if strings.HasPrefix(arg, cgoRandomSeedPrefix) { ++ args[i] = seed ++ } ++ } ++} ++ ++func goBuildWorkRoot(args []string) string { ++ for i := 0; i+1 < len(args); i++ { ++ if args[i] != "-o" && args[i] != "-c" { ++ continue ++ } ++ for dir := filepath.Dir(args[i+1]); ; dir = filepath.Dir(dir) { ++ base := filepath.Base(dir) ++ if strings.HasPrefix(base, "go-build") { ++ return dir ++ } ++ parent := filepath.Dir(dir) ++ if parent == dir { ++ break ++ } ++ } ++ } ++ return "" ++} +diff --git a/go/tools/builders/cc_test.go b/go/tools/builders/cc_test.go +new file mode 100644 +index 0000000..f570843 +--- /dev/null ++++ b/go/tools/builders/cc_test.go +@@ -0,0 +1,50 @@ ++package main ++ ++import ( ++ "path/filepath" ++ "strings" ++ "testing" ++) ++ ++func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { ++ firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") ++ secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") ++ firstWorkRoot := filepath.Join("tmp", "go-build111") ++ secondWorkRoot := filepath.Join("tmp", "go-build222") ++ ++ first := []string{ ++ "clang", ++ "-I" + filepath.Join(firstExecRoot, "external", "sysroot", "include"), ++ "-frandom-seed=volatile-first", ++ "-o", filepath.Join(firstWorkRoot, "b001", "_x001.o"), ++ "-c", filepath.Join(firstWorkRoot, "b001", "_cgo_export.c"), ++ } ++ second := []string{ ++ "clang", ++ "-I" + filepath.Join(secondExecRoot, "external", "sysroot", "include"), ++ "-frandom-seed=volatile-second", ++ "-o", filepath.Join(secondWorkRoot, "b001", "_x001.o"), ++ "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), ++ } ++ ++ normalizeCgoRandomSeed(first, firstExecRoot) ++ normalizeCgoRandomSeed(second, secondExecRoot) ++ if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { ++ t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) ++ } ++ ++ second = append(second, "-DPROFILE=changed") ++ normalizeCgoRandomSeed(second, secondExecRoot) ++ if randomSeedArg(first) == randomSeedArg(second) { ++ t.Fatal("meaningful compiler arguments must affect the normalized seed") ++ } ++} ++ ++func randomSeedArg(args []string) string { ++ for _, arg := range args { ++ if strings.HasPrefix(arg, "-frandom-seed=") { ++ return arg ++ } ++ } ++ return "" ++} diff --git a/go/tools/builders/compilepkg.go b/go/tools/builders/compilepkg.go index a1782d4..5d3ae22 100644 --- a/go/tools/builders/compilepkg.go diff --git a/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch index d11f3cb9..bab8cb61 100644 --- a/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch @@ -3298,7 +3298,7 @@ index e3b2e71..396cbd0 100644 def _check_ternary(name, value): diff --git a/go/tools/builders/BUILD.bazel b/go/tools/builders/BUILD.bazel -index 93cfebe..c2fb8ed 100644 +index 93cfebe..3a8d7a7 100644 --- a/go/tools/builders/BUILD.bazel +++ b/go/tools/builders/BUILD.bazel @@ -32,11 +32,8 @@ go_test( @@ -3327,7 +3327,7 @@ index 93cfebe..c2fb8ed 100644 ], data = [ "@go_default_sdk//:files", -@@ -89,9 +83,76 @@ go_test( +@@ -89,9 +83,77 @@ go_test( name = "env_test", size = "small", srcs = [ @@ -3384,11 +3384,12 @@ index 93cfebe..c2fb8ed 100644 + name = "orchestrion_test", + size = "small", + srcs = [ ++ "cc_test.go", + "orchestrion_cache_test.go", + "orchestrion_mode_test.go", + "orchestrion_skip_test.go", -+ "orchestrion_test_helpers_test.go", + "orchestrion_test.go", ++ "orchestrion_test_helpers_test.go", + "stdlib_test.go", + ":builder_srcs", + ], @@ -3406,7 +3407,7 @@ index 93cfebe..c2fb8ed 100644 ], ) -@@ -134,6 +195,7 @@ filegroup( +@@ -134,6 +196,7 @@ filegroup( "edit.go", "embedcfg.go", "env.go", @@ -3414,7 +3415,7 @@ index 93cfebe..c2fb8ed 100644 "filter.go", "filter_buildid.go", "flags.go", -@@ -141,12 +203,20 @@ filegroup( +@@ -141,12 +204,20 @@ filegroup( "generate_test_main.go", "importcfg.go", "link.go", @@ -3435,11 +3436,18 @@ index 93cfebe..c2fb8ed 100644 ] + select({ "@bazel_tools//src/conditions:windows": ["path_windows.go"], "//conditions:default": ["path.go"], -@@ -167,6 +237,14 @@ go_source( +@@ -160,6 +231,7 @@ go_source( + "constants.go", + "env.go", + "flags.go", ++ "module_proxy.go", + "nogo_fix.go", + "nogo_goversions_go118.go", + "nogo_goversions_go121.go", +@@ -167,6 +239,13 @@ go_source( "nogo_main.go", "nogo_version.go", "nolint.go", -+ "module_proxy.go", + "orchestrion.go", + "orchestrion_cache.go", + "orchestrion_mode.go", @@ -3450,7 +3458,7 @@ index 93cfebe..c2fb8ed 100644 ], # //go/tools/builders:nogo_srcs is considered a different target by # Bazel's visibility check than -@@ -188,6 +266,14 @@ go_binary( +@@ -188,6 +267,14 @@ go_binary( "env.go", "flags.go", "go_path.go", @@ -3465,7 +3473,7 @@ index 93cfebe..c2fb8ed 100644 ], visibility = ["//visibility:public"], ) -@@ -204,6 +290,7 @@ go_binary( +@@ -204,6 +291,7 @@ go_binary( "env.go", "flags.go", "info.go", @@ -3473,7 +3481,7 @@ index 93cfebe..c2fb8ed 100644 ], visibility = ["//visibility:public"], ) -@@ -221,6 +308,7 @@ go_binary( +@@ -221,6 +309,7 @@ go_binary( srcs = [ "env.go", "flags.go", @@ -3705,18 +3713,22 @@ index fdeda2d..76d69b6 100644 } } diff --git a/go/tools/builders/cc.go b/go/tools/builders/cc.go -index 6f67914..5451c67 100644 +index 6f67914..f7ba8b1 100644 --- a/go/tools/builders/cc.go +++ b/go/tools/builders/cc.go -@@ -2,6 +2,7 @@ package main +@@ -1,7 +1,11 @@ + package main import ( ++ "crypto/sha256" ++ "encoding/hex" "errors" + "fmt" ++ "io" "os" "os/exec" "path/filepath" -@@ -10,6 +11,70 @@ import ( +@@ -10,6 +14,70 @@ import ( "syscall" ) @@ -3787,7 +3799,7 @@ index 6f67914..5451c67 100644 func cc(args []string) error { cc := os.Getenv("GO_CC") if cc == "" { -@@ -20,7 +85,13 @@ func cc(args []string) error { +@@ -20,7 +88,13 @@ func cc(args []string) error { return errors.New("GO_CC_ROOT environment variable not set") } @@ -3802,6 +3814,135 @@ index 6f67914..5451c67 100644 normalized = append(normalized, args...) transformArgs(normalized, cgoAbsEnvFlags, func(s string) string { if strings.HasPrefix(s, cgoAbsPlaceholder) { +@@ -37,6 +111,11 @@ func cc(args []string) error { + } + return s + }) ++ // The go command derives this seed from its action ID. That ID contains the ++ // absolute -toolexec path, so otherwise identical stdlib actions running in ++ // different Bazel sandboxes make the CGO archive outputs differ. Recompute ++ // the seed from the compiler command after replacing ephemeral roots. ++ normalizeCgoRandomSeed(normalized, ccroot) + if runtime.GOOS == "windows" { + cmd := exec.Command(normalized[0], normalized[1:]...) + cmd.Stdout = os.Stdout +@@ -46,3 +125,60 @@ func cc(args []string) error { + return syscall.Exec(normalized[0], normalized, os.Environ()) + } + } ++ ++const cgoRandomSeedPrefix = "-frandom-seed=" ++ ++func normalizeCgoRandomSeed(args []string, execRoot string) { ++ hasRandomSeed := false ++ for _, arg := range args { ++ if strings.HasPrefix(arg, cgoRandomSeedPrefix) { ++ hasRandomSeed = true ++ break ++ } ++ } ++ if !hasRandomSeed { ++ return ++ } ++ ++ workRoot := goBuildWorkRoot(args) ++ digest := sha256.New() ++ for _, arg := range args { ++ if strings.HasPrefix(arg, cgoRandomSeedPrefix) { ++ continue ++ } ++ normalized := arg ++ if workRoot != "" { ++ normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") ++ } ++ if execRoot != "" { ++ normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") ++ } ++ _, _ = io.WriteString(digest, normalized) ++ _, _ = digest.Write([]byte{0}) ++ } ++ seed := cgoRandomSeedPrefix + hex.EncodeToString(digest.Sum(nil)) ++ for i, arg := range args { ++ if strings.HasPrefix(arg, cgoRandomSeedPrefix) { ++ args[i] = seed ++ } ++ } ++} ++ ++func goBuildWorkRoot(args []string) string { ++ for i := 0; i+1 < len(args); i++ { ++ if args[i] != "-o" && args[i] != "-c" { ++ continue ++ } ++ for dir := filepath.Dir(args[i+1]); ; dir = filepath.Dir(dir) { ++ base := filepath.Base(dir) ++ if strings.HasPrefix(base, "go-build") { ++ return dir ++ } ++ parent := filepath.Dir(dir) ++ if parent == dir { ++ break ++ } ++ } ++ } ++ return "" ++} +diff --git a/go/tools/builders/cc_test.go b/go/tools/builders/cc_test.go +new file mode 100644 +index 0000000..f570843 +--- /dev/null ++++ b/go/tools/builders/cc_test.go +@@ -0,0 +1,50 @@ ++package main ++ ++import ( ++ "path/filepath" ++ "strings" ++ "testing" ++) ++ ++func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { ++ firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") ++ secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") ++ firstWorkRoot := filepath.Join("tmp", "go-build111") ++ secondWorkRoot := filepath.Join("tmp", "go-build222") ++ ++ first := []string{ ++ "clang", ++ "-I" + filepath.Join(firstExecRoot, "external", "sysroot", "include"), ++ "-frandom-seed=volatile-first", ++ "-o", filepath.Join(firstWorkRoot, "b001", "_x001.o"), ++ "-c", filepath.Join(firstWorkRoot, "b001", "_cgo_export.c"), ++ } ++ second := []string{ ++ "clang", ++ "-I" + filepath.Join(secondExecRoot, "external", "sysroot", "include"), ++ "-frandom-seed=volatile-second", ++ "-o", filepath.Join(secondWorkRoot, "b001", "_x001.o"), ++ "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), ++ } ++ ++ normalizeCgoRandomSeed(first, firstExecRoot) ++ normalizeCgoRandomSeed(second, secondExecRoot) ++ if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { ++ t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) ++ } ++ ++ second = append(second, "-DPROFILE=changed") ++ normalizeCgoRandomSeed(second, secondExecRoot) ++ if randomSeedArg(first) == randomSeedArg(second) { ++ t.Fatal("meaningful compiler arguments must affect the normalized seed") ++ } ++} ++ ++func randomSeedArg(args []string) string { ++ for _, arg := range args { ++ if strings.HasPrefix(arg, "-frandom-seed=") { ++ return arg ++ } ++ } ++ return "" ++} diff --git a/go/tools/builders/compilepkg.go b/go/tools/builders/compilepkg.go index 98963c6..b64ae96 100644 --- a/go/tools/builders/compilepkg.go diff --git a/tools/dev/verify_rules_go_profiles.py b/tools/dev/verify_rules_go_profiles.py index f2ff0dfb..9aa7b1b9 100644 --- a/tools/dev/verify_rules_go_profiles.py +++ b/tools/dev/verify_rules_go_profiles.py @@ -174,6 +174,7 @@ def verify_workspace_runtime_functional_smoke( "--@io_bazel_rules_go//go/private/orchestrion:enabled=true", "--@io_bazel_rules_go//go/private/orchestrion:mode=test_optimization", ] + reproducibility_flags = cgo_reproducibility_flags(sys.platform) isolated_cache_flags = [ "--disk_cache=", "--remote_cache=", @@ -185,12 +186,12 @@ def verify_workspace_runtime_functional_smoke( ["build", *common_flags, "@go_sdk//:builder"], private_safe_patterns=private_safe_patterns, ) - first_plain = run_stdlib_inventory( + first_plain = run_plain_reproducibility_snapshot( bazel, bazel_output_user_root, workspace, command="build", - mode_flags=[*common_flags, *isolated_cache_flags], + mode_flags=[*common_flags, *reproducibility_flags, *isolated_cache_flags], target="//app:hello_test", private_safe_patterns=private_safe_patterns, ) @@ -199,7 +200,12 @@ def verify_workspace_runtime_functional_smoke( bazel_output_user_root, workspace, command="test", - mode_flags=[*orchestrion_flags, *isolated_cache_flags, "--test_output=errors"], + mode_flags=[ + *orchestrion_flags, + *reproducibility_flags, + *isolated_cache_flags, + "--test_output=errors", + ], target="//app:hello_test", private_safe_patterns=private_safe_patterns, ) @@ -207,18 +213,23 @@ def verify_workspace_runtime_functional_smoke( bazel, bazel_output_user_root, workspace, - ["aquery", *orchestrion_flags, 'mnemonic("GoCompilePkg", //app:hello_test)'], + [ + "aquery", + *orchestrion_flags, + *reproducibility_flags, + 'mnemonic("GoCompilePkg", //app:hello_test)', + ], private_safe_patterns=private_safe_patterns, ) assert_aquery_contains(aquery.stdout, patch) replay_output_user_root = work_root / "bazel_output_user_root_replay" - second_plain = run_stdlib_inventory( + second_plain = run_plain_reproducibility_snapshot( bazel, replay_output_user_root, workspace, command="build", - mode_flags=[*common_flags, *isolated_cache_flags], + mode_flags=[*common_flags, *reproducibility_flags, *isolated_cache_flags], target="//app:hello_test", private_safe_patterns=private_safe_patterns, ) @@ -227,18 +238,51 @@ def verify_workspace_runtime_functional_smoke( replay_output_user_root, workspace, command="test", - mode_flags=[*orchestrion_flags, *isolated_cache_flags, "--test_output=errors"], + mode_flags=[ + *orchestrion_flags, + *reproducibility_flags, + *isolated_cache_flags, + "--test_output=errors", + ], target="//app:hello_test", private_safe_patterns=private_safe_patterns, ) - assert_plain_stdlib_cache(first_plain, patch) - assert_plain_stdlib_cache(second_plain, patch) + assert_plain_stdlib_cache(first_plain.stdlib_cache, patch) + assert_plain_stdlib_cache(second_plain.stdlib_cache, patch) assert_orchestrion_stdlib_cache(first_orchestrion.stdlib_cache, patch) assert_orchestrion_stdlib_cache(second_orchestrion.stdlib_cache, patch) - if first_plain != second_plain: + if first_plain.stdlib_cache != second_plain.stdlib_cache: raise ValueError( "plain stdlib cache inventories differ for %s: %s" - % (patch, describe_snapshot_difference(first_plain, second_plain)) + % ( + patch, + describe_snapshot_difference( + first_plain.stdlib_cache, + second_plain.stdlib_cache, + ), + ) + ) + if first_plain.action_keys != second_plain.action_keys: + raise ValueError( + "plain GoStdlib action keys differ for %s: %s" + % ( + patch, + describe_mapping_difference( + first_plain.action_keys, + second_plain.action_keys, + ), + ) + ) + if first_plain.outputs != second_plain.outputs: + raise ValueError( + "plain GoStdlib outputs differ for %s: %s" + % ( + patch, + describe_mapping_difference( + first_plain.outputs, + second_plain.outputs, + ), + ) ) if first_orchestrion.stdlib_cache != second_orchestrion.stdlib_cache: raise ValueError( @@ -275,6 +319,32 @@ def verify_workspace_runtime_functional_smoke( ) +def cgo_reproducibility_flags(platform_name: str) -> list[str]: + """Mirror the CGO/debug conditions that exposed cross-worker drift.""" + flags = [ + "--compilation_mode=fastbuild", + "--incompatible_strict_action_env", + "--experimental_exec_configuration_distinguisher=diff_to_affected", + "--experimental_platform_in_output_dir", + "--@io_bazel_rules_go//go/config:pure=False", + "--strip=never", + "--copt=-O2", + "--copt=-fno-omit-frame-pointer", + "--copt=-g", + "--copt=-UNDEBUG", + ] + if platform_name.startswith("linux"): + flags.extend( + [ + "--repo_env=CC=clang", + "--linkopt=-fuse-ld=lld", + "--linkopt=-Wl,--build-id=md5", + "--linkopt=-Wl,--threads=4", + ] + ) + return flags + + def describe_snapshot_difference( first: StdlibCacheSnapshot, second: StdlibCacheSnapshot ) -> str: @@ -432,13 +502,67 @@ class StdlibCacheSnapshot: @dataclass(frozen=True) class ReproducibilitySnapshot: - """Cache contents, action keys, and outputs from one instrumented build.""" + """Cache contents, action keys, and outputs from one isolated build.""" stdlib_cache: StdlibCacheSnapshot action_keys: dict[str, str] outputs: dict[str, str] +def run_plain_reproducibility_snapshot( + bazel: Path, + output_user_root: Path, + workspace: Path, + *, + command: str, + mode_flags: list[str], + target: str, + private_safe_patterns: list[str] | None = None, +) -> ReproducibilitySnapshot: + """Build once and capture the ordinary CGO stdlib action and bytes.""" + stdlib_cache = run_stdlib_inventory( + bazel, + output_user_root, + workspace, + command=command, + mode_flags=mode_flags, + target=target, + private_safe_patterns=private_safe_patterns, + ) + aquery = run_bazel( + bazel, + output_user_root, + workspace, + [ + "aquery", + *mode_flags, + "--output=jsonproto", + 'mnemonic("GoStdlib", deps(%s))' % target, + ], + private_safe_patterns=private_safe_patterns, + ) + aquery_data = json.loads(aquery.stdout) + assert_cgo_reproducibility_actions( + aquery_data, + sys.platform, + expected_instrumented=False, + ) + action_keys, outputs = action_snapshot_from_aquery( + aquery_data, + workspace=workspace, + target_label=target, + ) + if not action_keys: + raise ValueError( + "plain reproducibility aquery for %s is missing GoStdlib" % target + ) + return ReproducibilitySnapshot( + stdlib_cache=stdlib_cache, + action_keys=action_keys, + outputs=outputs, + ) + + def run_orchestrion_reproducibility_snapshot( bazel: Path, output_user_root: Path, @@ -472,8 +596,14 @@ def run_orchestrion_reproducibility_snapshot( ], private_safe_patterns=private_safe_patterns, ) + aquery_data = json.loads(aquery.stdout) + assert_cgo_reproducibility_actions( + aquery_data, + sys.platform, + expected_instrumented=True, + ) action_keys, outputs = action_snapshot_from_aquery( - json.loads(aquery.stdout), + aquery_data, workspace=workspace, target_label=target, ) @@ -562,6 +692,41 @@ def action_snapshot_from_aquery( return action_keys, outputs +def assert_cgo_reproducibility_actions( + data: dict[str, object], + platform_name: str, + *, + expected_instrumented: bool, +) -> None: + """Require the replay to cover the expected CGO/debug stdlib mode.""" + matching = [] + for action in data.get("actions", []): + if action.get("mnemonic") != "GoStdlib": + continue + arguments = [str(arg) for arg in action.get("arguments", [])] + environment = { + str(item.get("key", item.get("name", ""))): str(item.get("value", "")) + for item in action.get("environmentVariables", []) + } + if environment.get("CGO_ENABLED") != "1": + continue + if "-g" not in environment.get("CGO_CFLAGS", "").split(): + continue + if platform_name.startswith("linux"): + ldflags = environment.get("CGO_LDFLAGS", "") + required = ["-fuse-ld=lld", "--build-id=md5", "--threads=4"] + if any(flag not in ldflags for flag in required): + continue + if ("-orchestrion" in arguments) == expected_instrumented: + matching.append(action) + if not matching: + mode = "instrumented" if expected_instrumented else "plain" + raise ValueError( + "reproducibility aquery must contain a %s CGO-enabled " + "GoStdlib action with debug flags" % mode + ) + + def resolve_path_fragment( fragment_id: int, fragments: dict[int, dict[str, object]] ) -> str: diff --git a/tools/tests/python/test_rules_go_profile_tools.py b/tools/tests/python/test_rules_go_profile_tools.py index f1c64ce5..a5f59a7e 100644 --- a/tools/tests/python/test_rules_go_profile_tools.py +++ b/tools/tests/python/test_rules_go_profile_tools.py @@ -542,6 +542,53 @@ def test_action_snapshot_covers_cache_critical_instrumented_actions(self) -> Non self.assertEqual(6, len(outputs)) self.assertFalse(any("library.a" in path for path in outputs)) + def test_reproducibility_flags_mirror_cgo_debug_builds(self) -> None: + """The replay preserves the CGO, debug, LLD, and cache-sensitive inputs.""" + common = self.mod.cgo_reproducibility_flags("darwin") + self.assertIn("--@io_bazel_rules_go//go/config:pure=False", common) + self.assertIn("--incompatible_strict_action_env", common) + self.assertIn("--experimental_platform_in_output_dir", common) + self.assertIn("--copt=-g", common) + self.assertNotIn("--linkopt=-Wl,--threads=4", common) + + linux = self.mod.cgo_reproducibility_flags("linux") + self.assertIn("--repo_env=CC=clang", linux) + self.assertIn("--linkopt=-fuse-ld=lld", linux) + self.assertIn("--linkopt=-Wl,--build-id=md5", linux) + self.assertIn("--linkopt=-Wl,--threads=4", linux) + + def test_reproducibility_aquery_requires_requested_cgo_mode(self) -> None: + """Each isolated replay must expose its requested CGO stdlib mode.""" + environment = [ + {"key": "CGO_ENABLED", "value": "1"}, + {"key": "CGO_CFLAGS", "value": "-O2 -g"}, + { + "key": "CGO_LDFLAGS", + "value": "-fuse-ld=lld -Wl,--build-id=md5 -Wl,--threads=4", + }, + ] + plain = { + "mnemonic": "GoStdlib", + "arguments": ["builder", "stdlib"], + "environmentVariables": environment, + } + instrumented = { + **plain, + "arguments": ["builder", "stdlib", "-orchestrion", "orchestrion"], + } + self.mod.assert_cgo_reproducibility_actions( + {"actions": [plain]}, "linux", expected_instrumented=False + ) + self.mod.assert_cgo_reproducibility_actions( + {"actions": [instrumented]}, "linux", expected_instrumented=True + ) + with self.assertRaisesRegex(ValueError, "plain CGO-enabled"): + self.mod.assert_cgo_reproducibility_actions( + {"actions": [instrumented]}, + "linux", + expected_instrumented=False, + ) + def test_action_output_digest_is_independent_of_its_root(self) -> None: """Logical paths, modes, and bytes determine a declared output digest.""" with tempfile.TemporaryDirectory() as raw_tmp: From 8a81b9e667eb08b26e80a1ae970897415f951d11 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 9 Sep 2026 17:57:02 +0200 Subject: [PATCH 3/6] rules_go: provide CGO to synthetic testmain builds --- .../base/go/private/actions/compilepkg.bzl | 17 +++-- .../base/go/private/actions/stdlib.bzl | 11 +-- .../base/go/private/actions/compilepkg.bzl | 17 +++-- .../base/go/private/actions/stdlib.bzl | 11 +-- .../base/go/private/actions/compilepkg.bzl | 17 +++-- .../base/go/private/actions/stdlib.bzl | 11 +-- .../base/go/private/actions/compilepkg.bzl | 17 +++-- .../base/go/private/actions/stdlib.bzl | 11 +-- .../v0_60_0/base/0001-full-delta.patch | 67 ++++++++++++------- .../v0_61_1/base/0001-full-delta.patch | 67 ++++++++++++------- .../v0_62_0/base/0001-full-delta.patch | 67 ++++++++++++------- .../v0_63_0/base/0001-full-delta.patch | 65 ++++++++++++------ .../assert_orchestrion_module_proxy_aquery.py | 38 +++++++++++ 13 files changed, 289 insertions(+), 127 deletions(-) diff --git a/third_party/rgo/v0_60_0/base/go/private/actions/compilepkg.bzl b/third_party/rgo/v0_60_0/base/go/private/actions/compilepkg.bzl index 036bf482..915610d1 100644 --- a/third_party/rgo/v0_60_0/base/go/private/actions/compilepkg.bzl +++ b/third_party/rgo/v0_60_0/base/go/private/actions/compilepkg.bzl @@ -17,6 +17,7 @@ load( "//go/private:mode.bzl", "link_mode_arg", ) +load("//go/private/actions:stdlib.bzl", "stdlib_env") load("//go/private/actions:utils.bzl", "quote_opts") load("//go/private/orchestrion:pin_files.bzl", "OrchestrionPinFilesInfo") @@ -254,13 +255,13 @@ def emit_compilepkg( if link_mode_flag: compile_args.add("-asmflags", link_mode_flag) - # cgo and the linker action don't support path mapping yet - # TODO: Remove the second condition after https://github.com/bazelbuild/bazel/pull/21921. + # cgo and nested Go commands that may invoke it do not support path mapping. + # TODO: Remove the local-tag condition after https://github.com/bazelbuild/bazel/pull/21921. orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None) if compile_orchestrion else None orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None) if compile_orchestrion else None orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None) if compile_orchestrion else None - if cgo or "local" in go._ctx.attr.tags: - # cgo doesn't support path mapping yet + synthetic_testmain_needs_cgo = out_synthetic_testmain_manifest != None and not go.mode.pure + if cgo or synthetic_testmain_needs_cgo or "local" in go._ctx.attr.tags: env = _orchestrion_action_env( go, go.env, @@ -300,6 +301,14 @@ def emit_compilepkg( if clinkopts: compile_args.add("-ldflags", quote_opts(clinkopts)) + if synthetic_testmain_needs_cgo: + if not cgo: + inputs_transitive.append(go.cc_toolchain_files) + + # The nested Go commands consume the woven stdlib cache, so their cgo + # settings must match the action that produced that cache. + env = stdlib_env(go, env) + if go.mode.pgoprofile: compile_args.add("-pgoprofile", go.mode.pgoprofile) inputs_direct.append(go.mode.pgoprofile) diff --git a/third_party/rgo/v0_60_0/base/go/private/actions/stdlib.bzl b/third_party/rgo/v0_60_0/base/go/private/actions/stdlib.bzl index ebf389c6..0833c392 100644 --- a/third_party/rgo/v0_60_0/base/go/private/actions/stdlib.bzl +++ b/third_party/rgo/v0_60_0/base/go/private/actions/stdlib.bzl @@ -149,10 +149,8 @@ def _build_stdlib_list_json(go): ) return out, cache_dir -def _stdlib_list_env(go): - return _stdlib_env(go, go.env) - -def _stdlib_env(go, base_env): +def stdlib_env(go, base_env): + """Returns the canonical environment for actions that consume this stdlib.""" env = dict(base_env) if go.mode.pure: @@ -178,6 +176,9 @@ def _stdlib_env(go, base_env): return env +def _stdlib_list_env(go): + return stdlib_env(go, go.env) + def _stdlib_action_env(go, orchestrion_trace_version_file, orchestrion_proxy_root_marker, orchestrion_tool_version_file): return _orchestrion_action_env( go, @@ -243,7 +244,7 @@ def _build_shared_synthetic_testmain_helpers(go, goroot_file, stdlib_cache_dir): arguments = [args], env = _orchestrion_action_env( go, - _stdlib_env(go, go.env_for_path_mapping), + stdlib_env(go, go.env_for_path_mapping), orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None), orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None), orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None), diff --git a/third_party/rgo/v0_61_1/base/go/private/actions/compilepkg.bzl b/third_party/rgo/v0_61_1/base/go/private/actions/compilepkg.bzl index a77773d4..d369b7b5 100644 --- a/third_party/rgo/v0_61_1/base/go/private/actions/compilepkg.bzl +++ b/third_party/rgo/v0_61_1/base/go/private/actions/compilepkg.bzl @@ -17,6 +17,7 @@ load( "//go/private:mode.bzl", "link_mode_arg", ) +load("//go/private/actions:stdlib.bzl", "stdlib_env") load("//go/private/actions:utils.bzl", "quote_opts") load("//go/private/orchestrion:pin_files.bzl", "OrchestrionPinFilesInfo") @@ -254,13 +255,13 @@ def emit_compilepkg( if link_mode_flag: compile_args.add("-asmflags", link_mode_flag) - # cgo and the linker action don't support path mapping yet - # TODO: Remove the second condition after https://github.com/bazelbuild/bazel/pull/21921. + # cgo and nested Go commands that may invoke it do not support path mapping. + # TODO: Remove the local-tag condition after https://github.com/bazelbuild/bazel/pull/21921. orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None) if compile_orchestrion else None orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None) if compile_orchestrion else None orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None) if compile_orchestrion else None - if cgo or "local" in go._ctx.attr.tags: - # cgo doesn't support path mapping yet + synthetic_testmain_needs_cgo = out_synthetic_testmain_manifest != None and not go.mode.pure + if cgo or synthetic_testmain_needs_cgo or "local" in go._ctx.attr.tags: env = _orchestrion_action_env( go, go.env, @@ -298,6 +299,14 @@ def emit_compilepkg( if objcxxopts: compile_args.add("-objcxxflags", quote_opts(objcxxopts)) + if synthetic_testmain_needs_cgo: + if not cgo: + inputs_transitive.append(go.cc_toolchain_files) + + # The nested Go commands consume the woven stdlib cache, so their cgo + # settings must match the action that produced that cache. + env = stdlib_env(go, env) + if go.mode.pgoprofile: compile_args.add("-pgoprofile", go.mode.pgoprofile) inputs_direct.append(go.mode.pgoprofile) diff --git a/third_party/rgo/v0_61_1/base/go/private/actions/stdlib.bzl b/third_party/rgo/v0_61_1/base/go/private/actions/stdlib.bzl index ebf389c6..0833c392 100644 --- a/third_party/rgo/v0_61_1/base/go/private/actions/stdlib.bzl +++ b/third_party/rgo/v0_61_1/base/go/private/actions/stdlib.bzl @@ -149,10 +149,8 @@ def _build_stdlib_list_json(go): ) return out, cache_dir -def _stdlib_list_env(go): - return _stdlib_env(go, go.env) - -def _stdlib_env(go, base_env): +def stdlib_env(go, base_env): + """Returns the canonical environment for actions that consume this stdlib.""" env = dict(base_env) if go.mode.pure: @@ -178,6 +176,9 @@ def _stdlib_env(go, base_env): return env +def _stdlib_list_env(go): + return stdlib_env(go, go.env) + def _stdlib_action_env(go, orchestrion_trace_version_file, orchestrion_proxy_root_marker, orchestrion_tool_version_file): return _orchestrion_action_env( go, @@ -243,7 +244,7 @@ def _build_shared_synthetic_testmain_helpers(go, goroot_file, stdlib_cache_dir): arguments = [args], env = _orchestrion_action_env( go, - _stdlib_env(go, go.env_for_path_mapping), + stdlib_env(go, go.env_for_path_mapping), orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None), orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None), orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None), diff --git a/third_party/rgo/v0_62_0/base/go/private/actions/compilepkg.bzl b/third_party/rgo/v0_62_0/base/go/private/actions/compilepkg.bzl index a77773d4..d369b7b5 100644 --- a/third_party/rgo/v0_62_0/base/go/private/actions/compilepkg.bzl +++ b/third_party/rgo/v0_62_0/base/go/private/actions/compilepkg.bzl @@ -17,6 +17,7 @@ load( "//go/private:mode.bzl", "link_mode_arg", ) +load("//go/private/actions:stdlib.bzl", "stdlib_env") load("//go/private/actions:utils.bzl", "quote_opts") load("//go/private/orchestrion:pin_files.bzl", "OrchestrionPinFilesInfo") @@ -254,13 +255,13 @@ def emit_compilepkg( if link_mode_flag: compile_args.add("-asmflags", link_mode_flag) - # cgo and the linker action don't support path mapping yet - # TODO: Remove the second condition after https://github.com/bazelbuild/bazel/pull/21921. + # cgo and nested Go commands that may invoke it do not support path mapping. + # TODO: Remove the local-tag condition after https://github.com/bazelbuild/bazel/pull/21921. orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None) if compile_orchestrion else None orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None) if compile_orchestrion else None orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None) if compile_orchestrion else None - if cgo or "local" in go._ctx.attr.tags: - # cgo doesn't support path mapping yet + synthetic_testmain_needs_cgo = out_synthetic_testmain_manifest != None and not go.mode.pure + if cgo or synthetic_testmain_needs_cgo or "local" in go._ctx.attr.tags: env = _orchestrion_action_env( go, go.env, @@ -298,6 +299,14 @@ def emit_compilepkg( if objcxxopts: compile_args.add("-objcxxflags", quote_opts(objcxxopts)) + if synthetic_testmain_needs_cgo: + if not cgo: + inputs_transitive.append(go.cc_toolchain_files) + + # The nested Go commands consume the woven stdlib cache, so their cgo + # settings must match the action that produced that cache. + env = stdlib_env(go, env) + if go.mode.pgoprofile: compile_args.add("-pgoprofile", go.mode.pgoprofile) inputs_direct.append(go.mode.pgoprofile) diff --git a/third_party/rgo/v0_62_0/base/go/private/actions/stdlib.bzl b/third_party/rgo/v0_62_0/base/go/private/actions/stdlib.bzl index 955916a7..e3d06349 100644 --- a/third_party/rgo/v0_62_0/base/go/private/actions/stdlib.bzl +++ b/third_party/rgo/v0_62_0/base/go/private/actions/stdlib.bzl @@ -149,10 +149,8 @@ def _build_stdlib_list_json(go): ) return out, cache_dir -def _stdlib_list_env(go): - return _stdlib_env(go, go.env) - -def _stdlib_env(go, base_env): +def stdlib_env(go, base_env): + """Returns the canonical environment for actions that consume this stdlib.""" env = dict(base_env) if go.mode.pure: @@ -178,6 +176,9 @@ def _stdlib_env(go, base_env): return env +def _stdlib_list_env(go): + return stdlib_env(go, go.env) + def _stdlib_action_env(go, orchestrion_trace_version_file, orchestrion_proxy_root_marker, orchestrion_tool_version_file): return _orchestrion_action_env( go, @@ -244,7 +245,7 @@ def _build_shared_synthetic_testmain_helpers(go, goroot_file, stdlib_cache_dir): arguments = [args], env = _orchestrion_action_env( go, - _stdlib_env(go, go.env_for_path_mapping), + stdlib_env(go, go.env_for_path_mapping), orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None), orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None), orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None), diff --git a/third_party/rgo/v0_63_0/base/go/private/actions/compilepkg.bzl b/third_party/rgo/v0_63_0/base/go/private/actions/compilepkg.bzl index a77773d4..d369b7b5 100644 --- a/third_party/rgo/v0_63_0/base/go/private/actions/compilepkg.bzl +++ b/third_party/rgo/v0_63_0/base/go/private/actions/compilepkg.bzl @@ -17,6 +17,7 @@ load( "//go/private:mode.bzl", "link_mode_arg", ) +load("//go/private/actions:stdlib.bzl", "stdlib_env") load("//go/private/actions:utils.bzl", "quote_opts") load("//go/private/orchestrion:pin_files.bzl", "OrchestrionPinFilesInfo") @@ -254,13 +255,13 @@ def emit_compilepkg( if link_mode_flag: compile_args.add("-asmflags", link_mode_flag) - # cgo and the linker action don't support path mapping yet - # TODO: Remove the second condition after https://github.com/bazelbuild/bazel/pull/21921. + # cgo and nested Go commands that may invoke it do not support path mapping. + # TODO: Remove the local-tag condition after https://github.com/bazelbuild/bazel/pull/21921. orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None) if compile_orchestrion else None orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None) if compile_orchestrion else None orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None) if compile_orchestrion else None - if cgo or "local" in go._ctx.attr.tags: - # cgo doesn't support path mapping yet + synthetic_testmain_needs_cgo = out_synthetic_testmain_manifest != None and not go.mode.pure + if cgo or synthetic_testmain_needs_cgo or "local" in go._ctx.attr.tags: env = _orchestrion_action_env( go, go.env, @@ -298,6 +299,14 @@ def emit_compilepkg( if objcxxopts: compile_args.add("-objcxxflags", quote_opts(objcxxopts)) + if synthetic_testmain_needs_cgo: + if not cgo: + inputs_transitive.append(go.cc_toolchain_files) + + # The nested Go commands consume the woven stdlib cache, so their cgo + # settings must match the action that produced that cache. + env = stdlib_env(go, env) + if go.mode.pgoprofile: compile_args.add("-pgoprofile", go.mode.pgoprofile) inputs_direct.append(go.mode.pgoprofile) diff --git a/third_party/rgo/v0_63_0/base/go/private/actions/stdlib.bzl b/third_party/rgo/v0_63_0/base/go/private/actions/stdlib.bzl index ed34125c..96779f81 100644 --- a/third_party/rgo/v0_63_0/base/go/private/actions/stdlib.bzl +++ b/third_party/rgo/v0_63_0/base/go/private/actions/stdlib.bzl @@ -135,10 +135,8 @@ def _stdlib_execution_requirements(go): # CGO_CFLAGS/CGO_LDFLAGS. Bazel does not path-map environment values. return SUPPORTS_PATH_MAPPING_REQUIREMENT if go.mode.pure else {} -def _stdlib_list_env(go): - return _stdlib_env(go, go.env) - -def _stdlib_env(go, base_env): +def stdlib_env(go, base_env): + """Returns the canonical environment for actions that consume this stdlib.""" env = dict(base_env) if go.mode.pure: @@ -164,6 +162,9 @@ def _stdlib_env(go, base_env): return env +def _stdlib_list_env(go): + return stdlib_env(go, go.env) + def _stdlib_action_env(go, orchestrion_trace_version_file, orchestrion_proxy_root_marker, orchestrion_tool_version_file): return _orchestrion_action_env( go, @@ -219,7 +220,7 @@ def _build_shared_synthetic_testmain_helpers(go, goroot_file, stdlib_cache_dir): arguments = [args], env = _orchestrion_action_env( go, - _stdlib_env(go, go.env_for_path_mapping), + stdlib_env(go, go.env_for_path_mapping), orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None), orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None), orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None), diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index 44d3f669..7462efe0 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -1231,12 +1231,14 @@ index 8b17c19..585be0e 100644 runfiles = source.runfiles, _validation_output = out_nogo_validation, diff --git a/go/private/actions/compilepkg.bzl b/go/private/actions/compilepkg.bzl -index a339eef..036bf48 100644 +index a339eef..915610d 100644 --- a/go/private/actions/compilepkg.bzl +++ b/go/private/actions/compilepkg.bzl -@@ -18,6 +18,13 @@ load( +@@ -17,7 +17,15 @@ load( + "//go/private:mode.bzl", "link_mode_arg", ) ++load("//go/private/actions:stdlib.bzl", "stdlib_env") load("//go/private/actions:utils.bzl", "quote_opts") +load("//go/private/orchestrion:pin_files.bzl", "OrchestrionPinFilesInfo") + @@ -1248,7 +1250,7 @@ index a339eef..036bf48 100644 def _archive(v): importpaths = [v.data.importpath] -@@ -49,6 +56,69 @@ def _embedlookupdir_arg(src): +@@ -49,6 +57,69 @@ def _embedlookupdir_arg(src): root_relative = root_relative[1:] return root_relative @@ -1318,7 +1320,7 @@ index a339eef..036bf48 100644 def emit_compilepkg( go, sources = None, -@@ -69,6 +139,8 @@ def emit_compilepkg( +@@ -69,6 +140,8 @@ def emit_compilepkg( clinkopts = [], out_lib = None, out_export = None, @@ -1327,7 +1329,7 @@ index a339eef..036bf48 100644 out_facts = None, out_diagnostics = None, out_nogo_validation = None, -@@ -94,10 +166,17 @@ def emit_compilepkg( +@@ -94,10 +167,17 @@ def emit_compilepkg( archives = archives + [go.coverdata] sdk = go.sdk @@ -1345,7 +1347,7 @@ index a339eef..036bf48 100644 shared_args = go.builder_args(go) shared_args.add_all(sources, before_each = "-src") -@@ -140,6 +219,18 @@ def emit_compilepkg( +@@ -140,6 +220,18 @@ def emit_compilepkg( compile_args.add("-lo", out_lib) compile_args.add("-o", out_export) @@ -1364,16 +1366,22 @@ index a339eef..036bf48 100644 if out_cgo_export_h: compile_args.add("-cgoexport", out_cgo_export_h) outputs.append(out_cgo_export_h) -@@ -165,12 +256,27 @@ def emit_compilepkg( +@@ -163,14 +255,29 @@ def emit_compilepkg( + if link_mode_flag: + compile_args.add("-asmflags", link_mode_flag) - # cgo and the linker action don't support path mapping yet - # TODO: Remove the second condition after https://github.com/bazelbuild/bazel/pull/21921. +- # cgo and the linker action don't support path mapping yet +- # TODO: Remove the second condition after https://github.com/bazelbuild/bazel/pull/21921. +- if cgo or "local" in go._ctx.attr.tags: +- # cgo doesn't support path mapping yet +- env = go.env ++ # cgo and nested Go commands that may invoke it do not support path mapping. ++ # TODO: Remove the local-tag condition after https://github.com/bazelbuild/bazel/pull/21921. + orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None) if compile_orchestrion else None + orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None) if compile_orchestrion else None + orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None) if compile_orchestrion else None - if cgo or "local" in go._ctx.attr.tags: - # cgo doesn't support path mapping yet -- env = go.env ++ synthetic_testmain_needs_cgo = out_synthetic_testmain_manifest != None and not go.mode.pure ++ if cgo or synthetic_testmain_needs_cgo or "local" in go._ctx.attr.tags: + env = _orchestrion_action_env( + go, + go.env, @@ -1394,7 +1402,19 @@ index a339eef..036bf48 100644 execution_requirements = SUPPORTS_PATH_MAPPING_REQUIREMENT cgo_go_srcs = None if cgo: -@@ -198,6 +304,44 @@ def emit_compilepkg( +@@ -194,10 +301,56 @@ def emit_compilepkg( + if clinkopts: + compile_args.add("-ldflags", quote_opts(clinkopts)) + ++ if synthetic_testmain_needs_cgo: ++ if not cgo: ++ inputs_transitive.append(go.cc_toolchain_files) ++ ++ # The nested Go commands consume the woven stdlib cache, so their cgo ++ # settings must match the action that produced that cache. ++ env = stdlib_env(go, env) ++ + if go.mode.pgoprofile: compile_args.add("-pgoprofile", go.mode.pgoprofile) inputs_direct.append(go.mode.pgoprofile) @@ -1634,7 +1654,7 @@ index 7d8ff5f..ee189eb 100644 ) diff --git a/go/private/actions/stdlib.bzl b/go/private/actions/stdlib.bzl -index cd028a3..ebf389c 100644 +index cd028a3..0833c39 100644 --- a/go/private/actions/stdlib.bzl +++ b/go/private/actions/stdlib.bzl @@ -27,6 +27,7 @@ load( @@ -1725,7 +1745,7 @@ index cd028a3..ebf389c 100644 args = go.builder_args(go, "stdliblist") args.add("-out", out) args.add_all("-cache", [cache_dir], expand_directories = False) -@@ -85,14 +143,17 @@ def _build_stdlib_list_json(go): +@@ -85,14 +143,15 @@ def _build_stdlib_list_json(go): mnemonic = "GoStdlibList", executable = go.toolchain._builder, arguments = [args], @@ -1738,18 +1758,19 @@ index cd028a3..ebf389c 100644 -def _build_env(go): - env = go.env -+def _stdlib_list_env(go): -+ return _stdlib_env(go, go.env) -+ -+def _stdlib_env(go, base_env): ++def stdlib_env(go, base_env): ++ """Returns the canonical environment for actions that consume this stdlib.""" + env = dict(base_env) if go.mode.pure: env.update({"CGO_ENABLED": "0"}) -@@ -117,10 +178,20 @@ def _build_env(go): +@@ -117,10 +176,23 @@ def _build_env(go): return env ++def _stdlib_list_env(go): ++ return stdlib_env(go, go.env) ++ +def _stdlib_action_env(go, orchestrion_trace_version_file, orchestrion_proxy_root_marker, orchestrion_tool_version_file): + return _orchestrion_action_env( + go, @@ -1767,7 +1788,7 @@ index cd028a3..ebf389c 100644 cache_dir = depset([cache_dir]), libs = go.sdk.libs, root_file = go.sdk.root_file, -@@ -129,13 +200,68 @@ def _sdk_stdlib(go): +@@ -129,13 +201,68 @@ def _sdk_stdlib(go): def _dirname(file): return file.dirname @@ -1814,7 +1835,7 @@ index cd028a3..ebf389c 100644 + arguments = [args], + env = _orchestrion_action_env( + go, -+ _stdlib_env(go, go.env_for_path_mapping), ++ stdlib_env(go, go.env_for_path_mapping), + orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None), + orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None), + orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None), @@ -1836,7 +1857,7 @@ index cd028a3..ebf389c 100644 if go.mode.race: args.add("-race") if go.mode.msan: -@@ -164,21 +290,62 @@ def _build_stdlib(go): +@@ -164,21 +291,62 @@ def _build_stdlib(go): args.add("-pgoprofile", go.mode.pgoprofile) inputs_direct.append(go.mode.pgoprofile) diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index e65dd054..cb633c0d 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -1215,12 +1215,14 @@ index 3a5a76b..dd69fb7 100644 runfiles = source.runfiles, _validation_output = out_nogo_validation, diff --git a/go/private/actions/compilepkg.bzl b/go/private/actions/compilepkg.bzl -index 9c067b2..a77773d 100644 +index 9c067b2..d369b7b 100644 --- a/go/private/actions/compilepkg.bzl +++ b/go/private/actions/compilepkg.bzl -@@ -18,6 +18,13 @@ load( +@@ -17,7 +17,15 @@ load( + "//go/private:mode.bzl", "link_mode_arg", ) ++load("//go/private/actions:stdlib.bzl", "stdlib_env") load("//go/private/actions:utils.bzl", "quote_opts") +load("//go/private/orchestrion:pin_files.bzl", "OrchestrionPinFilesInfo") + @@ -1232,7 +1234,7 @@ index 9c067b2..a77773d 100644 def _archive(v): importpaths = [v.data.importpath] -@@ -49,6 +56,69 @@ def _embedlookupdir_arg(src): +@@ -49,6 +57,69 @@ def _embedlookupdir_arg(src): root_relative = root_relative[1:] return root_relative @@ -1302,7 +1304,7 @@ index 9c067b2..a77773d 100644 def emit_compilepkg( go, sources = None, -@@ -69,6 +139,8 @@ def emit_compilepkg( +@@ -69,6 +140,8 @@ def emit_compilepkg( ldflags = None, out_lib = None, out_export = None, @@ -1311,7 +1313,7 @@ index 9c067b2..a77773d 100644 out_facts = None, out_diagnostics = None, out_nogo_validation = None, -@@ -94,10 +166,17 @@ def emit_compilepkg( +@@ -94,10 +167,17 @@ def emit_compilepkg( archives = archives + [go.coverdata] sdk = go.sdk @@ -1329,7 +1331,7 @@ index 9c067b2..a77773d 100644 shared_args = go.builder_args(go) shared_args.add_all(sources, before_each = "-src") -@@ -140,6 +219,18 @@ def emit_compilepkg( +@@ -140,6 +220,18 @@ def emit_compilepkg( compile_args.add("-lo", out_lib) compile_args.add("-o", out_export) @@ -1348,16 +1350,22 @@ index 9c067b2..a77773d 100644 if out_cgo_export_h: compile_args.add("-cgoexport", out_cgo_export_h) outputs.append(out_cgo_export_h) -@@ -165,12 +256,27 @@ def emit_compilepkg( +@@ -163,14 +255,29 @@ def emit_compilepkg( + if link_mode_flag: + compile_args.add("-asmflags", link_mode_flag) - # cgo and the linker action don't support path mapping yet - # TODO: Remove the second condition after https://github.com/bazelbuild/bazel/pull/21921. +- # cgo and the linker action don't support path mapping yet +- # TODO: Remove the second condition after https://github.com/bazelbuild/bazel/pull/21921. +- if cgo or "local" in go._ctx.attr.tags: +- # cgo doesn't support path mapping yet +- env = go.env ++ # cgo and nested Go commands that may invoke it do not support path mapping. ++ # TODO: Remove the local-tag condition after https://github.com/bazelbuild/bazel/pull/21921. + orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None) if compile_orchestrion else None + orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None) if compile_orchestrion else None + orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None) if compile_orchestrion else None - if cgo or "local" in go._ctx.attr.tags: - # cgo doesn't support path mapping yet -- env = go.env ++ synthetic_testmain_needs_cgo = out_synthetic_testmain_manifest != None and not go.mode.pure ++ if cgo or synthetic_testmain_needs_cgo or "local" in go._ctx.attr.tags: + env = _orchestrion_action_env( + go, + go.env, @@ -1378,7 +1386,19 @@ index 9c067b2..a77773d 100644 execution_requirements = SUPPORTS_PATH_MAPPING_REQUIREMENT cgo_go_srcs = None if cgo: -@@ -196,6 +302,44 @@ def emit_compilepkg( +@@ -192,10 +299,56 @@ def emit_compilepkg( + if objcxxopts: + compile_args.add("-objcxxflags", quote_opts(objcxxopts)) + ++ if synthetic_testmain_needs_cgo: ++ if not cgo: ++ inputs_transitive.append(go.cc_toolchain_files) ++ ++ # The nested Go commands consume the woven stdlib cache, so their cgo ++ # settings must match the action that produced that cache. ++ env = stdlib_env(go, env) ++ + if go.mode.pgoprofile: compile_args.add("-pgoprofile", go.mode.pgoprofile) inputs_direct.append(go.mode.pgoprofile) @@ -1618,7 +1638,7 @@ index 61f076f..0e39764 100644 ) diff --git a/go/private/actions/stdlib.bzl b/go/private/actions/stdlib.bzl -index cd028a3..ebf389c 100644 +index cd028a3..0833c39 100644 --- a/go/private/actions/stdlib.bzl +++ b/go/private/actions/stdlib.bzl @@ -27,6 +27,7 @@ load( @@ -1709,7 +1729,7 @@ index cd028a3..ebf389c 100644 args = go.builder_args(go, "stdliblist") args.add("-out", out) args.add_all("-cache", [cache_dir], expand_directories = False) -@@ -85,14 +143,17 @@ def _build_stdlib_list_json(go): +@@ -85,14 +143,15 @@ def _build_stdlib_list_json(go): mnemonic = "GoStdlibList", executable = go.toolchain._builder, arguments = [args], @@ -1722,18 +1742,19 @@ index cd028a3..ebf389c 100644 -def _build_env(go): - env = go.env -+def _stdlib_list_env(go): -+ return _stdlib_env(go, go.env) -+ -+def _stdlib_env(go, base_env): ++def stdlib_env(go, base_env): ++ """Returns the canonical environment for actions that consume this stdlib.""" + env = dict(base_env) if go.mode.pure: env.update({"CGO_ENABLED": "0"}) -@@ -117,10 +178,20 @@ def _build_env(go): +@@ -117,10 +176,23 @@ def _build_env(go): return env ++def _stdlib_list_env(go): ++ return stdlib_env(go, go.env) ++ +def _stdlib_action_env(go, orchestrion_trace_version_file, orchestrion_proxy_root_marker, orchestrion_tool_version_file): + return _orchestrion_action_env( + go, @@ -1751,7 +1772,7 @@ index cd028a3..ebf389c 100644 cache_dir = depset([cache_dir]), libs = go.sdk.libs, root_file = go.sdk.root_file, -@@ -129,13 +200,68 @@ def _sdk_stdlib(go): +@@ -129,13 +201,68 @@ def _sdk_stdlib(go): def _dirname(file): return file.dirname @@ -1798,7 +1819,7 @@ index cd028a3..ebf389c 100644 + arguments = [args], + env = _orchestrion_action_env( + go, -+ _stdlib_env(go, go.env_for_path_mapping), ++ stdlib_env(go, go.env_for_path_mapping), + orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None), + orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None), + orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None), @@ -1820,7 +1841,7 @@ index cd028a3..ebf389c 100644 if go.mode.race: args.add("-race") if go.mode.msan: -@@ -164,21 +290,62 @@ def _build_stdlib(go): +@@ -164,21 +291,62 @@ def _build_stdlib(go): args.add("-pgoprofile", go.mode.pgoprofile) inputs_direct.append(go.mode.pgoprofile) diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index a1eaeb35..97511103 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -1215,12 +1215,14 @@ index 3a5a76b..dd69fb7 100644 runfiles = source.runfiles, _validation_output = out_nogo_validation, diff --git a/go/private/actions/compilepkg.bzl b/go/private/actions/compilepkg.bzl -index 9c067b2..a77773d 100644 +index 9c067b2..d369b7b 100644 --- a/go/private/actions/compilepkg.bzl +++ b/go/private/actions/compilepkg.bzl -@@ -18,6 +18,13 @@ load( +@@ -17,7 +17,15 @@ load( + "//go/private:mode.bzl", "link_mode_arg", ) ++load("//go/private/actions:stdlib.bzl", "stdlib_env") load("//go/private/actions:utils.bzl", "quote_opts") +load("//go/private/orchestrion:pin_files.bzl", "OrchestrionPinFilesInfo") + @@ -1232,7 +1234,7 @@ index 9c067b2..a77773d 100644 def _archive(v): importpaths = [v.data.importpath] -@@ -49,6 +56,69 @@ def _embedlookupdir_arg(src): +@@ -49,6 +57,69 @@ def _embedlookupdir_arg(src): root_relative = root_relative[1:] return root_relative @@ -1302,7 +1304,7 @@ index 9c067b2..a77773d 100644 def emit_compilepkg( go, sources = None, -@@ -69,6 +139,8 @@ def emit_compilepkg( +@@ -69,6 +140,8 @@ def emit_compilepkg( ldflags = None, out_lib = None, out_export = None, @@ -1311,7 +1313,7 @@ index 9c067b2..a77773d 100644 out_facts = None, out_diagnostics = None, out_nogo_validation = None, -@@ -94,10 +166,17 @@ def emit_compilepkg( +@@ -94,10 +167,17 @@ def emit_compilepkg( archives = archives + [go.coverdata] sdk = go.sdk @@ -1329,7 +1331,7 @@ index 9c067b2..a77773d 100644 shared_args = go.builder_args(go) shared_args.add_all(sources, before_each = "-src") -@@ -140,6 +219,18 @@ def emit_compilepkg( +@@ -140,6 +220,18 @@ def emit_compilepkg( compile_args.add("-lo", out_lib) compile_args.add("-o", out_export) @@ -1348,16 +1350,22 @@ index 9c067b2..a77773d 100644 if out_cgo_export_h: compile_args.add("-cgoexport", out_cgo_export_h) outputs.append(out_cgo_export_h) -@@ -165,12 +256,27 @@ def emit_compilepkg( +@@ -163,14 +255,29 @@ def emit_compilepkg( + if link_mode_flag: + compile_args.add("-asmflags", link_mode_flag) - # cgo and the linker action don't support path mapping yet - # TODO: Remove the second condition after https://github.com/bazelbuild/bazel/pull/21921. +- # cgo and the linker action don't support path mapping yet +- # TODO: Remove the second condition after https://github.com/bazelbuild/bazel/pull/21921. +- if cgo or "local" in go._ctx.attr.tags: +- # cgo doesn't support path mapping yet +- env = go.env ++ # cgo and nested Go commands that may invoke it do not support path mapping. ++ # TODO: Remove the local-tag condition after https://github.com/bazelbuild/bazel/pull/21921. + orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None) if compile_orchestrion else None + orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None) if compile_orchestrion else None + orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None) if compile_orchestrion else None - if cgo or "local" in go._ctx.attr.tags: - # cgo doesn't support path mapping yet -- env = go.env ++ synthetic_testmain_needs_cgo = out_synthetic_testmain_manifest != None and not go.mode.pure ++ if cgo or synthetic_testmain_needs_cgo or "local" in go._ctx.attr.tags: + env = _orchestrion_action_env( + go, + go.env, @@ -1378,7 +1386,19 @@ index 9c067b2..a77773d 100644 execution_requirements = SUPPORTS_PATH_MAPPING_REQUIREMENT cgo_go_srcs = None if cgo: -@@ -196,6 +302,44 @@ def emit_compilepkg( +@@ -192,10 +299,56 @@ def emit_compilepkg( + if objcxxopts: + compile_args.add("-objcxxflags", quote_opts(objcxxopts)) + ++ if synthetic_testmain_needs_cgo: ++ if not cgo: ++ inputs_transitive.append(go.cc_toolchain_files) ++ ++ # The nested Go commands consume the woven stdlib cache, so their cgo ++ # settings must match the action that produced that cache. ++ env = stdlib_env(go, env) ++ + if go.mode.pgoprofile: compile_args.add("-pgoprofile", go.mode.pgoprofile) inputs_direct.append(go.mode.pgoprofile) @@ -1618,7 +1638,7 @@ index c249ca7..10a9443 100644 exec_group = exec_group, ) diff --git a/go/private/actions/stdlib.bzl b/go/private/actions/stdlib.bzl -index 0974749..955916a 100644 +index 0974749..e3d0634 100644 --- a/go/private/actions/stdlib.bzl +++ b/go/private/actions/stdlib.bzl @@ -27,6 +27,7 @@ load( @@ -1709,7 +1729,7 @@ index 0974749..955916a 100644 args = go.builder_args(go, "stdliblist") args.add("-out", out) args.add_all("-cache", [cache_dir], expand_directories = False) -@@ -85,14 +143,17 @@ def _build_stdlib_list_json(go): +@@ -85,14 +143,15 @@ def _build_stdlib_list_json(go): mnemonic = "GoStdlibList", executable = go.toolchain._builder, arguments = [args], @@ -1722,18 +1742,19 @@ index 0974749..955916a 100644 -def _build_env(go): - env = go.env -+def _stdlib_list_env(go): -+ return _stdlib_env(go, go.env) -+ -+def _stdlib_env(go, base_env): ++def stdlib_env(go, base_env): ++ """Returns the canonical environment for actions that consume this stdlib.""" + env = dict(base_env) if go.mode.pure: env.update({"CGO_ENABLED": "0"}) -@@ -117,10 +178,20 @@ def _build_env(go): +@@ -117,10 +176,23 @@ def _build_env(go): return env ++def _stdlib_list_env(go): ++ return stdlib_env(go, go.env) ++ +def _stdlib_action_env(go, orchestrion_trace_version_file, orchestrion_proxy_root_marker, orchestrion_tool_version_file): + return _orchestrion_action_env( + go, @@ -1751,7 +1772,7 @@ index 0974749..955916a 100644 cgo_link_inputs = depset(), cache_dir = depset([cache_dir]), libs = go.sdk.libs, -@@ -130,13 +201,68 @@ def _sdk_stdlib(go): +@@ -130,13 +202,68 @@ def _sdk_stdlib(go): def _dirname(file): return file.dirname @@ -1798,7 +1819,7 @@ index 0974749..955916a 100644 + arguments = [args], + env = _orchestrion_action_env( + go, -+ _stdlib_env(go, go.env_for_path_mapping), ++ stdlib_env(go, go.env_for_path_mapping), + orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None), + orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None), + orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None), @@ -1820,7 +1841,7 @@ index 0974749..955916a 100644 if go.mode.race: args.add("-race") if go.mode.msan: -@@ -165,25 +291,66 @@ def _build_stdlib(go): +@@ -165,25 +292,66 @@ def _build_stdlib(go): args.add("-pgoprofile", go.mode.pgoprofile) inputs_direct.append(go.mode.pgoprofile) diff --git a/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch index bab8cb61..7dddb9a6 100644 --- a/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch @@ -258,12 +258,14 @@ index b7a6af6..9dd382c 100644 runfiles = source.runfiles, _validation_output = out_nogo_validation, diff --git a/go/private/actions/compilepkg.bzl b/go/private/actions/compilepkg.bzl -index 9c067b2..a77773d 100644 +index 9c067b2..d369b7b 100644 --- a/go/private/actions/compilepkg.bzl +++ b/go/private/actions/compilepkg.bzl -@@ -18,6 +18,13 @@ load( +@@ -17,7 +17,15 @@ load( + "//go/private:mode.bzl", "link_mode_arg", ) ++load("//go/private/actions:stdlib.bzl", "stdlib_env") load("//go/private/actions:utils.bzl", "quote_opts") +load("//go/private/orchestrion:pin_files.bzl", "OrchestrionPinFilesInfo") + @@ -275,7 +277,7 @@ index 9c067b2..a77773d 100644 def _archive(v): importpaths = [v.data.importpath] -@@ -49,6 +56,69 @@ def _embedlookupdir_arg(src): +@@ -49,6 +57,69 @@ def _embedlookupdir_arg(src): root_relative = root_relative[1:] return root_relative @@ -345,7 +347,7 @@ index 9c067b2..a77773d 100644 def emit_compilepkg( go, sources = None, -@@ -69,6 +139,8 @@ def emit_compilepkg( +@@ -69,6 +140,8 @@ def emit_compilepkg( ldflags = None, out_lib = None, out_export = None, @@ -354,7 +356,7 @@ index 9c067b2..a77773d 100644 out_facts = None, out_diagnostics = None, out_nogo_validation = None, -@@ -94,10 +166,17 @@ def emit_compilepkg( +@@ -94,10 +167,17 @@ def emit_compilepkg( archives = archives + [go.coverdata] sdk = go.sdk @@ -372,7 +374,7 @@ index 9c067b2..a77773d 100644 shared_args = go.builder_args(go) shared_args.add_all(sources, before_each = "-src") -@@ -140,6 +219,18 @@ def emit_compilepkg( +@@ -140,6 +220,18 @@ def emit_compilepkg( compile_args.add("-lo", out_lib) compile_args.add("-o", out_export) @@ -391,16 +393,22 @@ index 9c067b2..a77773d 100644 if out_cgo_export_h: compile_args.add("-cgoexport", out_cgo_export_h) outputs.append(out_cgo_export_h) -@@ -165,12 +256,27 @@ def emit_compilepkg( +@@ -163,14 +255,29 @@ def emit_compilepkg( + if link_mode_flag: + compile_args.add("-asmflags", link_mode_flag) - # cgo and the linker action don't support path mapping yet - # TODO: Remove the second condition after https://github.com/bazelbuild/bazel/pull/21921. +- # cgo and the linker action don't support path mapping yet +- # TODO: Remove the second condition after https://github.com/bazelbuild/bazel/pull/21921. +- if cgo or "local" in go._ctx.attr.tags: +- # cgo doesn't support path mapping yet +- env = go.env ++ # cgo and nested Go commands that may invoke it do not support path mapping. ++ # TODO: Remove the local-tag condition after https://github.com/bazelbuild/bazel/pull/21921. + orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None) if compile_orchestrion else None + orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None) if compile_orchestrion else None + orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None) if compile_orchestrion else None - if cgo or "local" in go._ctx.attr.tags: - # cgo doesn't support path mapping yet -- env = go.env ++ synthetic_testmain_needs_cgo = out_synthetic_testmain_manifest != None and not go.mode.pure ++ if cgo or synthetic_testmain_needs_cgo or "local" in go._ctx.attr.tags: + env = _orchestrion_action_env( + go, + go.env, @@ -421,7 +429,19 @@ index 9c067b2..a77773d 100644 execution_requirements = SUPPORTS_PATH_MAPPING_REQUIREMENT cgo_go_srcs = None if cgo: -@@ -196,6 +302,44 @@ def emit_compilepkg( +@@ -192,10 +299,56 @@ def emit_compilepkg( + if objcxxopts: + compile_args.add("-objcxxflags", quote_opts(objcxxopts)) + ++ if synthetic_testmain_needs_cgo: ++ if not cgo: ++ inputs_transitive.append(go.cc_toolchain_files) ++ ++ # The nested Go commands consume the woven stdlib cache, so their cgo ++ # settings must match the action that produced that cache. ++ env = stdlib_env(go, env) ++ + if go.mode.pgoprofile: compile_args.add("-pgoprofile", go.mode.pgoprofile) inputs_direct.append(go.mode.pgoprofile) @@ -661,7 +681,7 @@ index 8574a43..07844d4 100644 exec_group = exec_group, ) diff --git a/go/private/actions/stdlib.bzl b/go/private/actions/stdlib.bzl -index 06e21b7..ed34125 100644 +index 06e21b7..96779f8 100644 --- a/go/private/actions/stdlib.bzl +++ b/go/private/actions/stdlib.bzl @@ -25,6 +25,7 @@ load( @@ -752,24 +772,25 @@ index 06e21b7..ed34125 100644 toolchain = GO_TOOLCHAIN_LABEL, execution_requirements = _stdlib_execution_requirements(go), ) -@@ -79,8 +135,11 @@ def _stdlib_execution_requirements(go): +@@ -79,8 +135,9 @@ def _stdlib_execution_requirements(go): # CGO_CFLAGS/CGO_LDFLAGS. Bazel does not path-map environment values. return SUPPORTS_PATH_MAPPING_REQUIREMENT if go.mode.pure else {} -def _build_env(go): - env = go.env -+def _stdlib_list_env(go): -+ return _stdlib_env(go, go.env) -+ -+def _stdlib_env(go, base_env): ++def stdlib_env(go, base_env): ++ """Returns the canonical environment for actions that consume this stdlib.""" + env = dict(base_env) if go.mode.pure: env.update({"CGO_ENABLED": "0"}) -@@ -105,16 +164,80 @@ def _build_env(go): +@@ -105,16 +162,83 @@ def _build_env(go): return env ++def _stdlib_list_env(go): ++ return stdlib_env(go, go.env) ++ +def _stdlib_action_env(go, orchestrion_trace_version_file, orchestrion_proxy_root_marker, orchestrion_tool_version_file): + return _orchestrion_action_env( + go, @@ -825,7 +846,7 @@ index 06e21b7..ed34125 100644 + arguments = [args], + env = _orchestrion_action_env( + go, -+ _stdlib_env(go, go.env_for_path_mapping), ++ stdlib_env(go, go.env_for_path_mapping), + orchestrion_trace_version_file = getattr(go, "orchestrion_version_file", None), + orchestrion_proxy_root_marker = getattr(go, "orchestrion_module_proxy_root_marker", None), + orchestrion_tool_version_file = getattr(go, "orchestrion_tool_version_file", None), @@ -847,7 +868,7 @@ index 06e21b7..ed34125 100644 if go.mode.race: args.add("-race") if go.mode.msan: -@@ -141,25 +264,66 @@ def _build_stdlib(go): +@@ -141,25 +265,66 @@ def _build_stdlib(go): args.add("-pgoprofile", go.mode.pgoprofile) inputs_direct.append(go.mode.pgoprofile) diff --git a/tools/tests/integration/assert_orchestrion_module_proxy_aquery.py b/tools/tests/integration/assert_orchestrion_module_proxy_aquery.py index fd1549cc..3043a11f 100755 --- a/tools/tests/integration/assert_orchestrion_module_proxy_aquery.py +++ b/tools/tests/integration/assert_orchestrion_module_proxy_aquery.py @@ -472,6 +472,39 @@ def _assert_shared_synthetic_testmain_helper_action( _require(name in action.environment, f"shared synthetic testmain helper action is missing {name}") +def _assert_synthetic_testmain_cgo_wiring( + action: Action, + inputs: list[str], + stdlib_action: Action, +) -> None: + """Require non-pure testmain helpers to use the stdlib's C toolchain.""" + + if action.environment.get("CGO_ENABLED") != "1": + return + + for name in ("CGO_ENABLED", "CC", "CGO_CFLAGS", "CGO_LDFLAGS"): + _require( + name in action.environment, + f"non-pure synthetic testmain compile action is missing {name}", + ) + _require( + action.environment[name] == stdlib_action.environment.get(name), + f"synthetic testmain compile action does not match GoStdlib {name}", + ) + + c_compiler = action.environment["CC"] + if not Path(c_compiler).is_absolute(): + _require( + _contains_path_suffix(inputs, c_compiler), + "non-pure synthetic testmain compile action does not declare " + f"its C compiler input: {c_compiler}", + ) + _require( + "supports-path-mapping" not in action.execution_info, + "non-pure synthetic testmain compile action incorrectly supports path mapping", + ) + + def _assert_reduced_synthetic_testmain_link_inputs( action: Action, inputs: list[str], @@ -838,6 +871,11 @@ def main() -> int: _contains_path_fragment(inputs, "stdlib_/synthetic_testmain_helpers"), "synthetic testmain compile action is missing the shared helper bundle input", ) + _assert_synthetic_testmain_cgo_wiring( + action, + inputs, + orchestrion_stdlib_actions[0], + ) if args.require_reduced_synthetic_testmain_link_inputs: synthetic_testmain_link_actions = [ action for action in link_actions From 25bfd417da2ea70179f7b135b012d37b2ec1d74e Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Thu, 10 Sep 2026 12:49:42 +0200 Subject: [PATCH 4/6] rules_go: verify instrumented build determinism --- CONTRIBUTING.md | 1 + .../rgo/v0_60_0/base/go/tools/builders/cc.go | 51 +- .../v0_60_0/base/go/tools/builders/cc_test.go | 30 +- .../rgo/v0_61_1/base/go/tools/builders/cc.go | 51 +- .../v0_61_1/base/go/tools/builders/cc_test.go | 30 +- .../rgo/v0_62_0/base/go/tools/builders/cc.go | 51 +- .../v0_62_0/base/go/tools/builders/cc_test.go | 30 +- .../rgo/v0_63_0/base/go/tools/builders/cc.go | 51 +- .../v0_63_0/base/go/tools/builders/cc_test.go | 30 +- .../v0_60_0/base/0001-full-delta.patch | 89 ++- .../v0_61_1/base/0001-full-delta.patch | 89 ++- .../v0_62_0/base/0001-full-delta.patch | 89 ++- .../v0_63_0/base/0001-full-delta.patch | 89 ++- tools/dev/BUILD.bazel | 1 + tools/dev/check_release_archive_contents.py | 1 + tools/dev/compact_execution_log.py | 275 +++++++++ tools/dev/verify_rules_go_profiles.py | 560 +++++++++++------- tools/tests/python/BUILD.bazel | 1 + .../python/test_rules_go_profile_tools.py | 210 ++++++- 19 files changed, 1435 insertions(+), 294 deletions(-) create mode 100644 tools/dev/compact_execution_log.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ee498ad7..1750de2f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,6 +70,7 @@ This product includes software developed at Datadog - `shellcheck` (shell lint lane) - `buildifier` (Starlark formatting lane) - `jq` (integration harness payload/CODEOWNERS checks) + - `zstd` (rules_go compact execution-log verification) - Optional pre-commit setup: - `python3 -m pip install pre-commit && pre-commit install` - Optional Python syntax smoke check when editing tooling: diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/cc.go b/third_party/rgo/v0_60_0/base/go/tools/builders/cc.go index f7ba8b1c..967f8695 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/cc.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/cc.go @@ -128,7 +128,7 @@ func cc(args []string) error { const cgoRandomSeedPrefix = "-frandom-seed=" -func normalizeCgoRandomSeed(args []string, execRoot string) { +func normalizeCgoRandomSeed(args []string, cgoRoot string) { hasRandomSeed := false for _, arg := range args { if strings.HasPrefix(arg, cgoRandomSeedPrefix) { @@ -150,9 +150,10 @@ func normalizeCgoRandomSeed(args []string, execRoot string) { if workRoot != "" { normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") } - if execRoot != "" { - normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") + if cgoRoot != "" { + normalized = strings.ReplaceAll(normalized, cgoRoot, "$EXECROOT") } + normalized = normalizeBazelExecRoots(normalized) _, _ = io.WriteString(digest, normalized) _, _ = digest.Write([]byte{0}) } @@ -164,6 +165,50 @@ func normalizeCgoRandomSeed(args []string, execRoot string) { } } +// normalizeBazelExecRoots removes sandbox-specific prefixes from absolute +// paths that Bazel passes to the C compiler. Orchestrion may run `go install` +// from a source directory, so GO_CC_ROOT does not necessarily name the +// compiler's action execroot. +func normalizeBazelExecRoots(arg string) string { + for _, separator := range []byte{'/', '\\'} { + marker := string(separator) + "execroot" + string(separator) + for searchFrom := 0; searchFrom < len(arg); { + markerOffset := strings.Index(arg[searchFrom:], marker) + if markerOffset < 0 { + break + } + markerOffset += searchFrom + workspaceStart := markerOffset + len(marker) + workspaceEnd := workspaceStart + for workspaceEnd < len(arg) && arg[workspaceEnd] != separator { + workspaceEnd++ + } + if workspaceEnd == workspaceStart { + searchFrom = workspaceStart + continue + } + + pathStart := bazelAbsolutePathStart(arg, markerOffset, separator) + arg = arg[:pathStart] + "$EXECROOT" + arg[workspaceEnd:] + searchFrom = pathStart + len("$EXECROOT") + } + } + return arg +} + +func bazelAbsolutePathStart(arg string, markerOffset int, separator byte) int { + segmentStart := strings.LastIndexAny(arg[:markerOffset], "=,") + 1 + pathOffset := strings.IndexByte(arg[segmentStart:markerOffset], separator) + if pathOffset < 0 { + return segmentStart + } + pathStart := segmentStart + pathOffset + if pathStart >= segmentStart+2 && arg[pathStart-1] == ':' { + return pathStart - 2 + } + return pathStart +} + func goBuildWorkRoot(args []string) string { for i := 0; i+1 < len(args); i++ { if args[i] != "-o" && args[i] != "-c" { diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/cc_test.go b/third_party/rgo/v0_60_0/base/go/tools/builders/cc_test.go index f570843c..b8b4865f 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/cc_test.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/cc_test.go @@ -7,6 +7,8 @@ import ( ) func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { + firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") + secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") firstWorkRoot := filepath.Join("tmp", "go-build111") @@ -27,19 +29,41 @@ func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), } - normalizeCgoRandomSeed(first, firstExecRoot) - normalizeCgoRandomSeed(second, secondExecRoot) + // Orchestrion runs `go install` from the resolved module directory, while + // Bazel's C compiler remains rooted in the action execroot. The wrapper must + // normalize both independent roots even though GO_CC_ROOT only names the + // former. + normalizeCgoRandomSeed(first, firstCCRoot) + normalizeCgoRandomSeed(second, secondCCRoot) if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) } second = append(second, "-DPROFILE=changed") - normalizeCgoRandomSeed(second, secondExecRoot) + normalizeCgoRandomSeed(second, secondCCRoot) if randomSeedArg(first) == randomSeedArg(second) { t.Fatal("meaningful compiler arguments must affect the normalized seed") } } +func TestNormalizeBazelExecRoots(t *testing.T) { + tests := map[string]string{ + "/tmp/sandbox/1/execroot/workspace/external/cc": "$EXECROOT/external/cc", + "-I/tmp/sandbox/1/execroot/workspace/include": "-I$EXECROOT/include", + "--sysroot=/tmp/one/execroot/ws/sysroot": "--sysroot=$EXECROOT/sysroot", + "-ffile-prefix-map=/tmp/one/execroot/ws/src=/tmp/two/execroot/ws/src": "-ffile-prefix-map=$EXECROOT/src=$EXECROOT/src", + `C:\tmp\one\execroot\ws\external\cc.exe`: `$EXECROOT\external\cc.exe`, + `-IC:\tmp\one\execroot\ws\include`: `-I$EXECROOT\include`, + `C:/tmp/one/execroot/ws/external/cc.exe`: `$EXECROOT/external/cc.exe`, + `-IC:/tmp/one/execroot/ws/include`: `-I$EXECROOT/include`, + } + for input, want := range tests { + if got := normalizeBazelExecRoots(input); got != want { + t.Errorf("normalizeBazelExecRoots(%q) = %q, want %q", input, got, want) + } + } +} + func randomSeedArg(args []string) string { for _, arg := range args { if strings.HasPrefix(arg, "-frandom-seed=") { diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/cc.go b/third_party/rgo/v0_61_1/base/go/tools/builders/cc.go index f7ba8b1c..967f8695 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/cc.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/cc.go @@ -128,7 +128,7 @@ func cc(args []string) error { const cgoRandomSeedPrefix = "-frandom-seed=" -func normalizeCgoRandomSeed(args []string, execRoot string) { +func normalizeCgoRandomSeed(args []string, cgoRoot string) { hasRandomSeed := false for _, arg := range args { if strings.HasPrefix(arg, cgoRandomSeedPrefix) { @@ -150,9 +150,10 @@ func normalizeCgoRandomSeed(args []string, execRoot string) { if workRoot != "" { normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") } - if execRoot != "" { - normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") + if cgoRoot != "" { + normalized = strings.ReplaceAll(normalized, cgoRoot, "$EXECROOT") } + normalized = normalizeBazelExecRoots(normalized) _, _ = io.WriteString(digest, normalized) _, _ = digest.Write([]byte{0}) } @@ -164,6 +165,50 @@ func normalizeCgoRandomSeed(args []string, execRoot string) { } } +// normalizeBazelExecRoots removes sandbox-specific prefixes from absolute +// paths that Bazel passes to the C compiler. Orchestrion may run `go install` +// from a source directory, so GO_CC_ROOT does not necessarily name the +// compiler's action execroot. +func normalizeBazelExecRoots(arg string) string { + for _, separator := range []byte{'/', '\\'} { + marker := string(separator) + "execroot" + string(separator) + for searchFrom := 0; searchFrom < len(arg); { + markerOffset := strings.Index(arg[searchFrom:], marker) + if markerOffset < 0 { + break + } + markerOffset += searchFrom + workspaceStart := markerOffset + len(marker) + workspaceEnd := workspaceStart + for workspaceEnd < len(arg) && arg[workspaceEnd] != separator { + workspaceEnd++ + } + if workspaceEnd == workspaceStart { + searchFrom = workspaceStart + continue + } + + pathStart := bazelAbsolutePathStart(arg, markerOffset, separator) + arg = arg[:pathStart] + "$EXECROOT" + arg[workspaceEnd:] + searchFrom = pathStart + len("$EXECROOT") + } + } + return arg +} + +func bazelAbsolutePathStart(arg string, markerOffset int, separator byte) int { + segmentStart := strings.LastIndexAny(arg[:markerOffset], "=,") + 1 + pathOffset := strings.IndexByte(arg[segmentStart:markerOffset], separator) + if pathOffset < 0 { + return segmentStart + } + pathStart := segmentStart + pathOffset + if pathStart >= segmentStart+2 && arg[pathStart-1] == ':' { + return pathStart - 2 + } + return pathStart +} + func goBuildWorkRoot(args []string) string { for i := 0; i+1 < len(args); i++ { if args[i] != "-o" && args[i] != "-c" { diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/cc_test.go b/third_party/rgo/v0_61_1/base/go/tools/builders/cc_test.go index f570843c..b8b4865f 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/cc_test.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/cc_test.go @@ -7,6 +7,8 @@ import ( ) func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { + firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") + secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") firstWorkRoot := filepath.Join("tmp", "go-build111") @@ -27,19 +29,41 @@ func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), } - normalizeCgoRandomSeed(first, firstExecRoot) - normalizeCgoRandomSeed(second, secondExecRoot) + // Orchestrion runs `go install` from the resolved module directory, while + // Bazel's C compiler remains rooted in the action execroot. The wrapper must + // normalize both independent roots even though GO_CC_ROOT only names the + // former. + normalizeCgoRandomSeed(first, firstCCRoot) + normalizeCgoRandomSeed(second, secondCCRoot) if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) } second = append(second, "-DPROFILE=changed") - normalizeCgoRandomSeed(second, secondExecRoot) + normalizeCgoRandomSeed(second, secondCCRoot) if randomSeedArg(first) == randomSeedArg(second) { t.Fatal("meaningful compiler arguments must affect the normalized seed") } } +func TestNormalizeBazelExecRoots(t *testing.T) { + tests := map[string]string{ + "/tmp/sandbox/1/execroot/workspace/external/cc": "$EXECROOT/external/cc", + "-I/tmp/sandbox/1/execroot/workspace/include": "-I$EXECROOT/include", + "--sysroot=/tmp/one/execroot/ws/sysroot": "--sysroot=$EXECROOT/sysroot", + "-ffile-prefix-map=/tmp/one/execroot/ws/src=/tmp/two/execroot/ws/src": "-ffile-prefix-map=$EXECROOT/src=$EXECROOT/src", + `C:\tmp\one\execroot\ws\external\cc.exe`: `$EXECROOT\external\cc.exe`, + `-IC:\tmp\one\execroot\ws\include`: `-I$EXECROOT\include`, + `C:/tmp/one/execroot/ws/external/cc.exe`: `$EXECROOT/external/cc.exe`, + `-IC:/tmp/one/execroot/ws/include`: `-I$EXECROOT/include`, + } + for input, want := range tests { + if got := normalizeBazelExecRoots(input); got != want { + t.Errorf("normalizeBazelExecRoots(%q) = %q, want %q", input, got, want) + } + } +} + func randomSeedArg(args []string) string { for _, arg := range args { if strings.HasPrefix(arg, "-frandom-seed=") { diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/cc.go b/third_party/rgo/v0_62_0/base/go/tools/builders/cc.go index f7ba8b1c..967f8695 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/cc.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/cc.go @@ -128,7 +128,7 @@ func cc(args []string) error { const cgoRandomSeedPrefix = "-frandom-seed=" -func normalizeCgoRandomSeed(args []string, execRoot string) { +func normalizeCgoRandomSeed(args []string, cgoRoot string) { hasRandomSeed := false for _, arg := range args { if strings.HasPrefix(arg, cgoRandomSeedPrefix) { @@ -150,9 +150,10 @@ func normalizeCgoRandomSeed(args []string, execRoot string) { if workRoot != "" { normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") } - if execRoot != "" { - normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") + if cgoRoot != "" { + normalized = strings.ReplaceAll(normalized, cgoRoot, "$EXECROOT") } + normalized = normalizeBazelExecRoots(normalized) _, _ = io.WriteString(digest, normalized) _, _ = digest.Write([]byte{0}) } @@ -164,6 +165,50 @@ func normalizeCgoRandomSeed(args []string, execRoot string) { } } +// normalizeBazelExecRoots removes sandbox-specific prefixes from absolute +// paths that Bazel passes to the C compiler. Orchestrion may run `go install` +// from a source directory, so GO_CC_ROOT does not necessarily name the +// compiler's action execroot. +func normalizeBazelExecRoots(arg string) string { + for _, separator := range []byte{'/', '\\'} { + marker := string(separator) + "execroot" + string(separator) + for searchFrom := 0; searchFrom < len(arg); { + markerOffset := strings.Index(arg[searchFrom:], marker) + if markerOffset < 0 { + break + } + markerOffset += searchFrom + workspaceStart := markerOffset + len(marker) + workspaceEnd := workspaceStart + for workspaceEnd < len(arg) && arg[workspaceEnd] != separator { + workspaceEnd++ + } + if workspaceEnd == workspaceStart { + searchFrom = workspaceStart + continue + } + + pathStart := bazelAbsolutePathStart(arg, markerOffset, separator) + arg = arg[:pathStart] + "$EXECROOT" + arg[workspaceEnd:] + searchFrom = pathStart + len("$EXECROOT") + } + } + return arg +} + +func bazelAbsolutePathStart(arg string, markerOffset int, separator byte) int { + segmentStart := strings.LastIndexAny(arg[:markerOffset], "=,") + 1 + pathOffset := strings.IndexByte(arg[segmentStart:markerOffset], separator) + if pathOffset < 0 { + return segmentStart + } + pathStart := segmentStart + pathOffset + if pathStart >= segmentStart+2 && arg[pathStart-1] == ':' { + return pathStart - 2 + } + return pathStart +} + func goBuildWorkRoot(args []string) string { for i := 0; i+1 < len(args); i++ { if args[i] != "-o" && args[i] != "-c" { diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/cc_test.go b/third_party/rgo/v0_62_0/base/go/tools/builders/cc_test.go index f570843c..b8b4865f 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/cc_test.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/cc_test.go @@ -7,6 +7,8 @@ import ( ) func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { + firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") + secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") firstWorkRoot := filepath.Join("tmp", "go-build111") @@ -27,19 +29,41 @@ func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), } - normalizeCgoRandomSeed(first, firstExecRoot) - normalizeCgoRandomSeed(second, secondExecRoot) + // Orchestrion runs `go install` from the resolved module directory, while + // Bazel's C compiler remains rooted in the action execroot. The wrapper must + // normalize both independent roots even though GO_CC_ROOT only names the + // former. + normalizeCgoRandomSeed(first, firstCCRoot) + normalizeCgoRandomSeed(second, secondCCRoot) if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) } second = append(second, "-DPROFILE=changed") - normalizeCgoRandomSeed(second, secondExecRoot) + normalizeCgoRandomSeed(second, secondCCRoot) if randomSeedArg(first) == randomSeedArg(second) { t.Fatal("meaningful compiler arguments must affect the normalized seed") } } +func TestNormalizeBazelExecRoots(t *testing.T) { + tests := map[string]string{ + "/tmp/sandbox/1/execroot/workspace/external/cc": "$EXECROOT/external/cc", + "-I/tmp/sandbox/1/execroot/workspace/include": "-I$EXECROOT/include", + "--sysroot=/tmp/one/execroot/ws/sysroot": "--sysroot=$EXECROOT/sysroot", + "-ffile-prefix-map=/tmp/one/execroot/ws/src=/tmp/two/execroot/ws/src": "-ffile-prefix-map=$EXECROOT/src=$EXECROOT/src", + `C:\tmp\one\execroot\ws\external\cc.exe`: `$EXECROOT\external\cc.exe`, + `-IC:\tmp\one\execroot\ws\include`: `-I$EXECROOT\include`, + `C:/tmp/one/execroot/ws/external/cc.exe`: `$EXECROOT/external/cc.exe`, + `-IC:/tmp/one/execroot/ws/include`: `-I$EXECROOT/include`, + } + for input, want := range tests { + if got := normalizeBazelExecRoots(input); got != want { + t.Errorf("normalizeBazelExecRoots(%q) = %q, want %q", input, got, want) + } + } +} + func randomSeedArg(args []string) string { for _, arg := range args { if strings.HasPrefix(arg, "-frandom-seed=") { diff --git a/third_party/rgo/v0_63_0/base/go/tools/builders/cc.go b/third_party/rgo/v0_63_0/base/go/tools/builders/cc.go index f7ba8b1c..967f8695 100644 --- a/third_party/rgo/v0_63_0/base/go/tools/builders/cc.go +++ b/third_party/rgo/v0_63_0/base/go/tools/builders/cc.go @@ -128,7 +128,7 @@ func cc(args []string) error { const cgoRandomSeedPrefix = "-frandom-seed=" -func normalizeCgoRandomSeed(args []string, execRoot string) { +func normalizeCgoRandomSeed(args []string, cgoRoot string) { hasRandomSeed := false for _, arg := range args { if strings.HasPrefix(arg, cgoRandomSeedPrefix) { @@ -150,9 +150,10 @@ func normalizeCgoRandomSeed(args []string, execRoot string) { if workRoot != "" { normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") } - if execRoot != "" { - normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") + if cgoRoot != "" { + normalized = strings.ReplaceAll(normalized, cgoRoot, "$EXECROOT") } + normalized = normalizeBazelExecRoots(normalized) _, _ = io.WriteString(digest, normalized) _, _ = digest.Write([]byte{0}) } @@ -164,6 +165,50 @@ func normalizeCgoRandomSeed(args []string, execRoot string) { } } +// normalizeBazelExecRoots removes sandbox-specific prefixes from absolute +// paths that Bazel passes to the C compiler. Orchestrion may run `go install` +// from a source directory, so GO_CC_ROOT does not necessarily name the +// compiler's action execroot. +func normalizeBazelExecRoots(arg string) string { + for _, separator := range []byte{'/', '\\'} { + marker := string(separator) + "execroot" + string(separator) + for searchFrom := 0; searchFrom < len(arg); { + markerOffset := strings.Index(arg[searchFrom:], marker) + if markerOffset < 0 { + break + } + markerOffset += searchFrom + workspaceStart := markerOffset + len(marker) + workspaceEnd := workspaceStart + for workspaceEnd < len(arg) && arg[workspaceEnd] != separator { + workspaceEnd++ + } + if workspaceEnd == workspaceStart { + searchFrom = workspaceStart + continue + } + + pathStart := bazelAbsolutePathStart(arg, markerOffset, separator) + arg = arg[:pathStart] + "$EXECROOT" + arg[workspaceEnd:] + searchFrom = pathStart + len("$EXECROOT") + } + } + return arg +} + +func bazelAbsolutePathStart(arg string, markerOffset int, separator byte) int { + segmentStart := strings.LastIndexAny(arg[:markerOffset], "=,") + 1 + pathOffset := strings.IndexByte(arg[segmentStart:markerOffset], separator) + if pathOffset < 0 { + return segmentStart + } + pathStart := segmentStart + pathOffset + if pathStart >= segmentStart+2 && arg[pathStart-1] == ':' { + return pathStart - 2 + } + return pathStart +} + func goBuildWorkRoot(args []string) string { for i := 0; i+1 < len(args); i++ { if args[i] != "-o" && args[i] != "-c" { diff --git a/third_party/rgo/v0_63_0/base/go/tools/builders/cc_test.go b/third_party/rgo/v0_63_0/base/go/tools/builders/cc_test.go index f570843c..b8b4865f 100644 --- a/third_party/rgo/v0_63_0/base/go/tools/builders/cc_test.go +++ b/third_party/rgo/v0_63_0/base/go/tools/builders/cc_test.go @@ -7,6 +7,8 @@ import ( ) func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { + firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") + secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") firstWorkRoot := filepath.Join("tmp", "go-build111") @@ -27,19 +29,41 @@ func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), } - normalizeCgoRandomSeed(first, firstExecRoot) - normalizeCgoRandomSeed(second, secondExecRoot) + // Orchestrion runs `go install` from the resolved module directory, while + // Bazel's C compiler remains rooted in the action execroot. The wrapper must + // normalize both independent roots even though GO_CC_ROOT only names the + // former. + normalizeCgoRandomSeed(first, firstCCRoot) + normalizeCgoRandomSeed(second, secondCCRoot) if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) } second = append(second, "-DPROFILE=changed") - normalizeCgoRandomSeed(second, secondExecRoot) + normalizeCgoRandomSeed(second, secondCCRoot) if randomSeedArg(first) == randomSeedArg(second) { t.Fatal("meaningful compiler arguments must affect the normalized seed") } } +func TestNormalizeBazelExecRoots(t *testing.T) { + tests := map[string]string{ + "/tmp/sandbox/1/execroot/workspace/external/cc": "$EXECROOT/external/cc", + "-I/tmp/sandbox/1/execroot/workspace/include": "-I$EXECROOT/include", + "--sysroot=/tmp/one/execroot/ws/sysroot": "--sysroot=$EXECROOT/sysroot", + "-ffile-prefix-map=/tmp/one/execroot/ws/src=/tmp/two/execroot/ws/src": "-ffile-prefix-map=$EXECROOT/src=$EXECROOT/src", + `C:\tmp\one\execroot\ws\external\cc.exe`: `$EXECROOT\external\cc.exe`, + `-IC:\tmp\one\execroot\ws\include`: `-I$EXECROOT\include`, + `C:/tmp/one/execroot/ws/external/cc.exe`: `$EXECROOT/external/cc.exe`, + `-IC:/tmp/one/execroot/ws/include`: `-I$EXECROOT/include`, + } + for input, want := range tests { + if got := normalizeBazelExecRoots(input); got != want { + t.Errorf("normalizeBazelExecRoots(%q) = %q, want %q", input, got, want) + } + } +} + func randomSeedArg(args []string) string { for _, arg := range args { if strings.HasPrefix(arg, "-frandom-seed=") { diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index 7462efe0..c468ceae 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -4753,7 +4753,7 @@ index fdeda2d..76d69b6 100644 } } diff --git a/go/tools/builders/cc.go b/go/tools/builders/cc.go -index 6f67914..f7ba8b1 100644 +index 6f67914..967f869 100644 --- a/go/tools/builders/cc.go +++ b/go/tools/builders/cc.go @@ -1,7 +1,11 @@ @@ -4866,14 +4866,14 @@ index 6f67914..f7ba8b1 100644 if runtime.GOOS == "windows" { cmd := exec.Command(normalized[0], normalized[1:]...) cmd.Stdout = os.Stdout -@@ -46,3 +125,60 @@ func cc(args []string) error { +@@ -46,3 +125,105 @@ func cc(args []string) error { return syscall.Exec(normalized[0], normalized, os.Environ()) } } + +const cgoRandomSeedPrefix = "-frandom-seed=" + -+func normalizeCgoRandomSeed(args []string, execRoot string) { ++func normalizeCgoRandomSeed(args []string, cgoRoot string) { + hasRandomSeed := false + for _, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { @@ -4895,9 +4895,10 @@ index 6f67914..f7ba8b1 100644 + if workRoot != "" { + normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") + } -+ if execRoot != "" { -+ normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") ++ if cgoRoot != "" { ++ normalized = strings.ReplaceAll(normalized, cgoRoot, "$EXECROOT") + } ++ normalized = normalizeBazelExecRoots(normalized) + _, _ = io.WriteString(digest, normalized) + _, _ = digest.Write([]byte{0}) + } @@ -4909,6 +4910,50 @@ index 6f67914..f7ba8b1 100644 + } +} + ++// normalizeBazelExecRoots removes sandbox-specific prefixes from absolute ++// paths that Bazel passes to the C compiler. Orchestrion may run `go install` ++// from a source directory, so GO_CC_ROOT does not necessarily name the ++// compiler's action execroot. ++func normalizeBazelExecRoots(arg string) string { ++ for _, separator := range []byte{'/', '\\'} { ++ marker := string(separator) + "execroot" + string(separator) ++ for searchFrom := 0; searchFrom < len(arg); { ++ markerOffset := strings.Index(arg[searchFrom:], marker) ++ if markerOffset < 0 { ++ break ++ } ++ markerOffset += searchFrom ++ workspaceStart := markerOffset + len(marker) ++ workspaceEnd := workspaceStart ++ for workspaceEnd < len(arg) && arg[workspaceEnd] != separator { ++ workspaceEnd++ ++ } ++ if workspaceEnd == workspaceStart { ++ searchFrom = workspaceStart ++ continue ++ } ++ ++ pathStart := bazelAbsolutePathStart(arg, markerOffset, separator) ++ arg = arg[:pathStart] + "$EXECROOT" + arg[workspaceEnd:] ++ searchFrom = pathStart + len("$EXECROOT") ++ } ++ } ++ return arg ++} ++ ++func bazelAbsolutePathStart(arg string, markerOffset int, separator byte) int { ++ segmentStart := strings.LastIndexAny(arg[:markerOffset], "=,") + 1 ++ pathOffset := strings.IndexByte(arg[segmentStart:markerOffset], separator) ++ if pathOffset < 0 { ++ return segmentStart ++ } ++ pathStart := segmentStart + pathOffset ++ if pathStart >= segmentStart+2 && arg[pathStart-1] == ':' { ++ return pathStart - 2 ++ } ++ return pathStart ++} ++ +func goBuildWorkRoot(args []string) string { + for i := 0; i+1 < len(args); i++ { + if args[i] != "-o" && args[i] != "-c" { @@ -4929,10 +4974,10 @@ index 6f67914..f7ba8b1 100644 +} diff --git a/go/tools/builders/cc_test.go b/go/tools/builders/cc_test.go new file mode 100644 -index 0000000..f570843 +index 0000000..b8b4865 --- /dev/null +++ b/go/tools/builders/cc_test.go -@@ -0,0 +1,50 @@ +@@ -0,0 +1,74 @@ +package main + +import ( @@ -4942,6 +4987,8 @@ index 0000000..f570843 +) + +func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { ++ firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") ++ secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") + firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") + secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") + firstWorkRoot := filepath.Join("tmp", "go-build111") @@ -4962,19 +5009,41 @@ index 0000000..f570843 + "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), + } + -+ normalizeCgoRandomSeed(first, firstExecRoot) -+ normalizeCgoRandomSeed(second, secondExecRoot) ++ // Orchestrion runs `go install` from the resolved module directory, while ++ // Bazel's C compiler remains rooted in the action execroot. The wrapper must ++ // normalize both independent roots even though GO_CC_ROOT only names the ++ // former. ++ normalizeCgoRandomSeed(first, firstCCRoot) ++ normalizeCgoRandomSeed(second, secondCCRoot) + if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { + t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) + } + + second = append(second, "-DPROFILE=changed") -+ normalizeCgoRandomSeed(second, secondExecRoot) ++ normalizeCgoRandomSeed(second, secondCCRoot) + if randomSeedArg(first) == randomSeedArg(second) { + t.Fatal("meaningful compiler arguments must affect the normalized seed") + } +} + ++func TestNormalizeBazelExecRoots(t *testing.T) { ++ tests := map[string]string{ ++ "/tmp/sandbox/1/execroot/workspace/external/cc": "$EXECROOT/external/cc", ++ "-I/tmp/sandbox/1/execroot/workspace/include": "-I$EXECROOT/include", ++ "--sysroot=/tmp/one/execroot/ws/sysroot": "--sysroot=$EXECROOT/sysroot", ++ "-ffile-prefix-map=/tmp/one/execroot/ws/src=/tmp/two/execroot/ws/src": "-ffile-prefix-map=$EXECROOT/src=$EXECROOT/src", ++ `C:\tmp\one\execroot\ws\external\cc.exe`: `$EXECROOT\external\cc.exe`, ++ `-IC:\tmp\one\execroot\ws\include`: `-I$EXECROOT\include`, ++ `C:/tmp/one/execroot/ws/external/cc.exe`: `$EXECROOT/external/cc.exe`, ++ `-IC:/tmp/one/execroot/ws/include`: `-I$EXECROOT/include`, ++ } ++ for input, want := range tests { ++ if got := normalizeBazelExecRoots(input); got != want { ++ t.Errorf("normalizeBazelExecRoots(%q) = %q, want %q", input, got, want) ++ } ++ } ++} ++ +func randomSeedArg(args []string) string { + for _, arg := range args { + if strings.HasPrefix(arg, "-frandom-seed=") { diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index cb633c0d..164a0f2d 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -4751,7 +4751,7 @@ index fdeda2d..76d69b6 100644 } } diff --git a/go/tools/builders/cc.go b/go/tools/builders/cc.go -index 6f67914..f7ba8b1 100644 +index 6f67914..967f869 100644 --- a/go/tools/builders/cc.go +++ b/go/tools/builders/cc.go @@ -1,7 +1,11 @@ @@ -4864,14 +4864,14 @@ index 6f67914..f7ba8b1 100644 if runtime.GOOS == "windows" { cmd := exec.Command(normalized[0], normalized[1:]...) cmd.Stdout = os.Stdout -@@ -46,3 +125,60 @@ func cc(args []string) error { +@@ -46,3 +125,105 @@ func cc(args []string) error { return syscall.Exec(normalized[0], normalized, os.Environ()) } } + +const cgoRandomSeedPrefix = "-frandom-seed=" + -+func normalizeCgoRandomSeed(args []string, execRoot string) { ++func normalizeCgoRandomSeed(args []string, cgoRoot string) { + hasRandomSeed := false + for _, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { @@ -4893,9 +4893,10 @@ index 6f67914..f7ba8b1 100644 + if workRoot != "" { + normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") + } -+ if execRoot != "" { -+ normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") ++ if cgoRoot != "" { ++ normalized = strings.ReplaceAll(normalized, cgoRoot, "$EXECROOT") + } ++ normalized = normalizeBazelExecRoots(normalized) + _, _ = io.WriteString(digest, normalized) + _, _ = digest.Write([]byte{0}) + } @@ -4907,6 +4908,50 @@ index 6f67914..f7ba8b1 100644 + } +} + ++// normalizeBazelExecRoots removes sandbox-specific prefixes from absolute ++// paths that Bazel passes to the C compiler. Orchestrion may run `go install` ++// from a source directory, so GO_CC_ROOT does not necessarily name the ++// compiler's action execroot. ++func normalizeBazelExecRoots(arg string) string { ++ for _, separator := range []byte{'/', '\\'} { ++ marker := string(separator) + "execroot" + string(separator) ++ for searchFrom := 0; searchFrom < len(arg); { ++ markerOffset := strings.Index(arg[searchFrom:], marker) ++ if markerOffset < 0 { ++ break ++ } ++ markerOffset += searchFrom ++ workspaceStart := markerOffset + len(marker) ++ workspaceEnd := workspaceStart ++ for workspaceEnd < len(arg) && arg[workspaceEnd] != separator { ++ workspaceEnd++ ++ } ++ if workspaceEnd == workspaceStart { ++ searchFrom = workspaceStart ++ continue ++ } ++ ++ pathStart := bazelAbsolutePathStart(arg, markerOffset, separator) ++ arg = arg[:pathStart] + "$EXECROOT" + arg[workspaceEnd:] ++ searchFrom = pathStart + len("$EXECROOT") ++ } ++ } ++ return arg ++} ++ ++func bazelAbsolutePathStart(arg string, markerOffset int, separator byte) int { ++ segmentStart := strings.LastIndexAny(arg[:markerOffset], "=,") + 1 ++ pathOffset := strings.IndexByte(arg[segmentStart:markerOffset], separator) ++ if pathOffset < 0 { ++ return segmentStart ++ } ++ pathStart := segmentStart + pathOffset ++ if pathStart >= segmentStart+2 && arg[pathStart-1] == ':' { ++ return pathStart - 2 ++ } ++ return pathStart ++} ++ +func goBuildWorkRoot(args []string) string { + for i := 0; i+1 < len(args); i++ { + if args[i] != "-o" && args[i] != "-c" { @@ -4927,10 +4972,10 @@ index 6f67914..f7ba8b1 100644 +} diff --git a/go/tools/builders/cc_test.go b/go/tools/builders/cc_test.go new file mode 100644 -index 0000000..f570843 +index 0000000..b8b4865 --- /dev/null +++ b/go/tools/builders/cc_test.go -@@ -0,0 +1,50 @@ +@@ -0,0 +1,74 @@ +package main + +import ( @@ -4940,6 +4985,8 @@ index 0000000..f570843 +) + +func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { ++ firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") ++ secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") + firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") + secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") + firstWorkRoot := filepath.Join("tmp", "go-build111") @@ -4960,19 +5007,41 @@ index 0000000..f570843 + "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), + } + -+ normalizeCgoRandomSeed(first, firstExecRoot) -+ normalizeCgoRandomSeed(second, secondExecRoot) ++ // Orchestrion runs `go install` from the resolved module directory, while ++ // Bazel's C compiler remains rooted in the action execroot. The wrapper must ++ // normalize both independent roots even though GO_CC_ROOT only names the ++ // former. ++ normalizeCgoRandomSeed(first, firstCCRoot) ++ normalizeCgoRandomSeed(second, secondCCRoot) + if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { + t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) + } + + second = append(second, "-DPROFILE=changed") -+ normalizeCgoRandomSeed(second, secondExecRoot) ++ normalizeCgoRandomSeed(second, secondCCRoot) + if randomSeedArg(first) == randomSeedArg(second) { + t.Fatal("meaningful compiler arguments must affect the normalized seed") + } +} + ++func TestNormalizeBazelExecRoots(t *testing.T) { ++ tests := map[string]string{ ++ "/tmp/sandbox/1/execroot/workspace/external/cc": "$EXECROOT/external/cc", ++ "-I/tmp/sandbox/1/execroot/workspace/include": "-I$EXECROOT/include", ++ "--sysroot=/tmp/one/execroot/ws/sysroot": "--sysroot=$EXECROOT/sysroot", ++ "-ffile-prefix-map=/tmp/one/execroot/ws/src=/tmp/two/execroot/ws/src": "-ffile-prefix-map=$EXECROOT/src=$EXECROOT/src", ++ `C:\tmp\one\execroot\ws\external\cc.exe`: `$EXECROOT\external\cc.exe`, ++ `-IC:\tmp\one\execroot\ws\include`: `-I$EXECROOT\include`, ++ `C:/tmp/one/execroot/ws/external/cc.exe`: `$EXECROOT/external/cc.exe`, ++ `-IC:/tmp/one/execroot/ws/include`: `-I$EXECROOT/include`, ++ } ++ for input, want := range tests { ++ if got := normalizeBazelExecRoots(input); got != want { ++ t.Errorf("normalizeBazelExecRoots(%q) = %q, want %q", input, got, want) ++ } ++ } ++} ++ +func randomSeedArg(args []string) string { + for _, arg := range args { + if strings.HasPrefix(arg, "-frandom-seed=") { diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index 97511103..b3e331b7 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -4755,7 +4755,7 @@ index fdeda2d..76d69b6 100644 } } diff --git a/go/tools/builders/cc.go b/go/tools/builders/cc.go -index 6f67914..f7ba8b1 100644 +index 6f67914..967f869 100644 --- a/go/tools/builders/cc.go +++ b/go/tools/builders/cc.go @@ -1,7 +1,11 @@ @@ -4868,14 +4868,14 @@ index 6f67914..f7ba8b1 100644 if runtime.GOOS == "windows" { cmd := exec.Command(normalized[0], normalized[1:]...) cmd.Stdout = os.Stdout -@@ -46,3 +125,60 @@ func cc(args []string) error { +@@ -46,3 +125,105 @@ func cc(args []string) error { return syscall.Exec(normalized[0], normalized, os.Environ()) } } + +const cgoRandomSeedPrefix = "-frandom-seed=" + -+func normalizeCgoRandomSeed(args []string, execRoot string) { ++func normalizeCgoRandomSeed(args []string, cgoRoot string) { + hasRandomSeed := false + for _, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { @@ -4897,9 +4897,10 @@ index 6f67914..f7ba8b1 100644 + if workRoot != "" { + normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") + } -+ if execRoot != "" { -+ normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") ++ if cgoRoot != "" { ++ normalized = strings.ReplaceAll(normalized, cgoRoot, "$EXECROOT") + } ++ normalized = normalizeBazelExecRoots(normalized) + _, _ = io.WriteString(digest, normalized) + _, _ = digest.Write([]byte{0}) + } @@ -4911,6 +4912,50 @@ index 6f67914..f7ba8b1 100644 + } +} + ++// normalizeBazelExecRoots removes sandbox-specific prefixes from absolute ++// paths that Bazel passes to the C compiler. Orchestrion may run `go install` ++// from a source directory, so GO_CC_ROOT does not necessarily name the ++// compiler's action execroot. ++func normalizeBazelExecRoots(arg string) string { ++ for _, separator := range []byte{'/', '\\'} { ++ marker := string(separator) + "execroot" + string(separator) ++ for searchFrom := 0; searchFrom < len(arg); { ++ markerOffset := strings.Index(arg[searchFrom:], marker) ++ if markerOffset < 0 { ++ break ++ } ++ markerOffset += searchFrom ++ workspaceStart := markerOffset + len(marker) ++ workspaceEnd := workspaceStart ++ for workspaceEnd < len(arg) && arg[workspaceEnd] != separator { ++ workspaceEnd++ ++ } ++ if workspaceEnd == workspaceStart { ++ searchFrom = workspaceStart ++ continue ++ } ++ ++ pathStart := bazelAbsolutePathStart(arg, markerOffset, separator) ++ arg = arg[:pathStart] + "$EXECROOT" + arg[workspaceEnd:] ++ searchFrom = pathStart + len("$EXECROOT") ++ } ++ } ++ return arg ++} ++ ++func bazelAbsolutePathStart(arg string, markerOffset int, separator byte) int { ++ segmentStart := strings.LastIndexAny(arg[:markerOffset], "=,") + 1 ++ pathOffset := strings.IndexByte(arg[segmentStart:markerOffset], separator) ++ if pathOffset < 0 { ++ return segmentStart ++ } ++ pathStart := segmentStart + pathOffset ++ if pathStart >= segmentStart+2 && arg[pathStart-1] == ':' { ++ return pathStart - 2 ++ } ++ return pathStart ++} ++ +func goBuildWorkRoot(args []string) string { + for i := 0; i+1 < len(args); i++ { + if args[i] != "-o" && args[i] != "-c" { @@ -4931,10 +4976,10 @@ index 6f67914..f7ba8b1 100644 +} diff --git a/go/tools/builders/cc_test.go b/go/tools/builders/cc_test.go new file mode 100644 -index 0000000..f570843 +index 0000000..b8b4865 --- /dev/null +++ b/go/tools/builders/cc_test.go -@@ -0,0 +1,50 @@ +@@ -0,0 +1,74 @@ +package main + +import ( @@ -4944,6 +4989,8 @@ index 0000000..f570843 +) + +func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { ++ firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") ++ secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") + firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") + secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") + firstWorkRoot := filepath.Join("tmp", "go-build111") @@ -4964,19 +5011,41 @@ index 0000000..f570843 + "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), + } + -+ normalizeCgoRandomSeed(first, firstExecRoot) -+ normalizeCgoRandomSeed(second, secondExecRoot) ++ // Orchestrion runs `go install` from the resolved module directory, while ++ // Bazel's C compiler remains rooted in the action execroot. The wrapper must ++ // normalize both independent roots even though GO_CC_ROOT only names the ++ // former. ++ normalizeCgoRandomSeed(first, firstCCRoot) ++ normalizeCgoRandomSeed(second, secondCCRoot) + if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { + t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) + } + + second = append(second, "-DPROFILE=changed") -+ normalizeCgoRandomSeed(second, secondExecRoot) ++ normalizeCgoRandomSeed(second, secondCCRoot) + if randomSeedArg(first) == randomSeedArg(second) { + t.Fatal("meaningful compiler arguments must affect the normalized seed") + } +} + ++func TestNormalizeBazelExecRoots(t *testing.T) { ++ tests := map[string]string{ ++ "/tmp/sandbox/1/execroot/workspace/external/cc": "$EXECROOT/external/cc", ++ "-I/tmp/sandbox/1/execroot/workspace/include": "-I$EXECROOT/include", ++ "--sysroot=/tmp/one/execroot/ws/sysroot": "--sysroot=$EXECROOT/sysroot", ++ "-ffile-prefix-map=/tmp/one/execroot/ws/src=/tmp/two/execroot/ws/src": "-ffile-prefix-map=$EXECROOT/src=$EXECROOT/src", ++ `C:\tmp\one\execroot\ws\external\cc.exe`: `$EXECROOT\external\cc.exe`, ++ `-IC:\tmp\one\execroot\ws\include`: `-I$EXECROOT\include`, ++ `C:/tmp/one/execroot/ws/external/cc.exe`: `$EXECROOT/external/cc.exe`, ++ `-IC:/tmp/one/execroot/ws/include`: `-I$EXECROOT/include`, ++ } ++ for input, want := range tests { ++ if got := normalizeBazelExecRoots(input); got != want { ++ t.Errorf("normalizeBazelExecRoots(%q) = %q, want %q", input, got, want) ++ } ++ } ++} ++ +func randomSeedArg(args []string) string { + for _, arg := range args { + if strings.HasPrefix(arg, "-frandom-seed=") { diff --git a/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch index 7dddb9a6..e780bd4c 100644 --- a/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch @@ -3734,7 +3734,7 @@ index fdeda2d..76d69b6 100644 } } diff --git a/go/tools/builders/cc.go b/go/tools/builders/cc.go -index 6f67914..f7ba8b1 100644 +index 6f67914..967f869 100644 --- a/go/tools/builders/cc.go +++ b/go/tools/builders/cc.go @@ -1,7 +1,11 @@ @@ -3847,14 +3847,14 @@ index 6f67914..f7ba8b1 100644 if runtime.GOOS == "windows" { cmd := exec.Command(normalized[0], normalized[1:]...) cmd.Stdout = os.Stdout -@@ -46,3 +125,60 @@ func cc(args []string) error { +@@ -46,3 +125,105 @@ func cc(args []string) error { return syscall.Exec(normalized[0], normalized, os.Environ()) } } + +const cgoRandomSeedPrefix = "-frandom-seed=" + -+func normalizeCgoRandomSeed(args []string, execRoot string) { ++func normalizeCgoRandomSeed(args []string, cgoRoot string) { + hasRandomSeed := false + for _, arg := range args { + if strings.HasPrefix(arg, cgoRandomSeedPrefix) { @@ -3876,9 +3876,10 @@ index 6f67914..f7ba8b1 100644 + if workRoot != "" { + normalized = strings.ReplaceAll(normalized, workRoot, "$WORK") + } -+ if execRoot != "" { -+ normalized = strings.ReplaceAll(normalized, execRoot, "$EXECROOT") ++ if cgoRoot != "" { ++ normalized = strings.ReplaceAll(normalized, cgoRoot, "$EXECROOT") + } ++ normalized = normalizeBazelExecRoots(normalized) + _, _ = io.WriteString(digest, normalized) + _, _ = digest.Write([]byte{0}) + } @@ -3890,6 +3891,50 @@ index 6f67914..f7ba8b1 100644 + } +} + ++// normalizeBazelExecRoots removes sandbox-specific prefixes from absolute ++// paths that Bazel passes to the C compiler. Orchestrion may run `go install` ++// from a source directory, so GO_CC_ROOT does not necessarily name the ++// compiler's action execroot. ++func normalizeBazelExecRoots(arg string) string { ++ for _, separator := range []byte{'/', '\\'} { ++ marker := string(separator) + "execroot" + string(separator) ++ for searchFrom := 0; searchFrom < len(arg); { ++ markerOffset := strings.Index(arg[searchFrom:], marker) ++ if markerOffset < 0 { ++ break ++ } ++ markerOffset += searchFrom ++ workspaceStart := markerOffset + len(marker) ++ workspaceEnd := workspaceStart ++ for workspaceEnd < len(arg) && arg[workspaceEnd] != separator { ++ workspaceEnd++ ++ } ++ if workspaceEnd == workspaceStart { ++ searchFrom = workspaceStart ++ continue ++ } ++ ++ pathStart := bazelAbsolutePathStart(arg, markerOffset, separator) ++ arg = arg[:pathStart] + "$EXECROOT" + arg[workspaceEnd:] ++ searchFrom = pathStart + len("$EXECROOT") ++ } ++ } ++ return arg ++} ++ ++func bazelAbsolutePathStart(arg string, markerOffset int, separator byte) int { ++ segmentStart := strings.LastIndexAny(arg[:markerOffset], "=,") + 1 ++ pathOffset := strings.IndexByte(arg[segmentStart:markerOffset], separator) ++ if pathOffset < 0 { ++ return segmentStart ++ } ++ pathStart := segmentStart + pathOffset ++ if pathStart >= segmentStart+2 && arg[pathStart-1] == ':' { ++ return pathStart - 2 ++ } ++ return pathStart ++} ++ +func goBuildWorkRoot(args []string) string { + for i := 0; i+1 < len(args); i++ { + if args[i] != "-o" && args[i] != "-c" { @@ -3910,10 +3955,10 @@ index 6f67914..f7ba8b1 100644 +} diff --git a/go/tools/builders/cc_test.go b/go/tools/builders/cc_test.go new file mode 100644 -index 0000000..f570843 +index 0000000..b8b4865 --- /dev/null +++ b/go/tools/builders/cc_test.go -@@ -0,0 +1,50 @@ +@@ -0,0 +1,74 @@ +package main + +import ( @@ -3923,6 +3968,8 @@ index 0000000..f570843 +) + +func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { ++ firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") ++ secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") + firstExecRoot := filepath.Join("tmp", "sandbox", "1", "execroot", "workspace") + secondExecRoot := filepath.Join("tmp", "sandbox", "2", "execroot", "workspace") + firstWorkRoot := filepath.Join("tmp", "go-build111") @@ -3943,19 +3990,41 @@ index 0000000..f570843 + "-c", filepath.Join(secondWorkRoot, "b001", "_cgo_export.c"), + } + -+ normalizeCgoRandomSeed(first, firstExecRoot) -+ normalizeCgoRandomSeed(second, secondExecRoot) ++ // Orchestrion runs `go install` from the resolved module directory, while ++ // Bazel's C compiler remains rooted in the action execroot. The wrapper must ++ // normalize both independent roots even though GO_CC_ROOT only names the ++ // former. ++ normalizeCgoRandomSeed(first, firstCCRoot) ++ normalizeCgoRandomSeed(second, secondCCRoot) + if got, want := randomSeedArg(first), randomSeedArg(second); got == "" || got != want { + t.Fatalf("normalized seeds differ:\nfirst: %s\nsecond: %s", got, want) + } + + second = append(second, "-DPROFILE=changed") -+ normalizeCgoRandomSeed(second, secondExecRoot) ++ normalizeCgoRandomSeed(second, secondCCRoot) + if randomSeedArg(first) == randomSeedArg(second) { + t.Fatal("meaningful compiler arguments must affect the normalized seed") + } +} + ++func TestNormalizeBazelExecRoots(t *testing.T) { ++ tests := map[string]string{ ++ "/tmp/sandbox/1/execroot/workspace/external/cc": "$EXECROOT/external/cc", ++ "-I/tmp/sandbox/1/execroot/workspace/include": "-I$EXECROOT/include", ++ "--sysroot=/tmp/one/execroot/ws/sysroot": "--sysroot=$EXECROOT/sysroot", ++ "-ffile-prefix-map=/tmp/one/execroot/ws/src=/tmp/two/execroot/ws/src": "-ffile-prefix-map=$EXECROOT/src=$EXECROOT/src", ++ `C:\tmp\one\execroot\ws\external\cc.exe`: `$EXECROOT\external\cc.exe`, ++ `-IC:\tmp\one\execroot\ws\include`: `-I$EXECROOT\include`, ++ `C:/tmp/one/execroot/ws/external/cc.exe`: `$EXECROOT/external/cc.exe`, ++ `-IC:/tmp/one/execroot/ws/include`: `-I$EXECROOT/include`, ++ } ++ for input, want := range tests { ++ if got := normalizeBazelExecRoots(input); got != want { ++ t.Errorf("normalizeBazelExecRoots(%q) = %q, want %q", input, got, want) ++ } ++ } ++} ++ +func randomSeedArg(args []string) string { + for _, arg := range args { + if strings.HasPrefix(arg, "-frandom-seed=") { diff --git a/tools/dev/BUILD.bazel b/tools/dev/BUILD.bazel index a99766b4..7422dd98 100644 --- a/tools/dev/BUILD.bazel +++ b/tools/dev/BUILD.bazel @@ -12,6 +12,7 @@ exports_files( "check_bazelversion_sync.py", "check_module_versions.py", "check_release_archive_contents.py", + "compact_execution_log.py", "diff_rules_go_fork.py", "dotnet_bootstrap.bzl", "generate_rules_go_consumer_patch.py", diff --git a/tools/dev/check_release_archive_contents.py b/tools/dev/check_release_archive_contents.py index c87fb52c..40a3d667 100644 --- a/tools/dev/check_release_archive_contents.py +++ b/tools/dev/check_release_archive_contents.py @@ -60,6 +60,7 @@ def required_archive_paths(registry: ForkRegistry) -> set[str]: required = { ".bazelignore", "tools/dev/check_release_archive_contents.py", + "tools/dev/compact_execution_log.py", "tools/dev/generate_rules_go_consumer_patch.py", "tools/dev/generate_rules_go_fork_maps.py", "tools/dev/materialize_rules_go_fork.py", diff --git a/tools/dev/compact_execution_log.py b/tools/dev/compact_execution_log.py new file mode 100644 index 00000000..1b9b9405 --- /dev/null +++ b/tools/dev/compact_execution_log.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Read the action fields needed from a Bazel compact execution log. + +Bazel's compact format is a zstd-compressed stream of length-delimited +``ExecLogEntry`` protobuf messages. The profile verifier only needs spawn +identity, action keys, and output digests, so this module intentionally avoids +reconstructing inputs and runfiles. Keeping that projection small lets the +public repository validate Reprise's reproducibility contract without a +protobuf runtime or an internal Datadog dependency. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +import shutil +import subprocess +from typing import Iterator + + +class CompactExecutionLogError(ValueError): + """Raised when a compact execution log cannot be decoded safely.""" + + +@dataclass(frozen=True) +class CompactAction: + """Reprise-compatible projection of one executed Bazel spawn.""" + + target_label: str + mnemonic: str + command_args: tuple[str, ...] + environment_variables: tuple[tuple[str, str], ...] + listed_outputs: tuple[str, ...] + action_key: str + actual_outputs: tuple[tuple[str, str], ...] + + @property + def identity(self) -> tuple[str, str, tuple[str, ...]]: + """Return the stable cross-run identity used by Reprise.""" + return (self.target_label, self.mnemonic, self.listed_outputs) + + +@dataclass(frozen=True) +class _Artifact: + path: str + files: tuple[tuple[str, str], ...] + + +def read_compact_actions(path: Path) -> list[CompactAction]: + """Return executed spawns from one Bazel compact execution log.""" + zstd = shutil.which("zstd") + if zstd is None: + raise CompactExecutionLogError( + "zstd is required to read Bazel compact execution logs" + ) + result = subprocess.run( + [zstd, "--decompress", "--stdout", path.as_posix()], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if result.returncode != 0: + raise CompactExecutionLogError( + "failed to decompress %s: %s" + % (path, result.stderr.decode("utf-8", errors="replace").strip()) + ) + return list(_decode_entries(result.stdout)) + + +def _decode_entries(data: bytes) -> Iterator[CompactAction]: + artifacts: dict[int, _Artifact] = {} + offset = 0 + while offset < len(data): + size, offset = _read_varint(data, offset) + end = offset + size + if end > len(data): + raise CompactExecutionLogError("truncated ExecLogEntry message") + fields = list(_fields(data[offset:end])) + offset = end + + entry_id = _first_varint(fields, 1) + payload = _first_bytes(fields, 3) + if payload is not None: + artifacts[entry_id] = _decode_file(payload) + continue + payload = _first_bytes(fields, 4) + if payload is not None: + artifacts[entry_id] = _decode_directory(payload) + continue + payload = _first_bytes(fields, 5) + if payload is not None: + artifacts[entry_id] = _decode_symlink(payload) + continue + payload = _first_bytes(fields, 7) + if payload is not None: + yield _decode_spawn(payload, artifacts) + + +def _decode_file(data: bytes, parent: str = "") -> _Artifact: + fields = list(_fields(data)) + relative = _first_text(fields, 1) + path = _join_path(parent, relative) + return _Artifact(path=path, files=((path, _digest_hash(fields, 2)),)) + + +def _decode_directory(data: bytes) -> _Artifact: + fields = list(_fields(data)) + path = _first_text(fields, 1) + files = [] + for child in _all_bytes(fields, 2): + files.extend(_decode_file(child, path).files) + return _Artifact(path=path, files=tuple(files)) + + +def _decode_symlink(data: bytes) -> _Artifact: + fields = list(_fields(data)) + path = _first_text(fields, 1) + return _Artifact(path=path, files=((path, ""),)) + + +def _decode_spawn( + data: bytes, artifacts: dict[int, _Artifact] +) -> CompactAction: + fields = list(_fields(data)) + listed_outputs: set[str] = set() + actual_outputs: list[tuple[str, str]] = [] + for output in _all_bytes(fields, 6): + output_fields = list(_fields(output)) + output_id = _optional_varint(output_fields, 5) + if output_id is not None: + artifact = artifacts.get(output_id) + if artifact is None: + raise CompactExecutionLogError( + "spawn references unknown output id %d" % output_id + ) + listed_outputs.add(artifact.path) + actual_outputs.extend(artifact.files) + continue + invalid_path = _optional_text(output_fields, 4) + if invalid_path is not None: + listed_outputs.add(invalid_path) + + return CompactAction( + target_label=_first_text(fields, 7), + mnemonic=_first_text(fields, 8), + command_args=tuple(_all_text(fields, 1)), + environment_variables=tuple( + (_first_text(env, 1), _first_text(env, 2)) + for env in (list(_fields(value)) for value in _all_bytes(fields, 2)) + ), + listed_outputs=tuple(sorted(listed_outputs)), + action_key=_digest_hash(fields, 16), + actual_outputs=tuple(sorted(actual_outputs)), + ) + + +def _digest_hash(fields: list[tuple[int, int, int | bytes]], field: int) -> str: + digest = _first_bytes(fields, field) + if digest is None: + return "" + return _first_text(list(_fields(digest)), 1) + + +def _join_path(parent: str, child: str) -> str: + if not parent: + return child + return (PurePosixPath(parent) / child).as_posix() + + +def _fields(data: bytes) -> Iterator[tuple[int, int, int | bytes]]: + offset = 0 + while offset < len(data): + tag, offset = _read_varint(data, offset) + number = tag >> 3 + wire_type = tag & 7 + if number == 0: + raise CompactExecutionLogError("protobuf field number must be non-zero") + if wire_type == 0: + value, offset = _read_varint(data, offset) + elif wire_type == 1: + end = offset + 8 + value = data[offset:end] + offset = end + elif wire_type == 2: + size, offset = _read_varint(data, offset) + end = offset + size + value = data[offset:end] + offset = end + elif wire_type == 5: + end = offset + 4 + value = data[offset:end] + offset = end + else: + raise CompactExecutionLogError( + "unsupported protobuf wire type %d" % wire_type + ) + if offset > len(data): + raise CompactExecutionLogError("truncated protobuf field") + yield number, wire_type, value + + +def _read_varint(data: bytes, offset: int) -> tuple[int, int]: + value = 0 + for shift in range(0, 70, 7): + if offset >= len(data): + raise CompactExecutionLogError("truncated protobuf varint") + byte = data[offset] + offset += 1 + value |= (byte & 0x7F) << shift + if byte < 0x80: + return value, offset + raise CompactExecutionLogError("protobuf varint exceeds 64 bits") + + +def _all_bytes( + fields: list[tuple[int, int, int | bytes]], number: int +) -> Iterator[bytes]: + for field_number, wire_type, value in fields: + if field_number == number: + if wire_type != 2 or not isinstance(value, bytes): + raise CompactExecutionLogError( + "protobuf field %d is not length-delimited" % number + ) + yield value + + +def _first_bytes( + fields: list[tuple[int, int, int | bytes]], number: int +) -> bytes | None: + return next(_all_bytes(fields, number), None) + + +def _first_text( + fields: list[tuple[int, int, int | bytes]], number: int +) -> str: + return (_first_bytes(fields, number) or b"").decode("utf-8") + + +def _optional_text( + fields: list[tuple[int, int, int | bytes]], number: int +) -> str | None: + value = _first_bytes(fields, number) + return None if value is None else value.decode("utf-8") + + +def _all_text( + fields: list[tuple[int, int, int | bytes]], number: int +) -> Iterator[str]: + for value in _all_bytes(fields, number): + yield value.decode("utf-8") + + +def _optional_varint( + fields: list[tuple[int, int, int | bytes]], number: int +) -> int | None: + for field_number, wire_type, value in fields: + if field_number == number: + if wire_type != 0 or not isinstance(value, int): + raise CompactExecutionLogError( + "protobuf field %d is not a varint" % number + ) + return value + return None + + +def _first_varint( + fields: list[tuple[int, int, int | bytes]], number: int +) -> int: + return _optional_varint(fields, number) or 0 diff --git a/tools/dev/verify_rules_go_profiles.py b/tools/dev/verify_rules_go_profiles.py index 9aa7b1b9..3ee42a71 100644 --- a/tools/dev/verify_rules_go_profiles.py +++ b/tools/dev/verify_rules_go_profiles.py @@ -23,6 +23,7 @@ import tempfile try: + from tools.dev.compact_execution_log import CompactAction, read_compact_actions from tools.dev.generate_rules_go_consumer_patch import ( DEFAULT_PROFILE_ROOT, REPO_ROOT, @@ -38,6 +39,7 @@ from tools.dev.rules_go_fork_registry import DEFAULT_REGISTRY, ForkSelection, load_registry except ModuleNotFoundError: sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from tools.dev.compact_execution_log import CompactAction, read_compact_actions from tools.dev.generate_rules_go_consumer_patch import ( DEFAULT_PROFILE_ROOT, REPO_ROOT, @@ -82,7 +84,6 @@ def verify_profiles( ) upstream_ids = [upstream] with temporary_smoke_root() as smoke_root: - bazel_output_user_root = smoke_root / "bazel_output_user_root" for upstream_id in upstream_ids: selection = registry.resolve(upstream_id, "base") patch = output_dir / ("%s-%s.patch" % (upstream_id, profile)) @@ -113,7 +114,6 @@ def verify_profiles( selection=selection, patch=patch, work_root=smoke_root / upstream_id, - bazel_output_user_root=bazel_output_user_root, bazel=bazel, go_version=go_version, orchestrion_version=orchestrion_version, @@ -122,7 +122,6 @@ def verify_profiles( ) generated_paths.extend([patch, manifest]) print("verified %s" % patch) - _bazel_shutdown(bazel, bazel_output_user_root, private_safe_patterns) if public_denylist is not None or private_blocklist_file is not None: verify_private_safe( paths=generated_paths, @@ -142,111 +141,85 @@ def verify_workspace_runtime_functional_smoke( selection: ForkSelection, patch: Path, work_root: Path, - bazel_output_user_root: Path, bazel: Path, go_version: str, orchestrion_version: str, dd_trace_go_version: str, private_safe_patterns: list[str], ) -> None: - """Verify a generated workspace_runtime patch is usable without the base tree.""" + """Verify a generated patch in two independent consumer-style builds.""" upstream_source = download_upstream(selection, work_root / "download") - rules_go_root = work_root / "rules_go_patched" - workspace = work_root / "workspace" - copy_filtered_tree(upstream_source, rules_go_root) - run_private_safe( - ["git", "-C", rules_go_root.as_posix(), "apply", "--binary", "-p1", patch.as_posix()], - private_safe_patterns=private_safe_patterns, - ) - write_smoke_workspace( - workspace=workspace, - rules_go_root=rules_go_root, - go_version=go_version, - orchestrion_version=orchestrion_version, - dd_trace_go_version=dd_trace_go_version, - ) - common_flags = [ "--noenable_bzlmod", "--enable_workspace", ] - orchestrion_flags = common_flags + [ - "--@io_bazel_rules_go//go/private/orchestrion:enabled=true", - "--@io_bazel_rules_go//go/private/orchestrion:mode=test_optimization", - ] - reproducibility_flags = cgo_reproducibility_flags(sys.platform) - isolated_cache_flags = [ - "--disk_cache=", - "--remote_cache=", - ] - run_bazel( - bazel, - bazel_output_user_root, - workspace, - ["build", *common_flags, "@go_sdk//:builder"], - private_safe_patterns=private_safe_patterns, - ) - first_plain = run_plain_reproducibility_snapshot( - bazel, - bazel_output_user_root, - workspace, - command="build", - mode_flags=[*common_flags, *reproducibility_flags, *isolated_cache_flags], - target="//app:hello_test", - private_safe_patterns=private_safe_patterns, - ) - first_orchestrion = run_orchestrion_reproducibility_snapshot( - bazel, - bazel_output_user_root, - workspace, - command="test", - mode_flags=[ - *orchestrion_flags, - *reproducibility_flags, - *isolated_cache_flags, - "--test_output=errors", - ], - target="//app:hello_test", - private_safe_patterns=private_safe_patterns, - ) - aquery = run_bazel( - bazel, - bazel_output_user_root, - workspace, - [ - "aquery", - *orchestrion_flags, - *reproducibility_flags, - 'mnemonic("GoCompilePkg", //app:hello_test)', - ], - private_safe_patterns=private_safe_patterns, - ) - assert_aquery_contains(aquery.stdout, patch) + consumer_flags = cgo_reproducibility_flags() + plain_snapshots = [] + optimized_snapshots = [] + for run_name in ("first", "second"): + run_root = work_root / run_name + rules_go_root = run_root / "rules_go_patched" + workspace = run_root / "workspace" + output_user_root = run_root / "bazel_output_user_root" + copy_filtered_tree(upstream_source, rules_go_root) + run_private_safe( + [ + "git", + "-C", + rules_go_root.as_posix(), + "apply", + "--binary", + "-p1", + patch.as_posix(), + ], + private_safe_patterns=private_safe_patterns, + ) + write_smoke_workspace( + workspace=workspace, + rules_go_root=rules_go_root, + go_version=go_version, + orchestrion_version=orchestrion_version, + dd_trace_go_version=dd_trace_go_version, + ) + isolated_cache_flags = [ + "--disk_cache=%s" % (run_root / "disk_cache").as_posix(), + "--remote_cache=", + ] + mode_flags = [*common_flags, *consumer_flags, *isolated_cache_flags] + run_bazel( + bazel, + output_user_root, + workspace, + ["build", *common_flags, "@go_sdk//:builder"], + private_safe_patterns=private_safe_patterns, + ) + optimized_snapshots.append( + run_orchestrion_reproducibility_snapshot( + bazel, + output_user_root, + workspace, + command="test", + mode_flags=mode_flags, + target="//app:hello_test.topt", + raw_target="//app:hello_test.topt__raw_go_test", + execution_log=run_root / "execution.compact.zst", + private_safe_patterns=private_safe_patterns, + ) + ) + plain_snapshots.append( + run_plain_reproducibility_snapshot( + bazel, + output_user_root, + workspace, + command="build", + mode_flags=mode_flags, + target="//app:hello_test", + private_safe_patterns=private_safe_patterns, + ) + ) - replay_output_user_root = work_root / "bazel_output_user_root_replay" - second_plain = run_plain_reproducibility_snapshot( - bazel, - replay_output_user_root, - workspace, - command="build", - mode_flags=[*common_flags, *reproducibility_flags, *isolated_cache_flags], - target="//app:hello_test", - private_safe_patterns=private_safe_patterns, - ) - second_orchestrion = run_orchestrion_reproducibility_snapshot( - bazel, - replay_output_user_root, - workspace, - command="test", - mode_flags=[ - *orchestrion_flags, - *reproducibility_flags, - *isolated_cache_flags, - "--test_output=errors", - ], - target="//app:hello_test", - private_safe_patterns=private_safe_patterns, - ) + first_plain, second_plain = plain_snapshots + first_orchestrion, second_orchestrion = optimized_snapshots assert_plain_stdlib_cache(first_plain.stdlib_cache, patch) assert_plain_stdlib_cache(second_plain.stdlib_cache, patch) assert_orchestrion_stdlib_cache(first_orchestrion.stdlib_cache, patch) @@ -295,54 +268,27 @@ def verify_workspace_runtime_functional_smoke( ), ) ) - if first_orchestrion.action_keys != second_orchestrion.action_keys: - raise ValueError( - "Test Optimization action keys differ for %s: %s" - % ( - patch, - describe_mapping_difference( - first_orchestrion.action_keys, - second_orchestrion.action_keys, - ), - ) - ) - if first_orchestrion.outputs != second_orchestrion.outputs: - raise ValueError( - "Test Optimization action outputs differ for %s: %s" - % ( - patch, - describe_mapping_difference( - first_orchestrion.outputs, - second_orchestrion.outputs, - ), - ) - ) + assert_no_actionable_reproducibility_findings( + first_orchestrion.actions, + second_orchestrion.actions, + patch, + ) -def cgo_reproducibility_flags(platform_name: str) -> list[str]: - """Mirror the CGO/debug conditions that exposed cross-worker drift.""" - flags = [ +def cgo_reproducibility_flags() -> list[str]: + """Mirror consumer CGO/debug flags without adding deterministic policy.""" + return [ "--compilation_mode=fastbuild", "--incompatible_strict_action_env", "--experimental_exec_configuration_distinguisher=diff_to_affected", "--experimental_platform_in_output_dir", "--@io_bazel_rules_go//go/config:pure=False", + "--@io_bazel_rules_go//go/config:linkmode=normal", "--strip=never", - "--copt=-O2", "--copt=-fno-omit-frame-pointer", "--copt=-g", "--copt=-UNDEBUG", ] - if platform_name.startswith("linux"): - flags.extend( - [ - "--repo_env=CC=clang", - "--linkopt=-fuse-ld=lld", - "--linkopt=-Wl,--build-id=md5", - "--linkopt=-Wl,--threads=4", - ] - ) - return flags def describe_snapshot_difference( @@ -381,9 +327,23 @@ def write_smoke_workspace( orchestrion_version: str, dd_trace_go_version: str, ) -> None: - """Create a minimal WORKSPACE-mode Go project for generated profile smoke.""" + """Create a WORKSPACE-mode CGO project with a real .topt transition.""" app = workspace / "app" + transition_repo = workspace / "transition_rule" app.mkdir(parents=True, exist_ok=True) + transition_repo.mkdir(parents=True, exist_ok=True) + shutil.copyfile( + REPO_ROOT / "modules/go/topt_go_orchestrion.bzl", + transition_repo / "topt_go_orchestrion.bzl", + ) + transition_repo.joinpath("WORKSPACE").write_text( + 'workspace(name = "datadog_go_transition")\n', + encoding="utf-8", + ) + transition_repo.joinpath("BUILD.bazel").write_text( + 'exports_files(["topt_go_orchestrion.bzl"])\n', + encoding="utf-8", + ) workspace.joinpath("WORKSPACE").write_text( """workspace(name = "profile_smoke") @@ -392,6 +352,12 @@ def write_smoke_workspace( path = "%s", ) +local_repository( + name = "datadog_go_transition", + path = "%s", + repo_mapping = {"@rules_go": "@io_bazel_rules_go"}, +) + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( @@ -420,6 +386,7 @@ def write_smoke_workspace( """ % ( rules_go_root.as_posix(), + transition_repo.as_posix(), go_version, orchestrion_version, dd_trace_go_version, @@ -428,11 +395,15 @@ def write_smoke_workspace( encoding="utf-8", ) app.joinpath("BUILD.bazel").write_text( - """load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + """load("@datadog_go_transition//:topt_go_orchestrion.bzl", "orch_go_test") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +exports_files(["metadata.json"]) go_library( name = "hello_lib", - srcs = ["hello.go"], + srcs = ["hello.go", "hello_cgo.go"], + cgo = True, importpath = "example.com/profile_smoke/app", ) @@ -441,17 +412,50 @@ def write_smoke_workspace( srcs = ["hello_test.go"], embed = [":hello_lib"], ) + +go_test( + name = "hello_test.topt__raw_go_test", + srcs = ["hello_test.go"], + embed = [":hello_lib"], + tags = ["manual"], +) + +orch_go_test( + name = "hello_test.topt", + actual = ":hello_test.topt__raw_go_test", + metadata = ":metadata.json", + orchestrion_mode = "test_optimization", +) """, encoding="utf-8", ) app.joinpath("hello.go").write_text( - 'package app\n\nfunc Greeting() string { return "hello" }\n', + 'package app\n\nfunc Greeting() string { return CgoGreeting() }\n', + encoding="utf-8", + ) + app.joinpath("hello_cgo.go").write_text( + '''package app + +/* +#include +static int profile_smoke_value(void) { return 42; } +*/ +import "C" + +func CgoGreeting() string { + if C.profile_smoke_value() == 42 { + return "hello" + } + return "bad" +} +''', encoding="utf-8", ) app.joinpath("hello_test.go").write_text( 'package app\n\nimport "testing"\n\nfunc TestGreeting(t *testing.T) { if Greeting() != "hello" { t.Fatal("bad") } }\n', encoding="utf-8", ) + app.joinpath("metadata.json").write_text("{}\n", encoding="utf-8") def run_bazel( @@ -502,13 +506,21 @@ class StdlibCacheSnapshot: @dataclass(frozen=True) class ReproducibilitySnapshot: - """Cache contents, action keys, and outputs from one isolated build.""" + """Ordinary stdlib cache, action keys, and outputs from one build.""" stdlib_cache: StdlibCacheSnapshot action_keys: dict[str, str] outputs: dict[str, str] +@dataclass(frozen=True) +class OptimizedReproducibilitySnapshot: + """Instrumented cache and Reprise-compatible actions from one cold build.""" + + stdlib_cache: StdlibCacheSnapshot + actions: dict[tuple[str, str, tuple[str, ...]], CompactAction] + + def run_plain_reproducibility_snapshot( bazel: Path, output_user_root: Path, @@ -520,13 +532,11 @@ def run_plain_reproducibility_snapshot( private_safe_patterns: list[str] | None = None, ) -> ReproducibilitySnapshot: """Build once and capture the ordinary CGO stdlib action and bytes.""" - stdlib_cache = run_stdlib_inventory( + run_bazel( bazel, output_user_root, workspace, - command=command, - mode_flags=mode_flags, - target=target, + [command, *mode_flags, target], private_safe_patterns=private_safe_patterns, ) aquery = run_bazel( @@ -542,11 +552,11 @@ def run_plain_reproducibility_snapshot( private_safe_patterns=private_safe_patterns, ) aquery_data = json.loads(aquery.stdout) - assert_cgo_reproducibility_actions( + assert_cgo_aquery_actions( aquery_data, - sys.platform, expected_instrumented=False, ) + stdlib_cache = plain_stdlib_cache_from_aquery(aquery_data, workspace) action_keys, outputs = action_snapshot_from_aquery( aquery_data, workspace=workspace, @@ -563,6 +573,39 @@ def run_plain_reproducibility_snapshot( ) +def plain_stdlib_cache_from_aquery( + data: dict[str, object], workspace: Path +) -> StdlibCacheSnapshot: + """Inspect only gocache outputs declared by ordinary GoStdlib actions.""" + path_fragments = { + int(fragment["id"]): fragment for fragment in data.get("pathFragments", []) + } + artifacts = { + int(artifact["id"]): resolve_path_fragment( + int(artifact["pathFragmentId"]), path_fragments + ) + for artifact in data.get("artifacts", []) + } + cache_paths = set() + for action in data.get("actions", []): + if action.get("mnemonic") != "GoStdlib": + continue + for output_id in action.get("outputIds", []): + relative = artifacts[int(output_id)] + if PurePosixPath(relative).name == "gocache": + cache_paths.add(relative) + if not cache_paths: + raise ValueError("plain reproducibility aquery has no GoStdlib gocache") + for relative in sorted(cache_paths): + snapshot = canonical_tree_inventory(workspace / relative) + if snapshot.inventory or snapshot.manifest is not None: + raise ValueError( + "ordinary GoStdlib gocache is not empty: %s contains %s" + % (relative, sorted(snapshot.inventory)) + ) + return StdlibCacheSnapshot(inventory={}, manifest=None) + + def run_orchestrion_reproducibility_snapshot( bazel: Path, output_user_root: Path, @@ -571,9 +614,11 @@ def run_orchestrion_reproducibility_snapshot( command: str, mode_flags: list[str], target: str, + raw_target: str, + execution_log: Path, private_safe_patterns: list[str] | None = None, -) -> ReproducibilitySnapshot: - """Build once and capture the cache-critical Test Optimization actions.""" +) -> OptimizedReproducibilitySnapshot: + """Run one real .topt target and read its compact execution log.""" stdlib_cache = run_stdlib_inventory( bazel, output_user_root, @@ -581,49 +626,139 @@ def run_orchestrion_reproducibility_snapshot( command=command, mode_flags=mode_flags, target=target, + execution_log=execution_log, private_safe_patterns=private_safe_patterns, ) - aquery = run_bazel( - bazel, - output_user_root, - workspace, - [ - "aquery", - *mode_flags, - "--output=jsonproto", - 'mnemonic("(GoStdlib|GoSyntheticTestmainHelpers|GoCompilePkg|GoLink)", deps(%s))' - % target, - ], - private_safe_patterns=private_safe_patterns, - ) - aquery_data = json.loads(aquery.stdout) - assert_cgo_reproducibility_actions( - aquery_data, - sys.platform, - expected_instrumented=True, - ) - action_keys, outputs = action_snapshot_from_aquery( - aquery_data, - workspace=workspace, - target_label=target, + if not execution_log.is_file() or execution_log.stat().st_size == 0: + raise ValueError("Bazel did not write compact execution log %s" % execution_log) + actions = select_reproducibility_actions( + read_compact_actions(execution_log), + raw_target=raw_target, ) + assert_cgo_reproducibility_actions(actions.values(), expected_instrumented=True) required = { "GoCompilePkg", "GoLink", "GoStdlib", "GoSyntheticTestmainHelpers", } - found = {identity.split(" ", 1)[0] for identity in action_keys} + found = {action.mnemonic for action in actions.values()} missing = sorted(required - found) if missing: raise ValueError( - "reproducibility aquery for %s is missing actions: %s" + "compact execution log for %s is missing actions: %s" % (target, ", ".join(missing)) ) - return ReproducibilitySnapshot( + return OptimizedReproducibilitySnapshot( stdlib_cache=stdlib_cache, - action_keys=action_keys, - outputs=outputs, + actions=actions, + ) + + +def select_reproducibility_actions( + actions: list[CompactAction], *, raw_target: str +) -> dict[tuple[str, str, tuple[str, ...]], CompactAction]: + """Select the four action families whose outputs feed a .topt binary.""" + selected = {} + for action in actions: + if action.mnemonic in {"GoStdlib", "GoSyntheticTestmainHelpers"}: + pass + elif not bazel_labels_match(action.target_label, raw_target): + continue + elif action.mnemonic == "GoCompilePkg": + if not any("~testmain.a" in path for path in action.listed_outputs): + continue + elif action.mnemonic != "GoLink": + continue + + if action.identity in selected: + raise ValueError("duplicate compact-log action identity: %s" % (action.identity,)) + if not action.action_key: + raise ValueError( + "compact-log action has no cache digest: %s" % (action.identity,) + ) + if not action.actual_outputs: + raise ValueError( + "compact-log action has no output digests: %s" % (action.identity,) + ) + selected[action.identity] = action + return selected + + +def bazel_labels_match(actual: str, expected: str) -> bool: + """Compare main-repository labels across canonical-label spellings.""" + return actual.lstrip("@") == expected + + +@dataclass(frozen=True) +class ReproducibilityFinding: + """One actionable cell from Reprise's action-key/output classifier.""" + + action: CompactAction + kind: str + differing_outputs: tuple[str, ...] + + +def actionable_reproducibility_findings( + first: dict[tuple[str, str, tuple[str, ...]], CompactAction], + second: dict[tuple[str, str, tuple[str, ...]], CompactAction], +) -> list[ReproducibilityFinding]: + """Apply Reprise's 2x2 classifier and return output-changing findings.""" + findings = [] + for identity in sorted(set(first) & set(second)): + left = first[identity] + right = second[identity] + if left.actual_outputs == right.actual_outputs: + # A changed action key with identical bytes is Reprise's + # non-actionable wasted_rebuild case. + continue + right_outputs = dict(right.actual_outputs) + differing = tuple( + path + for path, digest in left.actual_outputs + if right_outputs.get(path) != digest + ) + left_paths = {path for path, _ in left.actual_outputs} + differing += tuple( + path for path, _ in right.actual_outputs if path not in left_paths + ) + keys_match = bool(left.action_key) and left.action_key == right.action_key + findings.append( + ReproducibilityFinding( + action=left, + kind="tool_nondeterminism" if keys_match else "input_driven", + differing_outputs=differing, + ) + ) + return findings + + +def assert_no_actionable_reproducibility_findings( + first: dict[tuple[str, str, tuple[str, ...]], CompactAction], + second: dict[tuple[str, str, tuple[str, ...]], CompactAction], + patch: Path, +) -> None: + """Fail when Reprise would report an output-changing selected action.""" + findings = actionable_reproducibility_findings(first, second) + if not findings: + return + details = [] + for finding in findings: + outputs = ", ".join(finding.differing_outputs[:5]) + if len(finding.differing_outputs) > 5: + outputs += ", ... and %d more" % (len(finding.differing_outputs) - 5) + details.append( + "%s %s [%s]: %s" + % ( + finding.action.mnemonic, + finding.action.target_label, + finding.kind, + outputs, + ) + ) + raise ValueError( + "Test Optimization actions are not reproducible for %s: %s" + % (patch, "; ".join(details)) ) @@ -692,13 +827,12 @@ def action_snapshot_from_aquery( return action_keys, outputs -def assert_cgo_reproducibility_actions( +def assert_cgo_aquery_actions( data: dict[str, object], - platform_name: str, *, expected_instrumented: bool, ) -> None: - """Require the replay to cover the expected CGO/debug stdlib mode.""" + """Require an aquery to cover the expected CGO/debug stdlib mode.""" matching = [] for action in data.get("actions", []): if action.get("mnemonic") != "GoStdlib": @@ -712,11 +846,6 @@ def assert_cgo_reproducibility_actions( continue if "-g" not in environment.get("CGO_CFLAGS", "").split(): continue - if platform_name.startswith("linux"): - ldflags = environment.get("CGO_LDFLAGS", "") - required = ["-fuse-ld=lld", "--build-id=md5", "--threads=4"] - if any(flag not in ldflags for flag in required): - continue if ("-orchestrion" in arguments) == expected_instrumented: matching.append(action) if not matching: @@ -727,6 +856,33 @@ def assert_cgo_reproducibility_actions( ) +def assert_cgo_reproducibility_actions( + actions, *, expected_instrumented: bool +) -> None: + """Require a compact log to contain the requested CGO stdlib mode.""" + for action in actions: + if action.mnemonic != "GoStdlib": + continue + environment = dict(action.environment_variables) + if environment.get("CGO_ENABLED") != "1": + continue + if "-g" not in environment.get("CGO_CFLAGS", "").split(): + continue + instrumented = ( + "-orchestrion" in action.command_args + and "-orchestrion_mode" in action.command_args + and "test_optimization" in action.command_args + and any("rules_go_orchestrion_tool" in arg for arg in action.command_args) + ) + if instrumented == expected_instrumented: + return + mode = "instrumented" if expected_instrumented else "plain" + raise ValueError( + "compact execution log must contain a %s CGO-enabled " + "GoStdlib action with debug flags" % mode + ) + + def resolve_path_fragment( fragment_id: int, fragments: dict[int, dict[str, object]] ) -> str: @@ -829,14 +985,21 @@ def run_stdlib_inventory( command: str, mode_flags: list[str], target: str, + execution_log: Path | None = None, private_safe_patterns: list[str] | None = None, ) -> StdlibCacheSnapshot: """Execute one stdlib action and inventory its declared cache TreeArtifact.""" + execution_log_flags = [] + if execution_log is not None: + execution_log.parent.mkdir(parents=True, exist_ok=True) + execution_log_flags = [ + "--execution_log_compact_file=%s" % execution_log.as_posix(), + ] run_bazel( bazel, output_user_root, workspace, - [command, *mode_flags, target], + [command, *mode_flags, *execution_log_flags, target], private_safe_patterns=private_safe_patterns, ) output_path_result = run_bazel( @@ -1054,21 +1217,6 @@ def run_private_safe( return result -def assert_aquery_contains(aquery_output: str, patch: Path) -> None: - """Assert the generated-patch smoke actually used Orchestrion test mode.""" - required = [ - "-orchestrion_mode", - "test_optimization", - "rules_go_orchestrion_tool", - ] - missing = [needle for needle in required if needle not in aquery_output] - if missing: - raise ValueError( - "functional smoke for %s did not prove Orchestrion test mode; missing %s" - % (patch, ", ".join(missing)) - ) - - def modified_tracked_files(private_blocklist_file: Path | None) -> list[Path]: """Return modified tracked repository files when a private scan is requested.""" if private_blocklist_file is None: @@ -1094,24 +1242,6 @@ def modified_tracked_files(private_blocklist_file: Path | None) -> list[Path]: return sorted(paths) -def _bazel_shutdown( - bazel: Path, - output_user_root: Path, - private_safe_patterns: list[str] | None = None, -) -> None: - """Best-effort shutdown for the smoke workspace Bazel server.""" - try: - run_bazel( - bazel, - output_user_root, - REPO_ROOT, - ["shutdown"], - private_safe_patterns=private_safe_patterns, - ) - except (OSError, RuntimeError): - return - - def main(argv: list[str] | None = None) -> int: """CLI entrypoint.""" parser = argparse.ArgumentParser(description=__doc__) diff --git a/tools/tests/python/BUILD.bazel b/tools/tests/python/BUILD.bazel index c1a1f09f..edad2d66 100644 --- a/tools/tests/python/BUILD.bazel +++ b/tools/tests/python/BUILD.bazel @@ -63,6 +63,7 @@ sh_test( "//tools/dev:check_bazelversion_sync.py", "//tools/dev:check_module_versions.py", "//tools/dev:check_release_archive_contents.py", + "//tools/dev:compact_execution_log.py", "//tools/dev:diff_rules_go_fork.py", "//tools/dev:generate_rules_go_consumer_patch.py", "//tools/dev:generate_rules_go_fork_maps.py", diff --git a/tools/tests/python/test_rules_go_profile_tools.py b/tools/tests/python/test_rules_go_profile_tools.py index a5f59a7e..cbf9cf1c 100644 --- a/tools/tests/python/test_rules_go_profile_tools.py +++ b/tools/tests/python/test_rules_go_profile_tools.py @@ -84,6 +84,23 @@ def _copy_tree(src: Path, dst: Path) -> None: shutil.copytree(src, dst, symlinks=True) +def _proto_varint(value: int) -> bytes: + encoded = bytearray() + while value > 0x7F: + encoded.append((value & 0x7F) | 0x80) + value >>= 7 + encoded.append(value) + return bytes(encoded) + + +def _proto_field(number: int, value: int | bytes | str) -> bytes: + if isinstance(value, int): + return _proto_varint(number << 3) + _proto_varint(value) + if isinstance(value, str): + value = value.encode("utf-8") + return _proto_varint((number << 3) | 2) + _proto_varint(len(value)) + value + + class RulesGoProfileToolTests(unittest.TestCase): """Test public profile patch generation behavior.""" @@ -402,8 +419,13 @@ def test_smoke_workspace_wires_hermetic_go_sdk_into_orchestrion(self) -> None: ) workspace_text = (workspace / "WORKSPACE").read_text(encoding="utf-8") + build_text = (workspace / "app/BUILD.bazel").read_text(encoding="utf-8") self.assertIn('go_sdk_root = "@go_sdk//:ROOT"', workspace_text) self.assertIn('go_sdk_version = "1.25.0"', workspace_text) + self.assertIn('name = "hello_test.topt"', build_text) + self.assertIn('orchestrion_mode = "test_optimization"', build_text) + self.assertIn("cgo = True", build_text) + self.assertTrue((workspace / "app/hello_cgo.go").is_file()) def test_run_bazel_scans_captured_output_before_failure_details(self) -> None: """Verifier command wrappers must not leak denylisted command output.""" @@ -543,19 +565,16 @@ def test_action_snapshot_covers_cache_critical_instrumented_actions(self) -> Non self.assertFalse(any("library.a" in path for path in outputs)) def test_reproducibility_flags_mirror_cgo_debug_builds(self) -> None: - """The replay preserves the CGO, debug, LLD, and cache-sensitive inputs.""" - common = self.mod.cgo_reproducibility_flags("darwin") + """The replay mirrors consumer flags without adding compiler policy.""" + common = self.mod.cgo_reproducibility_flags() self.assertIn("--@io_bazel_rules_go//go/config:pure=False", common) + self.assertIn("--@io_bazel_rules_go//go/config:linkmode=normal", common) self.assertIn("--incompatible_strict_action_env", common) self.assertIn("--experimental_platform_in_output_dir", common) self.assertIn("--copt=-g", common) - self.assertNotIn("--linkopt=-Wl,--threads=4", common) - - linux = self.mod.cgo_reproducibility_flags("linux") - self.assertIn("--repo_env=CC=clang", linux) - self.assertIn("--linkopt=-fuse-ld=lld", linux) - self.assertIn("--linkopt=-Wl,--build-id=md5", linux) - self.assertIn("--linkopt=-Wl,--threads=4", linux) + self.assertFalse(any(flag.startswith("--repo_env=CC=") for flag in common)) + self.assertFalse(any(flag.startswith("--linkopt=") for flag in common)) + self.assertNotIn("--copt=-O2", common) def test_reproducibility_aquery_requires_requested_cgo_mode(self) -> None: """Each isolated replay must expose its requested CGO stdlib mode.""" @@ -576,19 +595,104 @@ def test_reproducibility_aquery_requires_requested_cgo_mode(self) -> None: **plain, "arguments": ["builder", "stdlib", "-orchestrion", "orchestrion"], } - self.mod.assert_cgo_reproducibility_actions( - {"actions": [plain]}, "linux", expected_instrumented=False + self.mod.assert_cgo_aquery_actions( + {"actions": [plain]}, expected_instrumented=False ) - self.mod.assert_cgo_reproducibility_actions( - {"actions": [instrumented]}, "linux", expected_instrumented=True + self.mod.assert_cgo_aquery_actions( + {"actions": [instrumented]}, expected_instrumented=True ) with self.assertRaisesRegex(ValueError, "plain CGO-enabled"): - self.mod.assert_cgo_reproducibility_actions( + self.mod.assert_cgo_aquery_actions( {"actions": [instrumented]}, - "linux", expected_instrumented=False, ) + def test_compact_log_requires_test_optimization_transition(self) -> None: + """The compact-log gate rejects global or incomplete instrumentation.""" + def action(*arguments: str): + return self.mod.CompactAction( + target_label="@@io_bazel_rules_go//:stdlib", + mnemonic="GoStdlib", + command_args=arguments, + environment_variables=( + ("CGO_ENABLED", "1"), + ("CGO_CFLAGS", "-g"), + ), + listed_outputs=("bazel-out/stdlib/gocache",), + action_key="key", + actual_outputs=(("bazel-out/stdlib/gocache/net.a", "digest"),), + ) + + valid = action( + "builder", + "-orchestrion", + "external/rules_go_orchestrion_tool/orchestrion", + "-orchestrion_mode", + "test_optimization", + ) + self.mod.assert_cgo_reproducibility_actions( + [valid], expected_instrumented=True + ) + with self.assertRaisesRegex(ValueError, "instrumented CGO-enabled"): + self.mod.assert_cgo_reproducibility_actions( + [action("builder", "-orchestrion", "tool")], + expected_instrumented=True, + ) + + def test_reprise_classifier_only_returns_output_changing_actions(self) -> None: + """The verifier uses the same actionable cells as Reprise's 2x2 table.""" + + def action(key: str, output: str): + return self.mod.CompactAction( + target_label="//app:test.topt__raw_go_test", + mnemonic="GoLink", + command_args=(), + environment_variables=(), + listed_outputs=("bazel-out/app/test",), + action_key=key, + actual_outputs=(("bazel-out/app/test", output),), + ) + + baseline = action("key-a", "output-a") + identity = baseline.identity + self.assertEqual( + [], + self.mod.actionable_reproducibility_findings( + {identity: baseline}, {identity: action("key-a", "output-a")} + ), + ) + self.assertEqual( + [], + self.mod.actionable_reproducibility_findings( + {identity: baseline}, {identity: action("key-b", "output-a")} + ), + ) + tool_finding = self.mod.actionable_reproducibility_findings( + {identity: baseline}, {identity: action("key-a", "output-b")} + ) + self.assertEqual(["tool_nondeterminism"], [item.kind for item in tool_finding]) + input_finding = self.mod.actionable_reproducibility_findings( + {identity: baseline}, {identity: action("key-b", "output-b")} + ) + self.assertEqual(["input_driven"], [item.kind for item in input_finding]) + + def test_reproducibility_action_requires_output_digests(self) -> None: + """A selected action without observed bytes cannot prove determinism.""" + action = self.mod.CompactAction( + target_label="//app:test.topt__raw_go_test", + mnemonic="GoLink", + command_args=(), + environment_variables=(), + listed_outputs=("bazel-out/app/test",), + action_key="key", + actual_outputs=(), + ) + with self.assertRaisesRegex(ValueError, "has no output digests"): + self.mod.select_reproducibility_actions( + [action], + raw_target="//app:test.topt__raw_go_test", + ) + def test_action_output_digest_is_independent_of_its_root(self) -> None: """Logical paths, modes, and bytes determine a declared output digest.""" with tempfile.TemporaryDirectory() as raw_tmp: @@ -631,5 +735,81 @@ def test_plain_stdlib_cache_snapshot_must_be_empty(self) -> None: self.mod.assert_plain_stdlib_cache(nonempty, Path("profile.patch")) +class CompactExecutionLogTests(unittest.TestCase): + """Validate the dependency-free projection of Bazel compact logs.""" + + @classmethod + def setUpClass(cls) -> None: + cls.mod = _load_module( + "compact_execution_log", + "tools/dev/compact_execution_log.py", + ) + + def test_tree_artifact_outputs_expand_like_bazel_json_logs(self) -> None: + """A compact directory output becomes sorted path/digest pairs.""" + digest_a = _proto_field(1, "digest-a") + digest_b = _proto_field(1, "digest-b") + file_a = _proto_field(1, "a.a") + _proto_field(2, digest_a) + file_b = _proto_field(1, "b.a") + _proto_field(2, digest_b) + directory = ( + _proto_field(1, "bazel-out/stdlib") + + _proto_field(2, file_b) + + _proto_field(2, file_a) + ) + directory_entry = _proto_field(1, 1) + _proto_field(4, directory) + output = _proto_field(5, 1) + environment = _proto_field(1, "CGO_ENABLED") + _proto_field(2, "1") + action_digest = _proto_field(1, "action-key") + spawn = ( + _proto_field(1, "builder") + + _proto_field(2, environment) + + _proto_field(6, output) + + _proto_field(7, "@@rules_go//:stdlib") + + _proto_field(8, "GoStdlib") + + _proto_field(16, action_digest) + ) + spawn_entry = _proto_field(7, spawn) + stream = ( + _proto_varint(len(directory_entry)) + + directory_entry + + _proto_varint(len(spawn_entry)) + + spawn_entry + ) + + actions = list(self.mod._decode_entries(stream)) + + self.assertEqual(1, len(actions)) + self.assertEqual("action-key", actions[0].action_key) + self.assertEqual(("bazel-out/stdlib",), actions[0].listed_outputs) + self.assertEqual( + ( + ("bazel-out/stdlib/a.a", "digest-a"), + ("bazel-out/stdlib/b.a", "digest-b"), + ), + actions[0].actual_outputs, + ) + + def test_truncated_entry_reports_an_actionable_error(self) -> None: + """A declared entry length cannot extend beyond the compact stream.""" + with self.assertRaisesRegex( + self.mod.CompactExecutionLogError, + "truncated ExecLogEntry", + ): + list(self.mod._decode_entries(_proto_varint(2) + b"\x08")) + + def test_unknown_spawn_output_reports_an_actionable_error(self) -> None: + """A spawn cannot reference an output absent from the compact stream.""" + output = _proto_field(5, 99) + spawn = _proto_field(6, output) + spawn_entry = _proto_field(7, spawn) + stream = _proto_varint(len(spawn_entry)) + spawn_entry + + with self.assertRaisesRegex( + self.mod.CompactExecutionLogError, + "unknown output id 99", + ): + list(self.mod._decode_entries(stream)) + + if __name__ == "__main__": unittest.main() From a5bd61315dbf1da4d7d6f51a0be5c413c369cb1b Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Thu, 10 Sep 2026 17:05:42 +0200 Subject: [PATCH 5/6] rules_go: preserve Bazel execroot for CGO --- .../rgo/v0_60_0/base/go/tools/builders/cc.go | 16 ++++- .../v0_60_0/base/go/tools/builders/cc_test.go | 40 +++++++++++ .../rgo/v0_61_1/base/go/tools/builders/cc.go | 16 ++++- .../v0_61_1/base/go/tools/builders/cc_test.go | 40 +++++++++++ .../rgo/v0_62_0/base/go/tools/builders/cc.go | 16 ++++- .../v0_62_0/base/go/tools/builders/cc_test.go | 40 +++++++++++ .../rgo/v0_63_0/base/go/tools/builders/cc.go | 16 ++++- .../v0_63_0/base/go/tools/builders/cc_test.go | 40 +++++++++++ .../v0_60_0/base/0001-full-delta.patch | 70 ++++++++++++++++--- .../v0_61_1/base/0001-full-delta.patch | 70 ++++++++++++++++--- .../v0_62_0/base/0001-full-delta.patch | 70 ++++++++++++++++--- .../v0_63_0/base/0001-full-delta.patch | 70 ++++++++++++++++--- tools/dev/verify_rules_go_profiles.py | 41 +++++++---- .../python/test_rules_go_profile_tools.py | 56 +++++++++++++++ 14 files changed, 545 insertions(+), 56 deletions(-) diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/cc.go b/third_party/rgo/v0_60_0/base/go/tools/builders/cc.go index 967f8695..687ec3f8 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/cc.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/cc.go @@ -21,8 +21,14 @@ import ( // for the absolute path and we replace CC with this builder so that // we can expand the placeholder later. func absCCCompiler(envNameList []string, argList []string) error { - wrapped := cgoCompilerWrapperEnv(os.Environ(), envNameList, argList, abs("."), abs(os.Args[0])) - for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC"}, envNameList...) { + wrapped := cgoCompilerWrapperEnv( + os.Environ(), + envNameList, + argList, + moduleProxyResolutionBaseDir, + absolutePathFromBase(os.Args[0], moduleProxyResolutionBaseDir), + ) + for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC", "BAZEL_DD_SANDBOX_EXEC_ROOT"}, envNameList...) { if err := os.Setenv(envName, getEnv(wrapped, envName)); err != nil { return err } @@ -52,6 +58,12 @@ func normalizeGoSubprocessCompilerEnv(environ []string) []string { // anchored to the original execroot. func cgoCompilerWrapperEnv(environ []string, envNameList, argList []string, root, builder string) []string { env := append([]string{}, environ...) + // Go invokes CC from a package-specific temporary directory. Datadog's LLVM + // wrapper uses this value to keep its final debug-prefix mapping anchored to + // the Bazel execroot rather than that temporary directory. + if getEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT") == "" { + env = setEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT", root) + } if goCC := strings.TrimSpace(getEnv(env, "GO_CC")); goCC != "" && getEnv(env, "GO_CC_ROOT") != "" { // GoStdlib already installed the wrapper in the parent process. Keep it // idempotent, but normalize the wrapped compiler for nested go commands. diff --git a/third_party/rgo/v0_60_0/base/go/tools/builders/cc_test.go b/third_party/rgo/v0_60_0/base/go/tools/builders/cc_test.go index b8b4865f..f81dc31f 100644 --- a/third_party/rgo/v0_60_0/base/go/tools/builders/cc_test.go +++ b/third_party/rgo/v0_60_0/base/go/tools/builders/cc_test.go @@ -1,11 +1,51 @@ package main import ( + "os" "path/filepath" "strings" "testing" ) +func TestAbsCCCompilerPreservesInitialExecRoot(t *testing.T) { + initialExecRoot := filepath.Join(t.TempDir(), "execroot", "workspace") + orchestrionWorkDir := filepath.Join(t.TempDir(), "orchestrion-module") + if err := os.MkdirAll(orchestrionWorkDir, 0o755); err != nil { + t.Fatal(err) + } + + previousBaseDir := moduleProxyResolutionBaseDir + moduleProxyResolutionBaseDir = initialExecRoot + t.Cleanup(func() { moduleProxyResolutionBaseDir = previousBaseDir }) + previousWorkDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(orchestrionWorkDir); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(previousWorkDir) }) + + t.Setenv("CC", filepath.Join("external", "llvm_toolchain", "bin", "cc_wrapper.sh")) + t.Setenv("GO_CC", "") + t.Setenv("GO_CC_ROOT", "") + t.Setenv("BAZEL_DD_SANDBOX_EXEC_ROOT", "") + if err := absCCCompiler(cgoEnvVars, cgoAbsEnvFlags); err != nil { + t.Fatal(err) + } + + if got := os.Getenv("GO_CC_ROOT"); got != initialExecRoot { + t.Fatalf("GO_CC_ROOT = %q, want initial execroot %q", got, initialExecRoot) + } + if got := os.Getenv("BAZEL_DD_SANDBOX_EXEC_ROOT"); got != initialExecRoot { + t.Fatalf("BAZEL_DD_SANDBOX_EXEC_ROOT = %q, want initial execroot %q", got, initialExecRoot) + } + wantCompiler := filepath.Join(initialExecRoot, "external", "llvm_toolchain", "bin", "cc_wrapper.sh") + if got := os.Getenv("GO_CC"); got != wantCompiler { + t.Fatalf("GO_CC = %q, want %q", got, wantCompiler) + } +} + func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/cc.go b/third_party/rgo/v0_61_1/base/go/tools/builders/cc.go index 967f8695..687ec3f8 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/cc.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/cc.go @@ -21,8 +21,14 @@ import ( // for the absolute path and we replace CC with this builder so that // we can expand the placeholder later. func absCCCompiler(envNameList []string, argList []string) error { - wrapped := cgoCompilerWrapperEnv(os.Environ(), envNameList, argList, abs("."), abs(os.Args[0])) - for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC"}, envNameList...) { + wrapped := cgoCompilerWrapperEnv( + os.Environ(), + envNameList, + argList, + moduleProxyResolutionBaseDir, + absolutePathFromBase(os.Args[0], moduleProxyResolutionBaseDir), + ) + for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC", "BAZEL_DD_SANDBOX_EXEC_ROOT"}, envNameList...) { if err := os.Setenv(envName, getEnv(wrapped, envName)); err != nil { return err } @@ -52,6 +58,12 @@ func normalizeGoSubprocessCompilerEnv(environ []string) []string { // anchored to the original execroot. func cgoCompilerWrapperEnv(environ []string, envNameList, argList []string, root, builder string) []string { env := append([]string{}, environ...) + // Go invokes CC from a package-specific temporary directory. Datadog's LLVM + // wrapper uses this value to keep its final debug-prefix mapping anchored to + // the Bazel execroot rather than that temporary directory. + if getEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT") == "" { + env = setEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT", root) + } if goCC := strings.TrimSpace(getEnv(env, "GO_CC")); goCC != "" && getEnv(env, "GO_CC_ROOT") != "" { // GoStdlib already installed the wrapper in the parent process. Keep it // idempotent, but normalize the wrapped compiler for nested go commands. diff --git a/third_party/rgo/v0_61_1/base/go/tools/builders/cc_test.go b/third_party/rgo/v0_61_1/base/go/tools/builders/cc_test.go index b8b4865f..f81dc31f 100644 --- a/third_party/rgo/v0_61_1/base/go/tools/builders/cc_test.go +++ b/third_party/rgo/v0_61_1/base/go/tools/builders/cc_test.go @@ -1,11 +1,51 @@ package main import ( + "os" "path/filepath" "strings" "testing" ) +func TestAbsCCCompilerPreservesInitialExecRoot(t *testing.T) { + initialExecRoot := filepath.Join(t.TempDir(), "execroot", "workspace") + orchestrionWorkDir := filepath.Join(t.TempDir(), "orchestrion-module") + if err := os.MkdirAll(orchestrionWorkDir, 0o755); err != nil { + t.Fatal(err) + } + + previousBaseDir := moduleProxyResolutionBaseDir + moduleProxyResolutionBaseDir = initialExecRoot + t.Cleanup(func() { moduleProxyResolutionBaseDir = previousBaseDir }) + previousWorkDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(orchestrionWorkDir); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(previousWorkDir) }) + + t.Setenv("CC", filepath.Join("external", "llvm_toolchain", "bin", "cc_wrapper.sh")) + t.Setenv("GO_CC", "") + t.Setenv("GO_CC_ROOT", "") + t.Setenv("BAZEL_DD_SANDBOX_EXEC_ROOT", "") + if err := absCCCompiler(cgoEnvVars, cgoAbsEnvFlags); err != nil { + t.Fatal(err) + } + + if got := os.Getenv("GO_CC_ROOT"); got != initialExecRoot { + t.Fatalf("GO_CC_ROOT = %q, want initial execroot %q", got, initialExecRoot) + } + if got := os.Getenv("BAZEL_DD_SANDBOX_EXEC_ROOT"); got != initialExecRoot { + t.Fatalf("BAZEL_DD_SANDBOX_EXEC_ROOT = %q, want initial execroot %q", got, initialExecRoot) + } + wantCompiler := filepath.Join(initialExecRoot, "external", "llvm_toolchain", "bin", "cc_wrapper.sh") + if got := os.Getenv("GO_CC"); got != wantCompiler { + t.Fatalf("GO_CC = %q, want %q", got, wantCompiler) + } +} + func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/cc.go b/third_party/rgo/v0_62_0/base/go/tools/builders/cc.go index 967f8695..687ec3f8 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/cc.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/cc.go @@ -21,8 +21,14 @@ import ( // for the absolute path and we replace CC with this builder so that // we can expand the placeholder later. func absCCCompiler(envNameList []string, argList []string) error { - wrapped := cgoCompilerWrapperEnv(os.Environ(), envNameList, argList, abs("."), abs(os.Args[0])) - for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC"}, envNameList...) { + wrapped := cgoCompilerWrapperEnv( + os.Environ(), + envNameList, + argList, + moduleProxyResolutionBaseDir, + absolutePathFromBase(os.Args[0], moduleProxyResolutionBaseDir), + ) + for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC", "BAZEL_DD_SANDBOX_EXEC_ROOT"}, envNameList...) { if err := os.Setenv(envName, getEnv(wrapped, envName)); err != nil { return err } @@ -52,6 +58,12 @@ func normalizeGoSubprocessCompilerEnv(environ []string) []string { // anchored to the original execroot. func cgoCompilerWrapperEnv(environ []string, envNameList, argList []string, root, builder string) []string { env := append([]string{}, environ...) + // Go invokes CC from a package-specific temporary directory. Datadog's LLVM + // wrapper uses this value to keep its final debug-prefix mapping anchored to + // the Bazel execroot rather than that temporary directory. + if getEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT") == "" { + env = setEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT", root) + } if goCC := strings.TrimSpace(getEnv(env, "GO_CC")); goCC != "" && getEnv(env, "GO_CC_ROOT") != "" { // GoStdlib already installed the wrapper in the parent process. Keep it // idempotent, but normalize the wrapped compiler for nested go commands. diff --git a/third_party/rgo/v0_62_0/base/go/tools/builders/cc_test.go b/third_party/rgo/v0_62_0/base/go/tools/builders/cc_test.go index b8b4865f..f81dc31f 100644 --- a/third_party/rgo/v0_62_0/base/go/tools/builders/cc_test.go +++ b/third_party/rgo/v0_62_0/base/go/tools/builders/cc_test.go @@ -1,11 +1,51 @@ package main import ( + "os" "path/filepath" "strings" "testing" ) +func TestAbsCCCompilerPreservesInitialExecRoot(t *testing.T) { + initialExecRoot := filepath.Join(t.TempDir(), "execroot", "workspace") + orchestrionWorkDir := filepath.Join(t.TempDir(), "orchestrion-module") + if err := os.MkdirAll(orchestrionWorkDir, 0o755); err != nil { + t.Fatal(err) + } + + previousBaseDir := moduleProxyResolutionBaseDir + moduleProxyResolutionBaseDir = initialExecRoot + t.Cleanup(func() { moduleProxyResolutionBaseDir = previousBaseDir }) + previousWorkDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(orchestrionWorkDir); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(previousWorkDir) }) + + t.Setenv("CC", filepath.Join("external", "llvm_toolchain", "bin", "cc_wrapper.sh")) + t.Setenv("GO_CC", "") + t.Setenv("GO_CC_ROOT", "") + t.Setenv("BAZEL_DD_SANDBOX_EXEC_ROOT", "") + if err := absCCCompiler(cgoEnvVars, cgoAbsEnvFlags); err != nil { + t.Fatal(err) + } + + if got := os.Getenv("GO_CC_ROOT"); got != initialExecRoot { + t.Fatalf("GO_CC_ROOT = %q, want initial execroot %q", got, initialExecRoot) + } + if got := os.Getenv("BAZEL_DD_SANDBOX_EXEC_ROOT"); got != initialExecRoot { + t.Fatalf("BAZEL_DD_SANDBOX_EXEC_ROOT = %q, want initial execroot %q", got, initialExecRoot) + } + wantCompiler := filepath.Join(initialExecRoot, "external", "llvm_toolchain", "bin", "cc_wrapper.sh") + if got := os.Getenv("GO_CC"); got != wantCompiler { + t.Fatalf("GO_CC = %q, want %q", got, wantCompiler) + } +} + func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") diff --git a/third_party/rgo/v0_63_0/base/go/tools/builders/cc.go b/third_party/rgo/v0_63_0/base/go/tools/builders/cc.go index 967f8695..687ec3f8 100644 --- a/third_party/rgo/v0_63_0/base/go/tools/builders/cc.go +++ b/third_party/rgo/v0_63_0/base/go/tools/builders/cc.go @@ -21,8 +21,14 @@ import ( // for the absolute path and we replace CC with this builder so that // we can expand the placeholder later. func absCCCompiler(envNameList []string, argList []string) error { - wrapped := cgoCompilerWrapperEnv(os.Environ(), envNameList, argList, abs("."), abs(os.Args[0])) - for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC"}, envNameList...) { + wrapped := cgoCompilerWrapperEnv( + os.Environ(), + envNameList, + argList, + moduleProxyResolutionBaseDir, + absolutePathFromBase(os.Args[0], moduleProxyResolutionBaseDir), + ) + for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC", "BAZEL_DD_SANDBOX_EXEC_ROOT"}, envNameList...) { if err := os.Setenv(envName, getEnv(wrapped, envName)); err != nil { return err } @@ -52,6 +58,12 @@ func normalizeGoSubprocessCompilerEnv(environ []string) []string { // anchored to the original execroot. func cgoCompilerWrapperEnv(environ []string, envNameList, argList []string, root, builder string) []string { env := append([]string{}, environ...) + // Go invokes CC from a package-specific temporary directory. Datadog's LLVM + // wrapper uses this value to keep its final debug-prefix mapping anchored to + // the Bazel execroot rather than that temporary directory. + if getEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT") == "" { + env = setEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT", root) + } if goCC := strings.TrimSpace(getEnv(env, "GO_CC")); goCC != "" && getEnv(env, "GO_CC_ROOT") != "" { // GoStdlib already installed the wrapper in the parent process. Keep it // idempotent, but normalize the wrapped compiler for nested go commands. diff --git a/third_party/rgo/v0_63_0/base/go/tools/builders/cc_test.go b/third_party/rgo/v0_63_0/base/go/tools/builders/cc_test.go index b8b4865f..f81dc31f 100644 --- a/third_party/rgo/v0_63_0/base/go/tools/builders/cc_test.go +++ b/third_party/rgo/v0_63_0/base/go/tools/builders/cc_test.go @@ -1,11 +1,51 @@ package main import ( + "os" "path/filepath" "strings" "testing" ) +func TestAbsCCCompilerPreservesInitialExecRoot(t *testing.T) { + initialExecRoot := filepath.Join(t.TempDir(), "execroot", "workspace") + orchestrionWorkDir := filepath.Join(t.TempDir(), "orchestrion-module") + if err := os.MkdirAll(orchestrionWorkDir, 0o755); err != nil { + t.Fatal(err) + } + + previousBaseDir := moduleProxyResolutionBaseDir + moduleProxyResolutionBaseDir = initialExecRoot + t.Cleanup(func() { moduleProxyResolutionBaseDir = previousBaseDir }) + previousWorkDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(orchestrionWorkDir); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(previousWorkDir) }) + + t.Setenv("CC", filepath.Join("external", "llvm_toolchain", "bin", "cc_wrapper.sh")) + t.Setenv("GO_CC", "") + t.Setenv("GO_CC_ROOT", "") + t.Setenv("BAZEL_DD_SANDBOX_EXEC_ROOT", "") + if err := absCCCompiler(cgoEnvVars, cgoAbsEnvFlags); err != nil { + t.Fatal(err) + } + + if got := os.Getenv("GO_CC_ROOT"); got != initialExecRoot { + t.Fatalf("GO_CC_ROOT = %q, want initial execroot %q", got, initialExecRoot) + } + if got := os.Getenv("BAZEL_DD_SANDBOX_EXEC_ROOT"); got != initialExecRoot { + t.Fatalf("BAZEL_DD_SANDBOX_EXEC_ROOT = %q, want initial execroot %q", got, initialExecRoot) + } + wantCompiler := filepath.Join(initialExecRoot, "external", "llvm_toolchain", "bin", "cc_wrapper.sh") + if got := os.Getenv("GO_CC"); got != wantCompiler { + t.Fatalf("GO_CC = %q, want %q", got, wantCompiler) + } +} + func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") diff --git a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch index c468ceae..9b2797f0 100644 --- a/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_60_0/base/0001-full-delta.patch @@ -4753,7 +4753,7 @@ index fdeda2d..76d69b6 100644 } } diff --git a/go/tools/builders/cc.go b/go/tools/builders/cc.go -index 6f67914..967f869 100644 +index 6f67914..687ec3f 100644 --- a/go/tools/builders/cc.go +++ b/go/tools/builders/cc.go @@ -1,7 +1,11 @@ @@ -4768,7 +4768,7 @@ index 6f67914..967f869 100644 "os" "os/exec" "path/filepath" -@@ -10,6 +14,70 @@ import ( +@@ -10,6 +14,82 @@ import ( "syscall" ) @@ -4779,8 +4779,14 @@ index 6f67914..967f869 100644 +// for the absolute path and we replace CC with this builder so that +// we can expand the placeholder later. +func absCCCompiler(envNameList []string, argList []string) error { -+ wrapped := cgoCompilerWrapperEnv(os.Environ(), envNameList, argList, abs("."), abs(os.Args[0])) -+ for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC"}, envNameList...) { ++ wrapped := cgoCompilerWrapperEnv( ++ os.Environ(), ++ envNameList, ++ argList, ++ moduleProxyResolutionBaseDir, ++ absolutePathFromBase(os.Args[0], moduleProxyResolutionBaseDir), ++ ) ++ for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC", "BAZEL_DD_SANDBOX_EXEC_ROOT"}, envNameList...) { + if err := os.Setenv(envName, getEnv(wrapped, envName)); err != nil { + return err + } @@ -4810,6 +4816,12 @@ index 6f67914..967f869 100644 +// anchored to the original execroot. +func cgoCompilerWrapperEnv(environ []string, envNameList, argList []string, root, builder string) []string { + env := append([]string{}, environ...) ++ // Go invokes CC from a package-specific temporary directory. Datadog's LLVM ++ // wrapper uses this value to keep its final debug-prefix mapping anchored to ++ // the Bazel execroot rather than that temporary directory. ++ if getEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT") == "" { ++ env = setEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT", root) ++ } + if goCC := strings.TrimSpace(getEnv(env, "GO_CC")); goCC != "" && getEnv(env, "GO_CC_ROOT") != "" { + // GoStdlib already installed the wrapper in the parent process. Keep it + // idempotent, but normalize the wrapped compiler for nested go commands. @@ -4839,7 +4851,7 @@ index 6f67914..967f869 100644 func cc(args []string) error { cc := os.Getenv("GO_CC") if cc == "" { -@@ -20,7 +88,13 @@ func cc(args []string) error { +@@ -20,7 +100,13 @@ func cc(args []string) error { return errors.New("GO_CC_ROOT environment variable not set") } @@ -4854,7 +4866,7 @@ index 6f67914..967f869 100644 normalized = append(normalized, args...) transformArgs(normalized, cgoAbsEnvFlags, func(s string) string { if strings.HasPrefix(s, cgoAbsPlaceholder) { -@@ -37,6 +111,11 @@ func cc(args []string) error { +@@ -37,6 +123,11 @@ func cc(args []string) error { } return s }) @@ -4866,7 +4878,7 @@ index 6f67914..967f869 100644 if runtime.GOOS == "windows" { cmd := exec.Command(normalized[0], normalized[1:]...) cmd.Stdout = os.Stdout -@@ -46,3 +125,105 @@ func cc(args []string) error { +@@ -46,3 +137,105 @@ func cc(args []string) error { return syscall.Exec(normalized[0], normalized, os.Environ()) } } @@ -4974,18 +4986,58 @@ index 6f67914..967f869 100644 +} diff --git a/go/tools/builders/cc_test.go b/go/tools/builders/cc_test.go new file mode 100644 -index 0000000..b8b4865 +index 0000000..f81dc31 --- /dev/null +++ b/go/tools/builders/cc_test.go -@@ -0,0 +1,74 @@ +@@ -0,0 +1,114 @@ +package main + +import ( ++ "os" + "path/filepath" + "strings" + "testing" +) + ++func TestAbsCCCompilerPreservesInitialExecRoot(t *testing.T) { ++ initialExecRoot := filepath.Join(t.TempDir(), "execroot", "workspace") ++ orchestrionWorkDir := filepath.Join(t.TempDir(), "orchestrion-module") ++ if err := os.MkdirAll(orchestrionWorkDir, 0o755); err != nil { ++ t.Fatal(err) ++ } ++ ++ previousBaseDir := moduleProxyResolutionBaseDir ++ moduleProxyResolutionBaseDir = initialExecRoot ++ t.Cleanup(func() { moduleProxyResolutionBaseDir = previousBaseDir }) ++ previousWorkDir, err := os.Getwd() ++ if err != nil { ++ t.Fatal(err) ++ } ++ if err := os.Chdir(orchestrionWorkDir); err != nil { ++ t.Fatal(err) ++ } ++ t.Cleanup(func() { _ = os.Chdir(previousWorkDir) }) ++ ++ t.Setenv("CC", filepath.Join("external", "llvm_toolchain", "bin", "cc_wrapper.sh")) ++ t.Setenv("GO_CC", "") ++ t.Setenv("GO_CC_ROOT", "") ++ t.Setenv("BAZEL_DD_SANDBOX_EXEC_ROOT", "") ++ if err := absCCCompiler(cgoEnvVars, cgoAbsEnvFlags); err != nil { ++ t.Fatal(err) ++ } ++ ++ if got := os.Getenv("GO_CC_ROOT"); got != initialExecRoot { ++ t.Fatalf("GO_CC_ROOT = %q, want initial execroot %q", got, initialExecRoot) ++ } ++ if got := os.Getenv("BAZEL_DD_SANDBOX_EXEC_ROOT"); got != initialExecRoot { ++ t.Fatalf("BAZEL_DD_SANDBOX_EXEC_ROOT = %q, want initial execroot %q", got, initialExecRoot) ++ } ++ wantCompiler := filepath.Join(initialExecRoot, "external", "llvm_toolchain", "bin", "cc_wrapper.sh") ++ if got := os.Getenv("GO_CC"); got != wantCompiler { ++ t.Fatalf("GO_CC = %q, want %q", got, wantCompiler) ++ } ++} ++ +func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { + firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") + secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") diff --git a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch index 164a0f2d..36bef74c 100644 --- a/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_61_1/base/0001-full-delta.patch @@ -4751,7 +4751,7 @@ index fdeda2d..76d69b6 100644 } } diff --git a/go/tools/builders/cc.go b/go/tools/builders/cc.go -index 6f67914..967f869 100644 +index 6f67914..687ec3f 100644 --- a/go/tools/builders/cc.go +++ b/go/tools/builders/cc.go @@ -1,7 +1,11 @@ @@ -4766,7 +4766,7 @@ index 6f67914..967f869 100644 "os" "os/exec" "path/filepath" -@@ -10,6 +14,70 @@ import ( +@@ -10,6 +14,82 @@ import ( "syscall" ) @@ -4777,8 +4777,14 @@ index 6f67914..967f869 100644 +// for the absolute path and we replace CC with this builder so that +// we can expand the placeholder later. +func absCCCompiler(envNameList []string, argList []string) error { -+ wrapped := cgoCompilerWrapperEnv(os.Environ(), envNameList, argList, abs("."), abs(os.Args[0])) -+ for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC"}, envNameList...) { ++ wrapped := cgoCompilerWrapperEnv( ++ os.Environ(), ++ envNameList, ++ argList, ++ moduleProxyResolutionBaseDir, ++ absolutePathFromBase(os.Args[0], moduleProxyResolutionBaseDir), ++ ) ++ for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC", "BAZEL_DD_SANDBOX_EXEC_ROOT"}, envNameList...) { + if err := os.Setenv(envName, getEnv(wrapped, envName)); err != nil { + return err + } @@ -4808,6 +4814,12 @@ index 6f67914..967f869 100644 +// anchored to the original execroot. +func cgoCompilerWrapperEnv(environ []string, envNameList, argList []string, root, builder string) []string { + env := append([]string{}, environ...) ++ // Go invokes CC from a package-specific temporary directory. Datadog's LLVM ++ // wrapper uses this value to keep its final debug-prefix mapping anchored to ++ // the Bazel execroot rather than that temporary directory. ++ if getEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT") == "" { ++ env = setEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT", root) ++ } + if goCC := strings.TrimSpace(getEnv(env, "GO_CC")); goCC != "" && getEnv(env, "GO_CC_ROOT") != "" { + // GoStdlib already installed the wrapper in the parent process. Keep it + // idempotent, but normalize the wrapped compiler for nested go commands. @@ -4837,7 +4849,7 @@ index 6f67914..967f869 100644 func cc(args []string) error { cc := os.Getenv("GO_CC") if cc == "" { -@@ -20,7 +88,13 @@ func cc(args []string) error { +@@ -20,7 +100,13 @@ func cc(args []string) error { return errors.New("GO_CC_ROOT environment variable not set") } @@ -4852,7 +4864,7 @@ index 6f67914..967f869 100644 normalized = append(normalized, args...) transformArgs(normalized, cgoAbsEnvFlags, func(s string) string { if strings.HasPrefix(s, cgoAbsPlaceholder) { -@@ -37,6 +111,11 @@ func cc(args []string) error { +@@ -37,6 +123,11 @@ func cc(args []string) error { } return s }) @@ -4864,7 +4876,7 @@ index 6f67914..967f869 100644 if runtime.GOOS == "windows" { cmd := exec.Command(normalized[0], normalized[1:]...) cmd.Stdout = os.Stdout -@@ -46,3 +125,105 @@ func cc(args []string) error { +@@ -46,3 +137,105 @@ func cc(args []string) error { return syscall.Exec(normalized[0], normalized, os.Environ()) } } @@ -4972,18 +4984,58 @@ index 6f67914..967f869 100644 +} diff --git a/go/tools/builders/cc_test.go b/go/tools/builders/cc_test.go new file mode 100644 -index 0000000..b8b4865 +index 0000000..f81dc31 --- /dev/null +++ b/go/tools/builders/cc_test.go -@@ -0,0 +1,74 @@ +@@ -0,0 +1,114 @@ +package main + +import ( ++ "os" + "path/filepath" + "strings" + "testing" +) + ++func TestAbsCCCompilerPreservesInitialExecRoot(t *testing.T) { ++ initialExecRoot := filepath.Join(t.TempDir(), "execroot", "workspace") ++ orchestrionWorkDir := filepath.Join(t.TempDir(), "orchestrion-module") ++ if err := os.MkdirAll(orchestrionWorkDir, 0o755); err != nil { ++ t.Fatal(err) ++ } ++ ++ previousBaseDir := moduleProxyResolutionBaseDir ++ moduleProxyResolutionBaseDir = initialExecRoot ++ t.Cleanup(func() { moduleProxyResolutionBaseDir = previousBaseDir }) ++ previousWorkDir, err := os.Getwd() ++ if err != nil { ++ t.Fatal(err) ++ } ++ if err := os.Chdir(orchestrionWorkDir); err != nil { ++ t.Fatal(err) ++ } ++ t.Cleanup(func() { _ = os.Chdir(previousWorkDir) }) ++ ++ t.Setenv("CC", filepath.Join("external", "llvm_toolchain", "bin", "cc_wrapper.sh")) ++ t.Setenv("GO_CC", "") ++ t.Setenv("GO_CC_ROOT", "") ++ t.Setenv("BAZEL_DD_SANDBOX_EXEC_ROOT", "") ++ if err := absCCCompiler(cgoEnvVars, cgoAbsEnvFlags); err != nil { ++ t.Fatal(err) ++ } ++ ++ if got := os.Getenv("GO_CC_ROOT"); got != initialExecRoot { ++ t.Fatalf("GO_CC_ROOT = %q, want initial execroot %q", got, initialExecRoot) ++ } ++ if got := os.Getenv("BAZEL_DD_SANDBOX_EXEC_ROOT"); got != initialExecRoot { ++ t.Fatalf("BAZEL_DD_SANDBOX_EXEC_ROOT = %q, want initial execroot %q", got, initialExecRoot) ++ } ++ wantCompiler := filepath.Join(initialExecRoot, "external", "llvm_toolchain", "bin", "cc_wrapper.sh") ++ if got := os.Getenv("GO_CC"); got != wantCompiler { ++ t.Fatalf("GO_CC = %q, want %q", got, wantCompiler) ++ } ++} ++ +func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { + firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") + secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") diff --git a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch index b3e331b7..a5d754be 100644 --- a/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_62_0/base/0001-full-delta.patch @@ -4755,7 +4755,7 @@ index fdeda2d..76d69b6 100644 } } diff --git a/go/tools/builders/cc.go b/go/tools/builders/cc.go -index 6f67914..967f869 100644 +index 6f67914..687ec3f 100644 --- a/go/tools/builders/cc.go +++ b/go/tools/builders/cc.go @@ -1,7 +1,11 @@ @@ -4770,7 +4770,7 @@ index 6f67914..967f869 100644 "os" "os/exec" "path/filepath" -@@ -10,6 +14,70 @@ import ( +@@ -10,6 +14,82 @@ import ( "syscall" ) @@ -4781,8 +4781,14 @@ index 6f67914..967f869 100644 +// for the absolute path and we replace CC with this builder so that +// we can expand the placeholder later. +func absCCCompiler(envNameList []string, argList []string) error { -+ wrapped := cgoCompilerWrapperEnv(os.Environ(), envNameList, argList, abs("."), abs(os.Args[0])) -+ for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC"}, envNameList...) { ++ wrapped := cgoCompilerWrapperEnv( ++ os.Environ(), ++ envNameList, ++ argList, ++ moduleProxyResolutionBaseDir, ++ absolutePathFromBase(os.Args[0], moduleProxyResolutionBaseDir), ++ ) ++ for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC", "BAZEL_DD_SANDBOX_EXEC_ROOT"}, envNameList...) { + if err := os.Setenv(envName, getEnv(wrapped, envName)); err != nil { + return err + } @@ -4812,6 +4818,12 @@ index 6f67914..967f869 100644 +// anchored to the original execroot. +func cgoCompilerWrapperEnv(environ []string, envNameList, argList []string, root, builder string) []string { + env := append([]string{}, environ...) ++ // Go invokes CC from a package-specific temporary directory. Datadog's LLVM ++ // wrapper uses this value to keep its final debug-prefix mapping anchored to ++ // the Bazel execroot rather than that temporary directory. ++ if getEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT") == "" { ++ env = setEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT", root) ++ } + if goCC := strings.TrimSpace(getEnv(env, "GO_CC")); goCC != "" && getEnv(env, "GO_CC_ROOT") != "" { + // GoStdlib already installed the wrapper in the parent process. Keep it + // idempotent, but normalize the wrapped compiler for nested go commands. @@ -4841,7 +4853,7 @@ index 6f67914..967f869 100644 func cc(args []string) error { cc := os.Getenv("GO_CC") if cc == "" { -@@ -20,7 +88,13 @@ func cc(args []string) error { +@@ -20,7 +100,13 @@ func cc(args []string) error { return errors.New("GO_CC_ROOT environment variable not set") } @@ -4856,7 +4868,7 @@ index 6f67914..967f869 100644 normalized = append(normalized, args...) transformArgs(normalized, cgoAbsEnvFlags, func(s string) string { if strings.HasPrefix(s, cgoAbsPlaceholder) { -@@ -37,6 +111,11 @@ func cc(args []string) error { +@@ -37,6 +123,11 @@ func cc(args []string) error { } return s }) @@ -4868,7 +4880,7 @@ index 6f67914..967f869 100644 if runtime.GOOS == "windows" { cmd := exec.Command(normalized[0], normalized[1:]...) cmd.Stdout = os.Stdout -@@ -46,3 +125,105 @@ func cc(args []string) error { +@@ -46,3 +137,105 @@ func cc(args []string) error { return syscall.Exec(normalized[0], normalized, os.Environ()) } } @@ -4976,18 +4988,58 @@ index 6f67914..967f869 100644 +} diff --git a/go/tools/builders/cc_test.go b/go/tools/builders/cc_test.go new file mode 100644 -index 0000000..b8b4865 +index 0000000..f81dc31 --- /dev/null +++ b/go/tools/builders/cc_test.go -@@ -0,0 +1,74 @@ +@@ -0,0 +1,114 @@ +package main + +import ( ++ "os" + "path/filepath" + "strings" + "testing" +) + ++func TestAbsCCCompilerPreservesInitialExecRoot(t *testing.T) { ++ initialExecRoot := filepath.Join(t.TempDir(), "execroot", "workspace") ++ orchestrionWorkDir := filepath.Join(t.TempDir(), "orchestrion-module") ++ if err := os.MkdirAll(orchestrionWorkDir, 0o755); err != nil { ++ t.Fatal(err) ++ } ++ ++ previousBaseDir := moduleProxyResolutionBaseDir ++ moduleProxyResolutionBaseDir = initialExecRoot ++ t.Cleanup(func() { moduleProxyResolutionBaseDir = previousBaseDir }) ++ previousWorkDir, err := os.Getwd() ++ if err != nil { ++ t.Fatal(err) ++ } ++ if err := os.Chdir(orchestrionWorkDir); err != nil { ++ t.Fatal(err) ++ } ++ t.Cleanup(func() { _ = os.Chdir(previousWorkDir) }) ++ ++ t.Setenv("CC", filepath.Join("external", "llvm_toolchain", "bin", "cc_wrapper.sh")) ++ t.Setenv("GO_CC", "") ++ t.Setenv("GO_CC_ROOT", "") ++ t.Setenv("BAZEL_DD_SANDBOX_EXEC_ROOT", "") ++ if err := absCCCompiler(cgoEnvVars, cgoAbsEnvFlags); err != nil { ++ t.Fatal(err) ++ } ++ ++ if got := os.Getenv("GO_CC_ROOT"); got != initialExecRoot { ++ t.Fatalf("GO_CC_ROOT = %q, want initial execroot %q", got, initialExecRoot) ++ } ++ if got := os.Getenv("BAZEL_DD_SANDBOX_EXEC_ROOT"); got != initialExecRoot { ++ t.Fatalf("BAZEL_DD_SANDBOX_EXEC_ROOT = %q, want initial execroot %q", got, initialExecRoot) ++ } ++ wantCompiler := filepath.Join(initialExecRoot, "external", "llvm_toolchain", "bin", "cc_wrapper.sh") ++ if got := os.Getenv("GO_CC"); got != wantCompiler { ++ t.Fatalf("GO_CC = %q, want %q", got, wantCompiler) ++ } ++} ++ +func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { + firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") + secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") diff --git a/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch b/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch index e780bd4c..3a7371be 100644 --- a/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch +++ b/third_party/rules_go_orchestrion/patches/v0_63_0/base/0001-full-delta.patch @@ -3734,7 +3734,7 @@ index fdeda2d..76d69b6 100644 } } diff --git a/go/tools/builders/cc.go b/go/tools/builders/cc.go -index 6f67914..967f869 100644 +index 6f67914..687ec3f 100644 --- a/go/tools/builders/cc.go +++ b/go/tools/builders/cc.go @@ -1,7 +1,11 @@ @@ -3749,7 +3749,7 @@ index 6f67914..967f869 100644 "os" "os/exec" "path/filepath" -@@ -10,6 +14,70 @@ import ( +@@ -10,6 +14,82 @@ import ( "syscall" ) @@ -3760,8 +3760,14 @@ index 6f67914..967f869 100644 +// for the absolute path and we replace CC with this builder so that +// we can expand the placeholder later. +func absCCCompiler(envNameList []string, argList []string) error { -+ wrapped := cgoCompilerWrapperEnv(os.Environ(), envNameList, argList, abs("."), abs(os.Args[0])) -+ for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC"}, envNameList...) { ++ wrapped := cgoCompilerWrapperEnv( ++ os.Environ(), ++ envNameList, ++ argList, ++ moduleProxyResolutionBaseDir, ++ absolutePathFromBase(os.Args[0], moduleProxyResolutionBaseDir), ++ ) ++ for _, envName := range append([]string{"GO_CC", "GO_CC_ROOT", "CC", "BAZEL_DD_SANDBOX_EXEC_ROOT"}, envNameList...) { + if err := os.Setenv(envName, getEnv(wrapped, envName)); err != nil { + return err + } @@ -3791,6 +3797,12 @@ index 6f67914..967f869 100644 +// anchored to the original execroot. +func cgoCompilerWrapperEnv(environ []string, envNameList, argList []string, root, builder string) []string { + env := append([]string{}, environ...) ++ // Go invokes CC from a package-specific temporary directory. Datadog's LLVM ++ // wrapper uses this value to keep its final debug-prefix mapping anchored to ++ // the Bazel execroot rather than that temporary directory. ++ if getEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT") == "" { ++ env = setEnv(env, "BAZEL_DD_SANDBOX_EXEC_ROOT", root) ++ } + if goCC := strings.TrimSpace(getEnv(env, "GO_CC")); goCC != "" && getEnv(env, "GO_CC_ROOT") != "" { + // GoStdlib already installed the wrapper in the parent process. Keep it + // idempotent, but normalize the wrapped compiler for nested go commands. @@ -3820,7 +3832,7 @@ index 6f67914..967f869 100644 func cc(args []string) error { cc := os.Getenv("GO_CC") if cc == "" { -@@ -20,7 +88,13 @@ func cc(args []string) error { +@@ -20,7 +100,13 @@ func cc(args []string) error { return errors.New("GO_CC_ROOT environment variable not set") } @@ -3835,7 +3847,7 @@ index 6f67914..967f869 100644 normalized = append(normalized, args...) transformArgs(normalized, cgoAbsEnvFlags, func(s string) string { if strings.HasPrefix(s, cgoAbsPlaceholder) { -@@ -37,6 +111,11 @@ func cc(args []string) error { +@@ -37,6 +123,11 @@ func cc(args []string) error { } return s }) @@ -3847,7 +3859,7 @@ index 6f67914..967f869 100644 if runtime.GOOS == "windows" { cmd := exec.Command(normalized[0], normalized[1:]...) cmd.Stdout = os.Stdout -@@ -46,3 +125,105 @@ func cc(args []string) error { +@@ -46,3 +137,105 @@ func cc(args []string) error { return syscall.Exec(normalized[0], normalized, os.Environ()) } } @@ -3955,18 +3967,58 @@ index 6f67914..967f869 100644 +} diff --git a/go/tools/builders/cc_test.go b/go/tools/builders/cc_test.go new file mode 100644 -index 0000000..b8b4865 +index 0000000..f81dc31 --- /dev/null +++ b/go/tools/builders/cc_test.go -@@ -0,0 +1,74 @@ +@@ -0,0 +1,114 @@ +package main + +import ( ++ "os" + "path/filepath" + "strings" + "testing" +) + ++func TestAbsCCCompilerPreservesInitialExecRoot(t *testing.T) { ++ initialExecRoot := filepath.Join(t.TempDir(), "execroot", "workspace") ++ orchestrionWorkDir := filepath.Join(t.TempDir(), "orchestrion-module") ++ if err := os.MkdirAll(orchestrionWorkDir, 0o755); err != nil { ++ t.Fatal(err) ++ } ++ ++ previousBaseDir := moduleProxyResolutionBaseDir ++ moduleProxyResolutionBaseDir = initialExecRoot ++ t.Cleanup(func() { moduleProxyResolutionBaseDir = previousBaseDir }) ++ previousWorkDir, err := os.Getwd() ++ if err != nil { ++ t.Fatal(err) ++ } ++ if err := os.Chdir(orchestrionWorkDir); err != nil { ++ t.Fatal(err) ++ } ++ t.Cleanup(func() { _ = os.Chdir(previousWorkDir) }) ++ ++ t.Setenv("CC", filepath.Join("external", "llvm_toolchain", "bin", "cc_wrapper.sh")) ++ t.Setenv("GO_CC", "") ++ t.Setenv("GO_CC_ROOT", "") ++ t.Setenv("BAZEL_DD_SANDBOX_EXEC_ROOT", "") ++ if err := absCCCompiler(cgoEnvVars, cgoAbsEnvFlags); err != nil { ++ t.Fatal(err) ++ } ++ ++ if got := os.Getenv("GO_CC_ROOT"); got != initialExecRoot { ++ t.Fatalf("GO_CC_ROOT = %q, want initial execroot %q", got, initialExecRoot) ++ } ++ if got := os.Getenv("BAZEL_DD_SANDBOX_EXEC_ROOT"); got != initialExecRoot { ++ t.Fatalf("BAZEL_DD_SANDBOX_EXEC_ROOT = %q, want initial execroot %q", got, initialExecRoot) ++ } ++ wantCompiler := filepath.Join(initialExecRoot, "external", "llvm_toolchain", "bin", "cc_wrapper.sh") ++ if got := os.Getenv("GO_CC"); got != wantCompiler { ++ t.Fatalf("GO_CC = %q, want %q", got, wantCompiler) ++ } ++} ++ +func TestNormalizeCgoRandomSeedIgnoresEphemeralRoots(t *testing.T) { + firstCCRoot := filepath.Join("tmp", "source", "first", "workspace") + secondCCRoot := filepath.Join("tmp", "source", "second", "workspace") diff --git a/tools/dev/verify_rules_go_profiles.py b/tools/dev/verify_rules_go_profiles.py index 3ee42a71..e3fb8d12 100644 --- a/tools/dev/verify_rules_go_profiles.py +++ b/tools/dev/verify_rules_go_profiles.py @@ -673,10 +673,6 @@ def select_reproducibility_actions( if action.identity in selected: raise ValueError("duplicate compact-log action identity: %s" % (action.identity,)) - if not action.action_key: - raise ValueError( - "compact-log action has no cache digest: %s" % (action.identity,) - ) if not action.actual_outputs: raise ValueError( "compact-log action has no output digests: %s" % (action.identity,) @@ -692,7 +688,7 @@ def bazel_labels_match(actual: str, expected: str) -> bool: @dataclass(frozen=True) class ReproducibilityFinding: - """One actionable cell from Reprise's action-key/output classifier.""" + """One output or action-set difference between independent builds.""" action: CompactAction kind: str @@ -703,11 +699,24 @@ def actionable_reproducibility_findings( first: dict[tuple[str, str, tuple[str, ...]], CompactAction], second: dict[tuple[str, str, tuple[str, ...]], CompactAction], ) -> list[ReproducibilityFinding]: - """Apply Reprise's 2x2 classifier and return output-changing findings.""" + """Return output-changing findings, even when cache digests are absent.""" findings = [] - for identity in sorted(set(first) & set(second)): - left = first[identity] - right = second[identity] + for identity in sorted(set(first) | set(second)): + left = first.get(identity) + right = second.get(identity) + if left is None or right is None: + action = left or right + if action is None: + continue + outputs = tuple(path for path, _ in action.actual_outputs) + findings.append( + ReproducibilityFinding( + action=action, + kind="action_set_changed", + differing_outputs=outputs or action.listed_outputs, + ) + ) + continue if left.actual_outputs == right.actual_outputs: # A changed action key with identical bytes is Reprise's # non-actionable wasted_rebuild case. @@ -722,11 +731,16 @@ def actionable_reproducibility_findings( differing += tuple( path for path, _ in right.actual_outputs if path not in left_paths ) - keys_match = bool(left.action_key) and left.action_key == right.action_key + if not left.action_key or not right.action_key: + kind = "output_drift_without_action_key" + elif left.action_key == right.action_key: + kind = "tool_nondeterminism" + else: + kind = "input_driven" findings.append( ReproducibilityFinding( action=left, - kind="tool_nondeterminism" if keys_match else "input_driven", + kind=kind, differing_outputs=differing, ) ) @@ -1186,7 +1200,10 @@ def smoke_bazel_env(output_user_root: Path) -> dict[str, str]: "PATH": os.pathsep.join(dict.fromkeys(path_entries)), "TMPDIR": smoke_tmp.as_posix(), "USER": "rules_go_smoke", - "USE_BAZEL_VERSION": (REPO_ROOT / ".bazelversion").read_text().strip(), + "USE_BAZEL_VERSION": os.environ.get( + "USE_BAZEL_VERSION", + (REPO_ROOT / ".bazelversion").read_text().strip(), + ), } for key in ("JAVA_HOME", "SSL_CERT_FILE", "REQUESTS_CA_BUNDLE"): if key in os.environ: diff --git a/tools/tests/python/test_rules_go_profile_tools.py b/tools/tests/python/test_rules_go_profile_tools.py index cbf9cf1c..169810ce 100644 --- a/tools/tests/python/test_rules_go_profile_tools.py +++ b/tools/tests/python/test_rules_go_profile_tools.py @@ -21,6 +21,7 @@ import tempfile import types import unittest +from unittest import mock def _runfile(rel_path: str) -> Path: @@ -446,6 +447,14 @@ def test_run_bazel_scans_captured_output_before_failure_details(self) -> None: private_safe_patterns=["DENYLIST_SENTINEL"], ) + def test_smoke_environment_honors_explicit_bazel_version(self) -> None: + """Consumer reproductions may select a Bazel version without editing the repo.""" + with tempfile.TemporaryDirectory() as raw_tmp: + with mock.patch.dict(os.environ, {"USE_BAZEL_VERSION": "8.8.0"}): + env = self.mod.smoke_bazel_env(Path(raw_tmp) / "output-user-root") + + self.assertEqual("8.8.0", env["USE_BAZEL_VERSION"]) + def test_stdlib_cache_snapshot_accepts_manifested_data_entries(self) -> None: """The determinism verifier accepts only sorted manifested data entries.""" with tempfile.TemporaryDirectory() as raw_tmp: @@ -676,6 +685,53 @@ def action(key: str, output: str): ) self.assertEqual(["input_driven"], [item.kind for item in input_finding]) + missing_key_finding = self.mod.actionable_reproducibility_findings( + {identity: action("", "output-a")}, + {identity: action("", "output-b")}, + ) + self.assertEqual( + ["output_drift_without_action_key"], + [item.kind for item in missing_key_finding], + ) + + def test_reprise_classifier_rejects_action_set_drift(self) -> None: + """An action missing from either cold run makes the comparison incomplete.""" + action = self.mod.CompactAction( + target_label="//app:test.topt__raw_go_test", + mnemonic="GoLink", + command_args=(), + environment_variables=(), + listed_outputs=("bazel-out/app/test",), + action_key="", + actual_outputs=(("bazel-out/app/test", "output"),), + ) + + findings = self.mod.actionable_reproducibility_findings( + {action.identity: action}, + {}, + ) + + self.assertEqual(["action_set_changed"], [item.kind for item in findings]) + + def test_reproducibility_action_allows_missing_cache_digest(self) -> None: + """Cache-off Reprise logs still prove determinism from output digests.""" + action = self.mod.CompactAction( + target_label="//app:test.topt__raw_go_test", + mnemonic="GoLink", + command_args=(), + environment_variables=(), + listed_outputs=("bazel-out/app/test",), + action_key="", + actual_outputs=(("bazel-out/app/test", "output"),), + ) + + selected = self.mod.select_reproducibility_actions( + [action], + raw_target="//app:test.topt__raw_go_test", + ) + + self.assertEqual({action.identity: action}, selected) + def test_reproducibility_action_requires_output_digests(self) -> None: """A selected action without observed bytes cannot prove determinism.""" action = self.mod.CompactAction( From 6b2e253f84729353bbdb11e1a9b457e7e0f50db9 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Fri, 11 Sep 2026 13:23:51 +0200 Subject: [PATCH 6/6] docs: align guides with current runtime behavior --- AGENTS.md | 5 +- CHANGELOG.md | 7 + CONTRIBUTING.md | 2 +- README.md | 56 ++-- docs/Configuration_Reference.md | 2 +- docs/Initial_documentation.md | 72 ++++-- docs/Installation_Reference.md | 10 +- docs/Language_Onboarding.md | 7 + docs/Maintainers.md | 5 + docs/RFC.md | 2 +- docs/Uploader_Reference.md | 42 ++- docs/go_orchestrion_bazel_deep_dive.md | 239 +++++------------- docs/go_orchestrion_maintainer_state.md | 88 ++++--- ...python_parallel_uploader_migration_plan.md | 48 ++-- ...rules_go_orchestrion_probe_measurements.md | 4 +- .../rules_go_orchestrion_support_selection.md | 14 +- .../rules_go_stdlib_cache_determinism_plan.md | 104 ++++---- docs/rules_go_variant_maintenance_guide.md | 14 +- .../go-test-optimization-onboarding/SKILL.md | 34 ++- .../references/validation-checklist.md | 35 ++- .../SKILL.md | 28 +- .../references/validation-checklist.md | 16 +- .../SKILL.md | 28 +- .../references/validation-checklist.md | 18 +- .../SKILL.md | 27 +- .../references/migration-workflow.md | 16 +- .../references/troubleshooting.md | 5 +- .../references/validation-checklist.md | 25 +- 28 files changed, 548 insertions(+), 405 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a0bd027c..fbefc81c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -74,7 +74,10 @@ The sync rule creates `@test_optimization_data//` containing: - **Sanitization**: module names are converted into Bazel-safe labels using `sanitize_label_fragment()` (lowercase, `[a-z0-9_]` only, deterministic suffixes). - **Go importpath inference**: `topt_go_payloads_selector` mirrors rules_go importpath logic (explicit `importpath` > `embed` provider > fallback `/`). - **Vendored rules_go forks for root workflows**: the repository root pins `rules_go` as a dev-only dependency and redirects it to `third_party/rgo/v0_60_0/base` with `local_path_override(...)`; consumer-facing core usage remains rules_go-free. -- **Cross-platform uploader**: Unix uses Bash/curl; Windows uses PowerShell and .NET `HttpClient`. +- **Cross-platform uploader**: Python 3.10+ provides one implementation on + Linux, macOS, and Windows. Small Bash and PowerShell launchers locate Python + and Bazel runfiles. The former Bash/curl and PowerShell/.NET uploaders remain + available only through the explicit `use_python_uploader = False` rollback. ## Build, Test, and Development Commands - Canonical validation command matrix lives in `CONTRIBUTING.md`; keep this diff --git a/CHANGELOG.md b/CHANGELOG.md index faa71c76..51e389da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,13 @@ versioning. - Config-disabled Go analysis now resolves stable empty Orchestrion repository targets before host-Go discovery or source fetching, so ordinary targets do not require Go to be installed merely because the integration is declared. +- Generated-profile verification now checks deterministic output bytes for + `GoStdlib`, `GoSyntheticTestmainHelpers`, synthetic `GoCompilePkg`, and + `GoLink` actions across every maintained `rules_go` version. The builder + normalizes ephemeral execroot and Go work paths, CGO random seeds, source + trim paths, and copied helper build IDs while preserving relative CGO input + resolution. Ordinary Go actions cannot consume the instrumented stdlib + cache. ## [1.2.0] - 2026-06-03 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1750de2f..7b3c1ead 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -95,7 +95,7 @@ This product includes software developed at Datadog `--config=test-optimization` as the only user-facing switch. The shared config entry is `common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1`. - - Go additionally sets the existing `rules_go` Orchestrion `enabled=true` + - Go also sets the existing `rules_go` Orchestrion `enabled=true` build setting. Python-only consumers must not declare that Go-only label. - Omitting the config is the documented complete opt-out for Go and Python: metadata repositories use disabled stubs when `enabled_by_env = True`, Go diff --git a/README.md b/README.md index f39ba8f8..f233348e 100644 --- a/README.md +++ b/README.md @@ -556,6 +556,12 @@ Disabled or mismatched repositories fail analysis. This descriptor is an additive option; generated single-, multi-, and manifest-sync exports keep their existing behavior. +The `.topt` clone preserves the source test's execution policy. A source tagged +`no-remote-exec` therefore still runs its `TestRunner` locally, but that policy +does not leak into deterministic build actions such as stdlib preparation, +synthetic testmain helpers, compilation, or linking. Those actions remain +eligible for normal Bazel caching and remote execution. + Large consumers can warm the matching instrumented standard library in a dedicated cache-writing invocation: @@ -572,8 +578,9 @@ bazel build //tools/test_optimization:go_stdlib_warmup \ ``` The warmup target applies the same target-scoped Go configuration transitions, -enables Orchestrion in `test_optimization` mode, and materializes both the -instrumented stdlib and its Go build cache. The consumer +enables Orchestrion in `test_optimization` mode, and materializes the +instrumented stdlib plus its deterministic declared archive cache. Ordinary Go +actions neither publish nor consume that instrumented cache. The consumer remains responsible for restricting remote-cache writes to a trusted CI invocation. Without the named config, the target is a no-op so ordinary broad builds can @@ -1472,14 +1479,26 @@ The uploader is a normal Bazel rule (not a test) that runs via `bazel run` after 4. Then run the uploader via `bazel run` 5. The uploader discovers all `test.outputs/` directories, waits for quiescence, uploads, and deletes files -Before transport, the uploader enriches each test payload and splits any body -larger than 4,500,000 bytes along its `events` array. Parts preserve event order -and the original top-level envelope, are compressed independently when gzip is -enabled, and use independent retries. The original payload is deleted only -after every part uploads successfully. A terminal upload failure reports the -HTTP status, a bounded response body, and uncompressed/compressed/transmitted -sizes without requiring debug mode. A single event above the 5,000,000-byte -intake limit is rejected because it cannot be split safely. +The default Python runtime prepares CODEOWNERS, contexts, schemas, freshness, +and telemetry plans once. It then gives each source file to one of eight +independent workers. That worker owns enrichment, optional validation, +preventive splitting, upload retries, and cleanup for the file, whether it +contains test, coverage, or telemetry data. Starting several uploader processes +is neither necessary nor supported. + +After enrichment, a worker compacts a test payload and splits it along its +`events` array before any HTTP request when the encoded body would exceed +`4_718_592` bytes (4.5 MiB). Chunks preserve event order and the original +top-level envelope, and each chunk has its own retry lifecycle. HTTP `413` is a +terminal contract error: the uploader has already applied the preventive limit, +so retrying or splitting adaptively would hide a defect. A single event that +cannot fit within the limit is rejected locally without spending a request. + +At the end of the run, the uploader prints deterministic totals for files, +payload types, chunks, requests, retries, and cleanup. Use `--debug` for verbose, +redacted diagnostics. Use `--dry-run --validate-enrichment` to exercise +discovery, enrichment, validation, splitting, and request preparation without +contacting the backend or deleting source files. Telemetry-specific notes: - Telemetry files must contain one raw top-level tracer telemetry request body per file. @@ -1522,12 +1541,12 @@ $env:DD_SITE = "datadoghq.com" //... ``` -**IMPORTANT**: Always preserve the test exit code. When upload is enabled, the +Always preserve the test exit code. When upload is enabled, the wrapper runs the doctor, then validates enrichment and uploads every available fresh valid payload in one uploader pass even if tests or doctor failed. The earliest test, doctor, or uploader failure remains the job result. -### Important runtime requirements +### Runtime requirements 1. Use `bazel run` (not `bazel test`) for uploader execution. 2. Use a single uploader target per workspace (do not run concurrent uploaders). @@ -1720,13 +1739,12 @@ an `outputs.zip` archive to the requested `--output` path before the downloader contract; it does not ship credentials or a Datadog-internal CAS client. -Artifact staging requires Python at uploader runtime. Bash resolves -`DD_TEST_OPTIMIZATION_PYTHON`, then `PYTHON`, then `python3`, then `python`; -PowerShell uses the same discovery order. Existing local-only uploader flows -remain usable without Python except for support bundle generation and the -pre-existing optional schema and telemetry helpers. Bash BEP freshness parsing -still requires `jq` whenever BEP freshness validation is enabled in the Bash -uploader path. +The default uploader and artifact staging require Python 3.10 or newer. The +small Bash and PowerShell launchers resolve `DD_TEST_OPTIMIZATION_PYTHON`, then +`PYTHON`, then `python3`, then `python`. Only the explicit +`use_python_uploader = False` rollback path can upload local files without +Python. That legacy Bash path still requires `jq` for BEP freshness parsing, +schema validation, enrichment, and oversized-payload splitting. ### Enrichment validation diff --git a/docs/Configuration_Reference.md b/docs/Configuration_Reference.md index 99e286ef..b6d03222 100644 --- a/docs/Configuration_Reference.md +++ b/docs/Configuration_Reference.md @@ -211,7 +211,7 @@ Extension tag: `test_optimization_sync.test_optimization_sync(...)` | `test_management` | bool | `True` | Local switch for Test Management request. When `False`, request is skipped, a minimal stub is written, and settings are mutated to `test_management.enabled=false` | | `flaky_tests` | bool | `True` | Local switch for Flaky Tests request. When `False`, request is skipped, a minimal stub is written, and settings are mutated to `flaky_test_retries_enabled=false` | | `enabled` | bool | `True` | Hard enablement switch. When `False`, the repository emits the deterministic disabled interface and skips local Git discovery and metadata HTTP requests | -| `enabled_by_env` | bool | `False` | When `True`, additionally gate enablement on `DD_TEST_OPTIMIZATION_ENABLED` (`1`, `true`, `yes`, or `on`, case-insensitive). Unset and false values emit the disabled interface. The public Go extension and Go WORKSPACE helper override this low-level default to `True` | +| `enabled_by_env` | bool | `False` | When `True`, also gate enablement on `DD_TEST_OPTIMIZATION_ENABLED` (`1`, `true`, `yes`, or `on`, case-insensitive). Unset and false values emit the disabled interface. The public Go extension and Go WORKSPACE helper override this low-level default to `True` | | `require_git_metadata` | bool | `False` | Strict local/CI validation for settings-request Git metadata. When `True`, sync fails before HTTP if repository URL, branch or tag, and commit SHA cannot be resolved | | `debug` | bool | `False` | Enables verbose repository-rule logging | diff --git a/docs/Initial_documentation.md b/docs/Initial_documentation.md index da90add2..4bc4423c 100644 --- a/docs/Initial_documentation.md +++ b/docs/Initial_documentation.md @@ -11,9 +11,9 @@ This product includes software developed at Datadog This document explains the current implementation architecture in this repository. For installation and day-to-day usage, start with `README.md`. -> Last reviewed: 2026-07-27 +> Last reviewed: 2026-09-11 -## Approach Overview +## Approach overview The integration uses a Bazel module extension and repository rule to materialize the Test Optimization repository during module/repo resolution, a @@ -37,7 +37,7 @@ The steps are: - `@//:test_optimization_files` (core bundle, includes `cache/http/settings.json`) - `@//:test_optimization_context` (`context.json` plus `telemetry_facts.json`) - When an enabled response contains module data, the repository additionally + When an enabled response contains module data, the repository also exposes `@//:module_` bundles with `cache/http/settings.json` plus that module's known-tests, test-management, and flaky-tests files. @@ -59,7 +59,16 @@ The steps are: targets, or payload instrumentation. 3. **Payload validation and reporting**: - A single workspace-level doctor runs via `bazel run` after tests complete and validates local JSON payloads, Bazel target metadata, Git metadata, and invalid Go payload-selection states. A single workspace-level uploader then discovers all `test.outputs/` directories in `bazel-testlogs/`, waits for payloads to quiesce, enriches them with `context.json`, and uploads via agentless (`DD_API_KEY`, `DD_SITE`) or EVP proxy (`DD_TEST_OPTIMIZATION_AGENT_URL`). + A single workspace-level doctor runs via `bazel run` after tests complete and + validates local JSON payloads, Bazel target metadata, Git metadata, and + invalid payload-selection states. A single workspace-level uploader then + discovers local or BEP-staged `test.outputs/` directories and waits for + payloads to quiesce. Its coordinator prepares CODEOWNERS, contexts, schemas, + freshness, and telemetry plans once before starting a bounded pool of eight + workers by default. Each worker owns one source file through enrichment, + optional validation, preventive 4.5 MiB splitting for test payloads, upload + retries, and cleanup. A worker can process test, coverage, or telemetry data; + workers do not synchronize with one another. In mixed-runtime workspaces, the uploader can bundle multiple `context.json` files and select the matching one per payload using sibling `bazel_target_metadata.json` repo metadata instead of reusing one global @@ -69,10 +78,26 @@ The steps are: can use root labels; large monorepos should use a lightweight package such as `//tools/test_optimization`. Usage: run `bazel test`, then the doctor target, then one uploader pass with - `--validate-enrichment`; add `--dry-run` only when upload is disabled. - Preserve the earliest failure while still processing every available fresh + `--validate-enrichment`; add `--dry-run` only when upload is disabled. The + uploader prints final file, type, split, request, retry, and cleanup totals. + `--debug` adds verbose redacted diagnostics. Preserve the earliest test, + doctor, or uploader failure while still processing every available fresh valid payload. + ```mermaid + flowchart LR + D[Discover fresh source files] --> P[Prepare shared context and CODEOWNERS] + P --> Q[Bounded file queue] + Q --> W1[Worker 1: enrich, validate, split, send, clean] + Q --> WN[Worker N: enrich, validate, split, send, clean] + W1 --> S[Aggregate final statistics] + WN --> S + ``` + + Each worker owns its file from dequeue to final result. There is no serial + split or upload stage after enrichment, and workers do not exchange payload + state. + 4. **Language macros (optional)**: Thin wrappers (for Go/Python/Java/NodeJS/.NET/Ruby) set up the right runfiles/env so test code can read the synced files and write payloads to `TEST_UNDECLARED_OUTPUTS_DIR`. - Core module (`datadog-rules-test-optimization`) stays runtime-agnostic. @@ -281,16 +306,24 @@ generated per-service sync repository keys used by payload metadata. ## Runtime uploads and hermetic tests -Tests remain hermetic with network blocked. They write payloads to Bazel's built-in `TEST_UNDECLARED_OUTPUTS_DIR/payloads/{tests,coverage}`, which is automatically collected to `bazel-testlogs//test.outputs/`. A single workspace-level doctor validates those local outputs before upload. A single workspace-level uploader (via `bazel run`) then: +Tests remain hermetic with network blocked. They write payloads to Bazel's +built-in `TEST_UNDECLARED_OUTPUTS_DIR/payloads/{tests,coverage,telemetry}`, +which Bazel collects under `bazel-testlogs//test.outputs/`. A single +workspace-level doctor validates those outputs before upload. One uploader +process then: - Discovers all `test.outputs/` directories in `bazel-testlogs/`, - Waits for filesystem quiescence, -- Enriches test payloads with `context.json` when present, +- Prepares CODEOWNERS, contexts, schemas, freshness, and telemetry state once, +- Assigns each source file to one of eight workers by default, +- Lets that worker enrich, validate, split when required, upload with retries, + and clean up the file, - When multiple bundled contexts are present, matches them per payload using `bazel.test_optimization.repo_name` from sibling `bazel_target_metadata.json`, - Can dry-run the enrichment path without uploading or deleting files, - Uploads to Datadog using either `DD_API_KEY`/`DD_SITE` (agentless) or `DD_TEST_OPTIMIZATION_AGENT_URL` (EVP proxy), -- Deletes successfully uploaded payloads. +- Deletes successfully uploaded payloads, +- Prints aggregate statistics after all workers finish. No secrets are written to disk; all credentials are passed via environment variables. @@ -335,7 +368,7 @@ flowchart TD %% Test execution: hermetic, offline subgraph T[Test Execution (Hermetic)] T1[Tests (instrumented)] - P1[bazel-testlogs/.../test.outputs/\n payloads/tests/*.json\n payloads/coverage/*.json] + P1[bazel-testlogs/.../test.outputs/\n payloads/tests/*.json\n payloads/coverage/*.json\n payloads/telemetry/*.json] T1 -->|read runfiles| A3 T1 -->|write to TEST_UNDECLARED_OUTPUTS_DIR| P1 end @@ -343,13 +376,16 @@ flowchart TD %% Validate/upload steps: bazel run after tests subgraph U[Validate and upload via bazel run] U0[Doctor rule] - U1[Uploader rule] + U1[Uploader coordinator] + UW[Bounded file workers\n enrich, validate, split, send, clean] + US[Final statistics] U0 -->|validate| P1 U0 -->|validate context| A3 U0 --> U1 - U1 -->|enrich with| A3 - U1 -->|upload tests| G1{Agentless?\n DD_API_KEY} - U1 -->|upload coverage| G1 + U1 -->|prepare shared context| A3 + U1 --> UW + UW -->|upload tests, coverage, telemetry| G1{Agentless?\n DD_API_KEY} + UW --> US G1 -- Yes --> I1[(citestcycle/citestcov\n intake on )] G1 -- No --> I2[(EVP proxy\n ${DD_TEST_OPTIMIZATION_AGENT_URL})] end @@ -387,13 +423,15 @@ Build Graph Test Execution (Hermetic) [tests (instrumented)] --read runfiles--> synced JSONs - --write payloads--> TEST_UNDECLARED_OUTPUTS_DIR/payloads/{tests,coverage} (-> bazel-testlogs/.../test.outputs/) + --write payloads--> TEST_UNDECLARED_OUTPUTS_DIR/payloads/{tests,coverage,telemetry} (-> bazel-testlogs/.../test.outputs/) Validate and upload (via bazel run) [doctor rule] --validate--> payload JSON, bazel_target_metadata.json, context.json - [uploader rule] --dry-run enrichment or upload--> context.json - |-- agentless (DD_API_KEY, DD_SITE) --> citestcycle/citestcov intake + [uploader coordinator] --prepare once--> context/CODEOWNERS/schema/freshness + |-- bounded file workers: enrich -> validate -> split -> send -> clean + |-- agentless (DD_API_KEY, DD_SITE) --> citestcycle/citestcov/telemetry intake |-- EVP proxy (DD_TEST_OPTIMIZATION_AGENT_URL) -> /evp_proxy/... endpoints + `-- aggregate final statistics Optional: Multi-service aggregator @test_optimization_data//:test_optimization_files_ diff --git a/docs/Installation_Reference.md b/docs/Installation_Reference.md index 6ceb8eec..5fc16aa2 100644 --- a/docs/Installation_Reference.md +++ b/docs/Installation_Reference.md @@ -525,6 +525,14 @@ and validates enrichment in that same pass. It processes every available fresh valid payload even if tests or doctor failed while preserving the earliest failure as the job result. +The default uploader is one standard-library Python 3.10+ implementation on +Linux, macOS, and Windows. It prepares shared CODEOWNERS and context data once, +then uses eight independent file workers by default. Each worker handles one +test, coverage, or telemetry source through preparation, retries, and cleanup. +The run ends with human-readable and JSON statistics. Use `--debug` only when +the normal report is not enough; use `--dry-run --validate-enrichment` to verify +the complete outbound preparation without HTTP or deletion. + For manual Go extension wiring, set `module_path` to the Go module path from `go.mod`: @@ -968,7 +976,7 @@ dd_test_optimization_targets( ### 5) Forward environment variables in `.bazelrc` The metadata forwarding entries below apply to every runtime. Config-gated Go -and Python onboarding additionally includes: +and Python onboarding also includes: ```text common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1 diff --git a/docs/Language_Onboarding.md b/docs/Language_Onboarding.md index 214dbeab..ea833ec0 100644 --- a/docs/Language_Onboarding.md +++ b/docs/Language_Onboarding.md @@ -115,6 +115,13 @@ DD_API_KEY="$DD_API_KEY" DD_SITE="$DD_SITE" \ //... ``` +This command starts one cross-platform Python 3.10+ uploader. Its coordinator +prepares shared CODEOWNERS and context data once, then gives each source file to +one of eight workers by default. A worker handles enrichment, validation, +preventive splitting, retries, and cleanup for test, coverage, or telemetry +data. The command without `--upload` uses dry-run enrichment, makes no backend +requests, deletes no payloads, and still prints the final statistics. + ## Automatic managed Go/Python monorepos This path is separate from the static recipes below. It is appropriate when a diff --git a/docs/Maintainers.md b/docs/Maintainers.md index 14c71d55..c9dcf300 100644 --- a/docs/Maintainers.md +++ b/docs/Maintainers.md @@ -275,6 +275,11 @@ Notes: - CI also keeps a dedicated WORKSPACE-compat probe on Bazel `8.4.1` (separate from the `8.5.1` baseline lanes) so legacy `--enable_workspace` behavior is continuously exercised during Bazel 9 migration. +- The generated `rules_go` profile verifier rebuilds each maintained upstream + in two isolated output roots and compares action keys and bytes for + `GoStdlib`, `GoSyntheticTestmainHelpers`, synthetic `GoCompilePkg` + (`~testmain.a`), and `GoLink`. Ordinary `GoStdlib` outputs must not publish an + instrumented cache. Local compact execution-log checks require `zstd`. - Current PR baseline checks: ```sh diff --git a/docs/RFC.md b/docs/RFC.md index d6b83f55..4175d221 100644 --- a/docs/RFC.md +++ b/docs/RFC.md @@ -25,7 +25,7 @@ This section provides a short overview of Bazel concepts that are relevant to th ### Hermetic Sandboxes -Bazel executes builds and tests inside isolated sandboxes. Inputs are declared explicitly, network access is usually disabled, and outputs are cached deterministically. This ensures reproducibility but restricts ad-hoc network activity during test execution. +Bazel executes builds and tests inside isolated sandboxes. Inputs are declared explicitly, network access is usually disabled, and outputs are cached deterministically. That makes builds reproducible but restricts ad-hoc network activity during test execution. ### Cache diff --git a/docs/Uploader_Reference.md b/docs/Uploader_Reference.md index 26abaab8..010c7808 100644 --- a/docs/Uploader_Reference.md +++ b/docs/Uploader_Reference.md @@ -6,7 +6,7 @@ This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. --> -# Uploader Reference +# Uploader reference This page is the full runtime/upload reference for `dd_payload_uploader`. For a quick path, use the upload section in `README.md`. Examples below use a @@ -18,7 +18,8 @@ small repositories. 1. Tests write payloads to `$TEST_UNDECLARED_OUTPUTS_DIR/payloads/tests/*.json` and - `$TEST_UNDECLARED_OUTPUTS_DIR/payloads/coverage/*.json` + `$TEST_UNDECLARED_OUTPUTS_DIR/payloads/coverage/*.json`; tracers may also + write `$TEST_UNDECLARED_OUTPUTS_DIR/payloads/telemetry/*.json` 2. Bazel automatically collects these to `bazel-testlogs///test.outputs/` 3. After tests complete, run the doctor via `bazel run` to validate local @@ -26,8 +27,14 @@ small repositories. 4. Optionally run the uploader in dry-run enrichment mode to validate the exact outbound body without uploading or deleting files 5. Then run the uploader via `bazel run` -6. The uploader discovers local or BEP-staged `test.outputs/` directories, - waits for quiescence, uploads, and deletes uploaded payload files +6. The uploader discovers local or BEP-staged `test.outputs/` directories and + waits for quiescence +7. Its coordinator prepares shared CODEOWNERS, context, schema, freshness, and + telemetry state once, then starts a bounded pool of eight workers by default +8. Each worker owns one source file through enrichment, optional validation, + preventive splitting, upload retries, and cleanup; it can handle test, + coverage, or telemetry data without coordinating with other workers +9. The coordinator prints final statistics after every worker has stopped ## Basic usage @@ -667,13 +674,12 @@ to the requested `--output` path before `--bep-artifact-downloader-timeout-sec` expires. The public rule only defines the downloader contract; it does not ship credentials or a Datadog-internal CAS client. -Artifact staging requires Python at uploader runtime. Bash resolves -`DD_TEST_OPTIMIZATION_PYTHON`, then `PYTHON`, then `python3`, then `python`; -PowerShell uses the same discovery order. Existing local-only uploader flows -remain usable without Python except for support bundle generation and the -pre-existing optional schema and telemetry helpers. Bash BEP freshness parsing -still requires `jq` whenever BEP freshness validation is enabled in the Bash -uploader path. +The default uploader and artifact staging require Python 3.10 or newer. The +small Bash and PowerShell launchers resolve `DD_TEST_OPTIMIZATION_PYTHON`, then +`PYTHON`, then `python3`, then `python`. Only the explicit +`use_python_uploader = False` rollback path can upload local files without +Python. That legacy Bash path still requires `jq` for BEP freshness parsing, +schema validation, enrichment, and oversized-payload splitting. ### Legacy execution-log fallback @@ -746,6 +752,20 @@ payload discovery/quiescence before proceeding. The normalized policy above is the default uploader contract on Linux, macOS, and Windows. +### Final run summary + +Every controlled completion prints six compact lines prefixed with +`[dd-uploader]`: `summary`, `files`, `types`, `split`, `requests`, and `cleanup`. +Together they report the mode and result, configured and peak workers, elapsed +time, file outcomes, per-type outcomes, created and uploaded chunks, request +attempts and retries, and deleted or retained files. The optional JSON report +contains the same aggregate result plus structured diagnostics for automation. + +Dry-run mode reports planned requests but always reports zero attempted +requests and leaves every source file in place. Debug mode adds redacted +per-file and per-request diagnostics; it does not change the summary or the +result. + ### Legacy Bash and PowerShell uploaders - HTTP requests use a 60-second timeout. diff --git a/docs/go_orchestrion_bazel_deep_dive.md b/docs/go_orchestrion_bazel_deep_dive.md index 93ef89bc..a36332bc 100644 --- a/docs/go_orchestrion_bazel_deep_dive.md +++ b/docs/go_orchestrion_bazel_deep_dive.md @@ -6,7 +6,7 @@ This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. --> -# Go + Orchestrion + Bazel Deep Dive +# Go, Orchestrion, and Bazel deep dive > Scope note: consumers choose `third_party/rgo/v0_60_0/base` or a > versioned base tree. Maintainer-only local regression fixtures live in @@ -30,14 +30,8 @@ reverted, or the latest measured bottlenecks, start with [go_orchestrion_maintainer_state.md](./go_orchestrion_maintainer_state.md) and then return here for the lower-level architectural detail. -This is a maintainer document. It intentionally describes the current system, -not the debugging history that produced it. - -### Why This Section Exists - -The Go integration is split across bootstrap, Starlark, vendored `rules_go`, -and patched Orchestrion source. This section states upfront that the document is -meant to unify those pieces into one operational model. +This is a maintainer document about the current system, not its debugging +history. ## Scope @@ -70,13 +64,7 @@ Then use the same relative files under that tree. - [third_party/rgo/v0_60_0/base/go/tools/builders/link.go](../third_party/rgo/v0_60_0/base/go/tools/builders/link.go) - [third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion.go](../third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion.go) -### Why This Section Exists - -The repository is broader than the Go integration path. This section narrows -the document to the files that actually determine how Orchestrion-backed Go -tests work. - -## Mental Model +## Mental model The system is easiest to understand as three concentric layers: @@ -101,13 +89,10 @@ transition that enables the requested Orchestrion mode in the vendored toolchain. When the selected sync export is disabled, the macro instead creates only the caller's public raw `go_test`. -### Why This Section Exists +Orchestrion is not an external post-processing step. Bazel, the vendored +`rules_go`, and Orchestrion form one compile pipeline. -The most common architectural mistake is to think of Orchestrion as an external -post-processing step. This section sets the correct model: Bazel, vendored -`rules_go`, and Orchestrion are one compile pipeline. - -## High-Level Architecture +## High-level architecture ```mermaid flowchart TD @@ -137,18 +122,9 @@ flowchart TD P --> Q[CI Visibility runtime + payloads] ``` -### Why This Section Exists - -The rest of the document is easier to follow if the major boundaries are visible -first. This diagram is the system map the later sections zoom into. - -## Current User-Facing Flow - -### Why This Section Exists +## Current user-facing flow -The public API is intentionally simpler than the internal implementation. This -section explains the supported user-facing contract before moving into toolchain -mechanics. +The public API hides most of the toolchain mechanics described later. ### 1. Module setup @@ -176,8 +152,6 @@ materializes the metadata repo used by `dd_topt_go_test`. The generated metadata repo then provides the per-service and per-module payload labels consumed by `dd_topt_go_test`. -#### Why This Exists - Analysis-time payload selection depends on repository-level metadata. Module setup creates the repos and exports the macro needs later. @@ -207,8 +181,7 @@ architecture: - `orchestrion.tool.go` - `orchestrion.yml` -The important point is that bootstrap does not merely install an external tool. -It aligns: +Bootstrap does more than install a tool. It aligns: - Bazel module wiring - the vendored `rules_go` fork @@ -241,8 +214,6 @@ module with `-mod=readonly`, emits the same canonical `dd_trace_go_versions.json`, and keys the bootstrap cache by that resolved map. Explicit shared and per-module selections remain compatibility escape hatches. -#### Why This Exists - Bootstrap centralizes the one-time mutations needed to make Bazel and Orchestrion agree on the workspace shape. Without that central step, each test target would need to carry fragile setup knowledge. @@ -296,8 +267,6 @@ for arbitrary subpackages. That keeps typical module-root packages simple without implying that every Go test target in every package automatically receives those files. -#### Why This Exists - The macro is the policy boundary. It keeps user BUILD files simple while combining payload selection, runtime wiring, and Orchestrion-enabled compilation in one place. @@ -323,14 +292,12 @@ the same runfiles. This preserves normal Bazel test ergonomics while moving the actual build onto an Orchestrion-enabled configuration. -#### Why This Exists - The transition wrapper lets the raw test build under a different configuration without changing the public target label that users and CI invoke. Callers that filter tests by rule language must account for the mode-dependent rule class: `go` for the disabled raw target and `orch_go` for the enabled wrapper. -## Why the Vendored `rules_go` Fork Exists +## Why the vendored `rules_go` fork exists Orchestrion is fundamentally a `toolexec`-style integration. In a plain Go workflow, the intended shape is conceptually: @@ -357,15 +324,10 @@ For Orchestrion to behave correctly, all of those steps must agree on: - stdlib export family - synthetic test binary link inputs -That is why the repository vendors a `rules_go` fork instead of trying to bolt -Orchestrion on as an external wrapper. +The repository therefore vendors a `rules_go` fork instead of attaching +Orchestrion as an external wrapper. -### Why This Section Exists - -The vendored fork is the key architectural decision. If a maintainer does not -understand why it exists, they will naturally try to simplify the wrong layer. - -## Current Toolchain Topology +## Current toolchain topology ```mermaid flowchart LR @@ -380,19 +342,11 @@ flowchart LR I --> F ``` -### Why This Section Exists - -This is the toolchain-only view of the system. It highlights where state is -carried between compile and link, which is the part most maintainers need to -reason about when something breaks. - -## Vendored `rules_go`: What It Owns +## What the vendored `rules_go` fork owns The vendored fork owns the Orchestrion integration at the action and builder layer. -### Why This Section Exists - The fork is not just a copy of upstream `rules_go`; it is the layer that makes the Bazel pipeline and Orchestrion pipeline behave coherently. @@ -409,8 +363,6 @@ Responsibilities: - pass the synthetic `testmain` manifest and declared helper archive tree through analysis and execution -#### Why This Exists - Starlark is where Bazel decides which files and arguments are real. If the manifest, its helper archive tree, or the Orchestrion tool are not declared here, a later sandbox or remote worker cannot use them. @@ -432,12 +384,10 @@ Responsibilities: - persist synthetic `testmain` helper selections across compile and link - ensure final link reuses the same helper export family chosen during compile -#### Why This Exists - Execution-time coherence cannot be solved in analysis alone. The builder layer is where the environment, importcfg, and archive family are made consistent. -## Patched Orchestrion Source +## Patched Orchestrion source The vendored `rules_go` fork does not use upstream Orchestrion unchanged. @@ -457,8 +407,6 @@ The patches make dependency resolution less recursive and more Bazel-friendly, and they create compatibility symlinks in temporary work directories where needed. -#### Why This Exists - Upstream Orchestrion expects a more conventional Go execution environment than Bazel provides. These patches adapt resolver behavior to Bazel sandboxes and synthetic work directories. @@ -468,8 +416,6 @@ synthetic work directories. The compile proxy is patched so it can correctly understand linker-object outputs and cooperate with the synthetic `testmain` flow. -#### Why This Exists - The synthetic `testmain` path needs metadata to survive across Bazel's separate compile and link stages. The compile proxy has to preserve that information. @@ -479,8 +425,6 @@ The `oncompile`, `oncompile-main`, and `onlink` hooks are patched so dependency lookups run with the Bazel-specific import-path context they actually need, not only with upstream assumptions about a normal Go tool invocation. -#### Why This Exists - Under Bazel, package identity can differ from what upstream Orchestrion would infer from a normal `go` command. These patches keep lookup context aligned with the package Bazel is actually compiling. @@ -491,26 +435,18 @@ Some patches keep Orchestrion from dropping required aspects too early or failing to discover stdlib archives when Bazel's importcfg layout differs from what upstream Orchestrion normally sees. -#### Why This Exists - If Orchestrion rejects a package too early or cannot locate stdlib archives, later steps cannot recover. These patches keep the instrumentation path open. The result is still "Orchestrion from source", but it is a Bazel-adapted build of Orchestrion, not a stock upstream binary. -### Why This Section Exists - -The Orchestrion binary is part of the supported integration surface. This -section explains why the current design depends on a Bazel-adapted Orchestrion -build, not just on the vendored `rules_go` changes. +The Orchestrion binary is part of the supported integration surface. The design +depends on this Bazel-adapted build as well as the vendored `rules_go` changes. -## The Compile Path +## Compile path -### Why This Section Exists - -Compile is where the system first moves from "configured for instrumentation" -to "producing instrumented artifacts". This section explains that boundary. +Compile turns the configured instrumentation into build artifacts. ### Package compile @@ -538,8 +474,6 @@ The key Orchestrion-specific compile responsibilities are: - importcfg rewriting - synthetic `testmain` helper capture -#### Why This Exists - Package compile is the first place where wrong module context, wrong importcfg, or wrong stdlib selection can poison everything downstream. @@ -567,13 +501,11 @@ outside the execroot are copied into the declared helper tree, and the sidecar is rewritten to point there. This makes the compile-to-link contract portable across Bazel sandboxes and remote workers. -#### Why This Exists - `testing` instrumentation enters through synthetic `testmain`. The sidecar exists so final link can reuse the exact helper package family chosen during compile instead of reconstructing a different one. -## Importcfg Management +## Importcfg management The importcfg layer is where the Bazel/Orchestrion integration becomes most concrete. @@ -583,8 +515,6 @@ Implementation: This file owns several distinct jobs: -### Why This Section Exists - Importcfg is the explicit statement of what the toolchain can import. This section explains how the integration shapes that package universe. @@ -597,8 +527,6 @@ The builder can source stdlib packagefiles from: - seeded stdlib cache exports - persisted stdlib export manifests -#### Why This Exists - Orchestrion can weave stdlib packages, so compile and link need to resolve the woven stdlib archive family rather than silently falling back to the default unwoven one. @@ -608,8 +536,6 @@ unwoven one. The Datadog helper packages used for CI Visibility and tracing must exist as real `packagefile` entries in the importcfg seen by compile and link. -#### Why This Exists - Injected CI Visibility hooks become ordinary package dependencies after weaving. If their exports are missing from importcfg, the final binary cannot preserve the instrumentation path. @@ -619,8 +545,6 @@ the instrumentation path. The builder rewrites existing `packagefile` directives and appends missing ones so the toolchain sees a coherent package universe. -#### Why This Exists - Bazel's default importcfg contents do not fully describe the instrumented build graph. Rewriting is how the builders present the package set Orchestrion actually needs. @@ -631,12 +555,10 @@ The final link must not invent a different Datadog helper export family than the one synthetic `testmain` compile already used. The importcfg layer is where that reuse is made explicit. -#### Why This Exists - Compile and link are separate actions. Helper-root reuse is the mechanism that keeps them inside the same Datadog package universe. -## The Link Path +## Link path The link entry point is: @@ -653,19 +575,14 @@ packagefiles produced earlier, then keeps the link action free of unused Orchestrion proxy and pin-file inputs. Generic mode can still use the broader Orchestrion link behavior. -### Why This Section Exists - -Link is the last place where separate compile outputs can either converge into a -coherent binary or drift apart. This section explains how the current design -forces convergence. +Link is the last point where separate compile outputs can converge into a +coherent binary or drift apart. ### Normal link When linking a normal main package, link can run through Orchestrion directly and can append the broader Datadog helper closure it needs for the final binary. -#### Why This Exists - Normal link establishes the baseline behavior: link may complete the helper closure, but it must still stay inside one consistent export family. @@ -701,25 +618,16 @@ That is the core consistency rule of the current implementation: Without that, compile may instrument `testing`, but the final linked test binary may not preserve those hooks coherently. -#### Why This Exists +Bazel's synthetic test wrapper and Datadog's `testing` instrumentation meet on +this link path. The sidecar manifest makes that join deterministic. -This is the highest-risk link path because Bazel's synthetic test wrapper and -Datadog's `testing` instrumentation meet here. The sidecar manifest exists to -make that join deterministic. - -## The Orchestrion Runtime Environment Inside Builders +## Orchestrion runtime environment inside builders Implementation: - [orchestrion.go](../third_party/rgo/v0_60_0/base/go/tools/builders/orchestrion.go) -This file centralizes the environment preparation Orchestrion needs under Bazel. - -Important responsibilities: - -### Why This Section Exists - -The builders do not run in a normal developer shell. This section explains the -supporting environment that must exist before Orchestrion can do useful work. +This file prepares the environment Orchestrion needs inside Bazel actions, +which do not run in a normal developer shell. ### Go cache and module cache provisioning @@ -732,8 +640,6 @@ The builder ensures writable: That matters because Orchestrion shells out to Go tooling while resolving injectors and package files. -#### Why This Exists - Without writable caches and module storage, Orchestrion's internal `go` calls fail inside Bazel sandboxes even when the outer action is otherwise correct. @@ -750,8 +656,6 @@ path. This still lets Orchestrion reuse fetched modules instead of redownloading them for each sandboxed step. -#### Why This Exists - The shared cache is not just a performance optimization. It keeps bootstrap-time pinning and sandboxed builder steps working against the same downloaded module set. @@ -764,8 +668,6 @@ subprocesses assume. The builder normalizes that by: - resolving absolute SDK/GOROOT paths - creating compatibility symlinks when `GOROOT/src` is missing -#### Why This Exists - Orchestrion and the Go toolchain expect a usable GOROOT layout. Bazel's SDK presentation can differ enough that the builders need to repair that view. @@ -774,8 +676,6 @@ presentation can differ enough that the builders need to repair that view. Orchestrion expects a real Go module context. The builder prepares a synthetic module environment when the Bazel workdir does not already match that shape. -#### Why This Exists - Orchestrion resolves injectors and pinned module files through Go module semantics. Synthetic module preparation gives it a workspace shape compatible with those expectations. @@ -788,12 +688,29 @@ set. In `test_optimization` mode the synthetic module and helper closure are reduced to the standard Go `testing` Test Optimization path and exclude profiler and Datadog contrib HTTP/slog helper roots. -#### Why This Exists - Some required woven dependencies are first touched inside sandboxed steps. Warming them reduces failures caused by lazy first access in those contexts. -## End-to-End Flow for an Enabled Go Test +### Deterministic build outputs + +The Go build cache itself is writable process state, so it cannot be exposed as +a Bazel-declared output. `GoStdlib` builds in a private scratch `GOCACHE` and +publishes only sorted, manifested woven archives to its declared cache. An +ordinary Go action leaves that declared Orchestrion cache empty; a non-optimized +target must never consume the instrumented stdlib. + +Later synthetic helper, compile, and link actions seed their private caches from +the read-only published archives. Before hashing or copying results, the builder +normalizes the temporary Go work directory, Bazel execroot references, CGO +random seed, synthetic source trim paths, and helper archive build IDs. Actual +relative CGO inputs still resolve from the physical execroot. + +The generated-profile verifier repeats the build in isolated output roots and +compares action keys and bytes for `GoStdlib`, +`GoSyntheticTestmainHelpers`, synthetic `GoCompilePkg` (`~testmain.a`), and +`GoLink`. This is the same action chain that downstream Reprise checks observe. + +## End-to-end flow for an enabled Go test The disabled path stops at the macro: it creates the public raw `go_test` and does not enter the wrapper, vendored Orchestrion, or payload path below. @@ -820,13 +737,11 @@ sequenceDiagram Bin->>Bin: emit tracer / CI Visibility runtime ``` -### Why This Section Exists - The system crosses many boundaries. This sequence compresses them into one path so a maintainer can reason from the declared test target to the observed runtime behavior. -## Runtime Result +## Runtime result At runtime, a correctly wired binary shows evidence that the compile-time path worked: @@ -844,28 +759,19 @@ exports during compile/link consistency checks. Those names are useful build-time indicators, but they are not themselves user-visible runtime signals. -### Why This Section Exists - The global goal is not just "a successful build". It is a final test binary whose runtime behavior proves that CI Visibility instrumentation survived the full Bazel compile and link pipeline. -## Invariants Maintainers Should Preserve +## Invariants maintainers should preserve If you change this system, keep these invariants intact. -### Why This Section Exists - -This section marks the boundaries of safe refactoring. It separates incidental -implementation details from the properties that actually make the design work. - ### 1. The public API stays Bazel-native Users should keep writing `dd_topt_go_test`, not a custom shell rule or an alternate test runner. -#### Why This Exists - The integration is meant to feel like normal Bazel usage to consumers. If the public entry point changes shape, the maintenance burden moves into every consumer workspace. @@ -881,8 +787,6 @@ The bootstrap tool is the place that mutates: Do not push that complexity onto each test target. -#### Why This Exists - Workspace mutation has to happen in one place or the setup becomes fragile and non-idempotent. Bootstrap is that one place. @@ -891,8 +795,6 @@ non-idempotent. Bootstrap is that one place. The sidecar manifest, declared helper archive tree, and helper-root reuse are structural, not optional. -#### Why This Exists - This is the specific invariant that keeps `testing` instrumentation alive in the final binary. Losing it produces apparently successful builds with incomplete runtime behavior. @@ -902,8 +804,6 @@ runtime behavior. If compile or link needs a packagefile and it is not in importcfg, that is an importcfg problem first. -#### Why This Exists - Putting package-universe fixes in multiple places makes the pipeline impossible to reason about. Importcfg must remain the primary source of truth. @@ -912,8 +812,6 @@ to reason about. Importcfg must remain the primary source of truth. Builder-side synthetic module and cache preparation are not cosmetic. They are required because Orchestrion shells out to Go tooling internally. -#### Why This Exists - If Orchestrion cannot see a usable Go module environment, it stops behaving like the tool this design expects and starts failing in ways that look unrelated to the actual instrumentation logic. @@ -923,19 +821,14 @@ the actual instrumentation logic. The source patches in `extensions.bzl` are part of the supported design. Treat them as first-class integration code, not as temporary local hacks. -#### Why This Exists - The vendored `rules_go` changes alone are not enough. The built Orchestrion binary carries Bazel-specific behavior that the rest of the design relies on. -## Practical Debugging Map +## Debugging map When something breaks, this is the fastest place to look. -### Why This Section Exists - -The implementation surface is large. This section shortens the feedback loop by -mapping likely failure classes to the files that actually control them. +The entries below map failure classes to the files that control them. ### Bootstrap and workspace shape @@ -947,10 +840,7 @@ Look here if: - `rules_go_orchestrion_tool` is missing - `orchestrion.tool.go` was not pinned -#### Why This Exists - -Many failures that look like compile bugs are really workspace-shape bugs. This -section makes that distinction explicit. +Many apparent compile bugs originate in the workspace shape. ### Macro and transition behavior @@ -963,11 +853,8 @@ Look here if: - runfiles or executable naming are wrong - payload data selection is wrong -#### Why This Exists - -This is the place to debug analysis-time mistakes before looking at builder -internals. If the wrong target graph is produced, the toolchain never gets a -chance to do the right thing. +Check analysis-time mistakes before builder internals. A wrong target graph +never reaches the intended toolchain path. ### Orchestrion source patching @@ -979,10 +866,8 @@ Look here if: - dependency resolution differs from local `go` behavior - patched upstream assumptions drift -#### Why This Exists - -Some failures originate inside the generated Orchestrion binary, not inside -Starlark or the builders. This section points directly at that integration seam. +Some failures originate inside the generated Orchestrion binary rather than +Starlark or the builders. ### Compile/link consistency @@ -996,8 +881,6 @@ Look here if: - Datadog helper packages are unresolved - synthetic test binaries lose instrumentation at final link -#### Why This Exists - Most correctness bugs eventually reduce to compile/link inconsistency. This is the fastest entry point for the class of bugs that most directly threaten the global goal. @@ -1016,9 +899,3 @@ The current architecture is: That combination is what makes the current Go CI Visibility path work under Bazel while still behaving like a normal `go_test` target from the user's point of view. - -### Why This Section Exists - -The final summary compresses the design down to the few moving pieces that are -actually carrying the integration. If a future simplification preserves these -properties, it is likely safe. diff --git a/docs/go_orchestrion_maintainer_state.md b/docs/go_orchestrion_maintainer_state.md index 05f2c199..7585e016 100644 --- a/docs/go_orchestrion_maintainer_state.md +++ b/docs/go_orchestrion_maintainer_state.md @@ -6,14 +6,14 @@ This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. --> -# Go Orchestrion Maintainer State +# Go Orchestrion maintainer state ## Purpose -This document is the maintainer-facing summary of the current Go + Orchestrion -integration as it exists in this repository after the recent performance work. +This page summarizes the current Go and Orchestrion integration for +maintainers. -Use it when you need one place that answers: +Use it to find: - how the integration is wired end to end - why the fork exists and what it changes @@ -31,10 +31,10 @@ This is intentionally different from the other Go Orchestrion docs: - [rules_go_orchestrion_probe_measurements.md](./rules_go_orchestrion_probe_measurements.md) is the running measurement log -This document is the current state summary that ties those three views -together. +The supported upstream lines are `rules_go` v0.60.0, v0.61.1, v0.62.0, and +v0.63.0. They share the same Orchestrion behavior and determinism contract. -## Current Architecture In One View +## Current architecture ```mermaid flowchart TD @@ -58,7 +58,7 @@ flowchart TD M --> E ``` -The important model is: +In this design: - Bazel is still the build and test system - `rules_go` still owns the Go build pipeline @@ -77,7 +77,7 @@ The important model is: managed case, the consumer's central wrapper selects the exact `topt_data_by_target` entry before calling `dd_topt_go_test`. -## Mode Contract +## Mode contract `dd_topt_go_test` exposes two Orchestrion modes: @@ -93,7 +93,7 @@ instrumentation. Metadata emitted by the Go macro should include `bazel.go.test_binary_linker_optimization` so reviewers can tell whether the test-binary linker flag optimization was active. -## Why The Fork Exists +## Why the fork exists We are not using upstream `rules_go` and upstream Orchestrion as-is. @@ -113,7 +113,7 @@ Orchestrion and upstream `rules_go` do not provide together out of the box: In short: the fork is not ornamental. It is the compatibility layer that keeps Orchestrion coherent inside Bazel. -## Why The Orchestrion Tool Is Still Built When Enabled +## Why the Orchestrion tool is built when enabled On an enabled true cold start, Bazel still builds the Orchestrion binary because: @@ -145,9 +145,7 @@ The recent work changed one important part of this story: That reduced cold bootstrap cost substantially without changing the target-side tracer version that the instrumented binary actually loads. -## Why The Target Still Uses The Pinned Tracer Version - -This is the subtle point that mattered most in the recent work. +## Why the target uses the pinned tracer version There are two different module contexts: @@ -169,7 +167,35 @@ The kept design now does this: That is why runtime validation must show the target workspace's configured Datadog tracer version, even after tool-side `go.mod` rewriting was removed. -## Kept Performance Changes +## Deterministic action outputs + +Bazel can reuse the instrumented build only when identical inputs produce +identical action keys and bytes. The generated-profile verifier therefore runs +two isolated builds and compares these action families: + +- `GoStdlib`; +- `GoSyntheticTestmainHelpers`; +- the synthetic `GoCompilePkg` action that produces `~testmain.a`; +- `GoLink`. + +An ordinary `GoStdlib` action must leave its declared Orchestrion cache empty. +Only a Test Optimization action may publish the woven stdlib cache, and that +cache must contain deterministic, manifested archive data. This prevents an +ordinary Go build from consuming an instrumented standard library. + +The instrumented path normalizes values that vary between otherwise equivalent +Bazel executions: the temporary Go work directory, the Bazel execroot in +compiler arguments, the CGO random seed, synthetic source trim paths, and build +IDs in copied helper archives. Relative CGO inputs still resolve against the +real execroot; normalization changes identity, not input lookup. + +`tools/dev/verify_rules_go_profiles.py` implements this check with compact Bazel +execution logs and isolated output roots. Local runs need `zstd` to read those +logs. CI splits the supported versions across x86_64 and ARM64 shards. A changed +action key with identical bytes is reported as avoidable rebuilding; changed +output bytes are a reproducibility failure. + +## Kept performance changes These changes were kept because they improved performance without changing runtime behavior. @@ -234,7 +260,7 @@ Effect: - stdlib action time improved - no runtime regression was observed in the validated kept version -## Experiments That Failed +## Experiments that failed These failed experiments are worth documenting because they are tempting to try again. @@ -327,7 +353,7 @@ Lesson: - the current stdlib root set is coupled to the real Bazel test flow more than the surface comment suggests -## Current Best-Known Baseline +## Current best-known baseline The current baseline should be read in two modes: @@ -351,10 +377,8 @@ Current cold numbers: - download/extract: `32.066s` - tool `go build`: `113.334s` -The main conclusion is simple: - -- the largest remaining cold-start cost is still building the patched - Orchestrion tool from scratch +The largest remaining cold-start cost is building the patched Orchestrion tool +from scratch. ### Warm bootstrap reuse @@ -371,12 +395,9 @@ Current isolated warm-bootstrap observation: - cache identity source: the declared Bazel SDK, allowing restore before SDK materialization -The main conclusion is: - -- once the bootstrap artifact cache hits, the tool bootstrap is no longer the - main cost -- Bazel startup, repository mapping, and analysis account for most of the - remaining fresh-output-root command time +Once the bootstrap artifact cache hits, the tool bootstrap is no longer the +main cost. Bazel startup, repository mapping, and analysis account for most of +the remaining fresh-output-root command time. ### Runtime correctness validation @@ -396,7 +417,7 @@ What must be verified: This validation is part of the baseline, not optional extra checking. -## What Still Looks Expensive +## Remaining expensive work ### 1. Cold Orchestrion tool build @@ -422,7 +443,7 @@ The remaining stdlib cost now appears to be mostly real work: The easy cache tricks have mostly been exhausted or disproved. -## Recommended Next Steps +## Possible next steps ### If the goal is the biggest remaining performance win @@ -446,7 +467,7 @@ The most plausible local direction is: - do not assume cache-layout tricks are safe - keep validating runtime behavior, not just build success -## Validation Checklist For Future Changes +## Validation checklist for future changes Future changes in this area should not be considered done until they pass all of these checks: @@ -460,11 +481,14 @@ of these checks: 5. runtime validation with: - `DD_TRACE_DEBUG=1` - `DD_CIVISIBILITY_ENABLED=1` -6. confirmation that: +6. two isolated generated-profile runs compare `GoStdlib`, synthetic helpers, + synthetic `~testmain.a`, and `GoLink` action keys and bytes +7. confirmation that: - tracer startup logs are present - the pinned tracer version is the one loaded at runtime - a payload file is written under `test.outputs/payloads/tests` - metadata includes the expected `bazel.go.orchestrion.mode` - metadata includes the expected `bazel.go.test_binary_linker_optimization` -That final runtime check is the one that caught the bad stdlib snapshot idea. +The runtime check caught the rejected stdlib snapshot approach; the isolated +action comparison prevents that class of cache drift from reaching a consumer. diff --git a/docs/python_parallel_uploader_migration_plan.md b/docs/python_parallel_uploader_migration_plan.md index d6b3d646..a263e40e 100644 --- a/docs/python_parallel_uploader_migration_plan.md +++ b/docs/python_parallel_uploader_migration_plan.md @@ -6,7 +6,7 @@ This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. --> -# Python Parallel Uploader: Implementation Guide and Tracker +# Python parallel uploader implementation tracker ## Status @@ -30,6 +30,14 @@ The implementation currently provides: This is an implementation milestone, not the end of the migration. Open checkboxes are release gates, not deferred product ideas. +The merged rollout in +[rule PR #214](https://github.com/DataDog/rules_test_optimization/pull/214) +passed the Linux, macOS, and Windows rule matrix. The consumer matrix passed in +the merged +[`_tests` PR #108](https://github.com/ddoghq/rules_test_optimization_tests/pull/108). +The remaining unchecked items below are the proof still required before +deleting the legacy implementations. + ## Goals Replace two functional uploader implementations with one portable runtime while @@ -121,7 +129,7 @@ There is no synchronization between file workers beyond queue ownership and coordinator collection. A worker never waits for another file's enrichment, split, upload, or cleanup. -## Ownership Boundaries +## Ownership boundaries ### Launcher @@ -188,7 +196,7 @@ dequeued. Report writing is atomic. A report-write failure is printed as a warning and does not reinterpret completed uploads. -## Pre-worker Flow +## Pre-worker flow The following order is intentional: @@ -207,7 +215,7 @@ The following order is intentional: The lock remains held through worker completion, staging cleanup, and final reporting so a second uploader cannot race source deletion or staging cleanup. -## Configuration Precedence +## Configuration precedence Generated rule values are defaults. Environment overrides them, and explicit CLI values override environment where a CLI option exists. @@ -247,7 +255,7 @@ The parser produces immutable compiled rules and preserves: Workers share the matcher but keep a file-local source-to-match cache. They do not mutate matcher rules or share per-file match state. -## Worker Pool +## Worker pool - The queue is bounded to apply producer backpressure. - Threads are homogeneous and non-daemon. @@ -263,7 +271,7 @@ already owned finish their current complete pipeline, all threads join, owned temporary resources are cleaned, completed results are reported, and the process exits `130`. -## Test Payload Pipeline +## Test payload pipeline For each test JSON file, one worker performs: @@ -304,7 +312,7 @@ HTTP `413` is a terminal `payload_limit_contract_mismatch` for test chunks. It is not retried and does not trigger another split, because the preventive split should already have made the request valid. -## Coverage Pipeline +## Coverage pipeline Coverage accepts JSON and msgpack. The worker: @@ -317,7 +325,7 @@ Coverage accepts JSON and msgpack. The worker: Coverage is not split. A `413` is terminal and identifies unsupported oversized coverage rather than invoking test-split behavior. -## Telemetry Pipeline +## Telemetry pipeline Telemetry correlation is planned before workers start, but each source remains owned by one worker. The source worker: @@ -334,7 +342,7 @@ Unchanged primary telemetry preserves its original bytes. Changed and synthetic bodies use deterministic compact serialization. Telemetry is not split, and `413` is terminal. -## HTTP and Retry Contract +## HTTP and retry contract Each worker-local standard-library transport owns: @@ -366,7 +374,7 @@ Do not retry: seconds. Otherwise the configured fixed delay is used. Dry-run performs no sleep and creates no network connection. -## Dry-run and Debug +## Dry-run and debug ### Dry-run @@ -391,7 +399,7 @@ timing. Debug must never print API keys, authorization headers, full payload bodies, or unbounded response bodies. -## Cleanup and Outcome Rules +## Cleanup and outcome rules - Delete a source only after all requests derived from it succeed. - Keep sources on preparation, validation, split, transport, or HTTP failure; @@ -411,7 +419,7 @@ Severity precedence is: 3. one or more failed files (`1`); 4. success (`0`). -## Final Statistics +## Final statistics Every controlled completion prints a stable summary containing: @@ -427,7 +435,7 @@ Human output and JSON derive from the same aggregate. The schema-v1 report keeps legacy fields and adds explicit concurrency, split, request, warning, and failure sections. -## Package Map +## Package map | Module | Responsibility | |---|---| @@ -452,7 +460,7 @@ directories, strict JSON, credentials, discovery, expected targets, and resource loading. Avoid adding another layer unless it removes more complexity than it introduces. -## Implementation Tracker +## Implementation tracker ### Runtime foundations @@ -509,7 +517,7 @@ than it introduces. - [ ] Remove the temporary switch and legacy Bash/PowerShell runtimes. - [ ] Remove obsolete jq/curl/gzip uploader prerequisites from docs. -## Test Matrix +## Test matrix Required automated coverage: @@ -548,7 +556,7 @@ python3 tools/dev/lint_uploader_templates.py Also run every companion-module command from `CONTRIBUTING.md` and the relevant flow in `../rules_test_optimization_tests` with local overrides. -## Acceptance Criteria +## Acceptance criteria - [x] One functional Python uploader implementation exists. - [x] Platform launchers contain resolution only. @@ -569,12 +577,12 @@ flow in `../rules_test_optimization_tests` with local overrides. - [ ] Public Bash/PowerShell behavior is characterized and matched. - [ ] Standard-library HTTP behavior passes all supported OS/proxy/TLS lanes. - [ ] `workers=1` passes the complete cross-platform parity matrix. -- [ ] Linux, macOS, and Windows CI pass with the Python target. -- [ ] The sibling consumer fixture passes. +- [x] Linux, macOS, and Windows CI pass with the Python target. +- [x] The sibling consumer fixture passes. - [x] Python becomes the default. - [ ] Legacy functional scripts are removed. -## Performance Validation +## Performance validation Measure with identical payload fixtures and backend behavior: @@ -598,7 +606,7 @@ the default is `8`; consumers with split-heavy workloads or backend throttling can override it to `4`. Continue validating memory, temporary storage, and real-backend retry behavior on supported CI hosts. -## Rollout and Rollback +## Rollout and rollback 1. Compare dry-run outputs and loopback request captures against legacy. 2. Validate the default in representative consumers and supported CI platforms. diff --git a/docs/rules_go_orchestrion_probe_measurements.md b/docs/rules_go_orchestrion_probe_measurements.md index db070cc8..48f6b85b 100644 --- a/docs/rules_go_orchestrion_probe_measurements.md +++ b/docs/rules_go_orchestrion_probe_measurements.md @@ -651,8 +651,8 @@ So the cold miss is not “compile a few helper packages”. It becomes “compi large transitive non-cgo helper ecosystem that must remain source-compatible in the synthetic module”. -That matches the measured run: the expensive packages were not only the obvious -Datadog roots. They also included deeper transitive packages such as: +That matches the measured run: expensive work included the Datadog roots and +deeper transitive packages such as: - `github.com/DataDog/dd-trace-go/v2/ddtrace/mocktracer` - `github.com/DataDog/datadog-agent/pkg/trace/stats` diff --git a/docs/rules_go_orchestrion_support_selection.md b/docs/rules_go_orchestrion_support_selection.md index 0f830bf4..5ecd6b97 100644 --- a/docs/rules_go_orchestrion_support_selection.md +++ b/docs/rules_go_orchestrion_support_selection.md @@ -6,14 +6,14 @@ This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. --> -# rules_go Orchestrion Support Selection +# rules_go Orchestrion support selection This repository exposes one supported public `rules_go` tree variant, `base`. Consumers that already own a private `rules_go` patch stack can also consume a generated sparse patch profile as an input to a local rebase or merge inside their own repository. -## Published Variant +## Published variant - `base`: upstream `rules_go` plus the generic Orchestrion integration and correctness fixes maintained by this repository. @@ -29,7 +29,7 @@ support line. Omitting `rules_go_upstream` preserves the repository default. The registry currently supports `v0_60_0`, `v0_61_1`, `v0_62_0`, and `v0_63_0`. -## Selection Rule +## Selection rule - Use `base` for normal WORKSPACE and Bzlmod consumers. - Use a generated consumer patch profile only when the consuming repository @@ -43,7 +43,7 @@ Public consumer patch profiles live under python3 tools/dev/verify_rules_go_profiles.py --public-denylist tools/dev/private_leak_public_denylist.txt ``` -## Maintainer Workflow +## Maintainer workflow Maintainers track each supported upstream version with both: @@ -65,6 +65,12 @@ python3 tools/dev/materialize_rules_go_fork.py check --all python3 tools/dev/verify_rules_go_profiles.py --public-denylist tools/dev/private_leak_public_denylist.txt ``` +The verifier runs two isolated ordinary and Test Optimization builds for every +registered upstream. It compares `GoStdlib`, synthetic helper, synthetic +`~testmain.a`, and `GoLink` action keys and bytes, and rejects an instrumented +stdlib cache in ordinary mode. Local compact execution-log inspection requires +`zstd`. + Run the smoke lane before publishing: ```bash diff --git a/docs/rules_go_stdlib_cache_determinism_plan.md b/docs/rules_go_stdlib_cache_determinism_plan.md index 9a502221..241c5134 100644 --- a/docs/rules_go_stdlib_cache_determinism_plan.md +++ b/docs/rules_go_stdlib_cache_determinism_plan.md @@ -6,13 +6,15 @@ This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. --> -# Deterministic rules_go Standard-Library Cache Plan +# Deterministic rules_go build-output tracker ## Status -Draft implementation plan for the Orchestrion v1.12.0 update. This document -describes the intended implementation and validation sequence; the cache fix is -not implemented merely by adding this plan. +Implementation is in progress on PR #251. The stdlib ownership fix and the +isolated action-output verifier are implemented locally; publication and +downstream adoption remain open until the PR is merged and consumers are +repinned. This document now tracks that work instead of describing an +unimplemented proposal. The work starts in `rules_test_optimization`, is validated through `rules_test_optimization_tests`, and is adopted by `dd-source` only after a @@ -22,6 +24,7 @@ maintained `rules_go` profile: - `v0_60_0` - `v0_61_1` - `v0_62_0` +- `v0_63_0` It must preserve both supported Go execution modes: @@ -49,7 +52,7 @@ Instead, Test Optimization must explicitly publish only the deterministic archive data required by downstream actions. Ordinary Go builds must still create the declared TreeArtifact, but leave it empty. -## Problem Statement +## Problem statement The Go build cache is not a reproducible Bazel action output. In addition to archive data entries (`*-d`), the Go tool writes action index entries (`*-a`) @@ -77,7 +80,7 @@ correctly identifies the first writer and moves the primary stdlib build to a private cache. Porting that patch literally is insufficient for this repository because the two Orchestrion-specific writers would remain. -## Terminology and Data Flow +## Terminology and data flow This plan uses these names consistently: @@ -120,7 +123,7 @@ Later compile/link action +-- seedWovenStdlibCache copies/hard-links declared archives into GOCACHE ``` -## Required Invariants +## Required invariants The implementation is complete only if all of the following remain true. @@ -165,7 +168,7 @@ The implementation is complete only if all of the following remain true. ### In scope -- The base `rules_go` trees for all three registered upstreams. +- The base `rules_go` trees for all four registered upstreams. - Builder unit tests covering cache ownership, publication, manifest contents, path safety, and command environments. - Generated public patch profiles, metadata, and changed-files reports. @@ -188,7 +191,7 @@ The implementation is complete only if all of the following remain true. - Refactoring unrelated Orchestrion cache, jobserver, resolver, uploader, or payload code. -## Design Decisions +## Design decisions ### D-1: Fix the base trees, then regenerate @@ -244,7 +247,7 @@ and SHA-256 content digest. Filesystem mtimes are not part of the TreeArtifact's semantic content and must not be used to create a false failure. Symlinks, if any unexpectedly appear, must be reported rather than silently followed. -## Implementation Plan +## Implementation plan ### S-1: Introduce explicit cache setup in `stdlib.go` @@ -374,7 +377,7 @@ must enforce that the declared cache is never passed as its writable root. The existing builder test target includes `stdlib_test.go` in `@rules_go//go/tools/builders:orchestrion_test`. Add the tests to the base trees -and keep the test sources identical across all three versions where the +and keep the test sources identical across all four versions where the production sources are identical. #### Cache setup and ownership tests @@ -445,12 +448,12 @@ Keep existing tests for: ### S-6: Apply the implementation to every maintained upstream At the start of implementation, compare the relevant source files across the -three bases. They are currently byte-identical for `stdlib.go`, +four bases. They are currently byte-identical for `stdlib.go`, `stdlib_test.go`, `importcfg.go`, and `env_orchestrion.go`, so the preferred workflow is: 1. implement and review the change in `v0_60_0`; -2. copy the exact logical change to `v0_61_1` and `v0_62_0`; +2. copy the exact logical change to `v0_61_1`, `v0_62_0`, and `v0_63_0`; 3. compare the resulting files or focused diffs across versions; 4. retain a version-specific difference only when the upstream base requires it and document the reason in the diff. @@ -464,10 +467,17 @@ Extend the existing functional smoke in `tools/dev/verify_rules_go_profiles.py`; do not create a parallel profile verifier. -For each registered upstream, the verifier already materializes a pristine -upstream tree, applies the generated public patch, creates a temporary WORKSPACE -consumer, executes a Test Optimization test, and inspects `aquery`. Add a -determinism phase to that same temporary consumer. +For each registered upstream, the verifier materializes a pristine upstream +tree, applies the generated public patch, creates a temporary WORKSPACE +consumer, executes a Test Optimization test, and inspects structured Bazel +output. Its determinism phase runs twice with isolated output roots and reads +the compact execution logs. Local runs require `zstd`. + +The comparison covers the full Reprise-sensitive chain: `GoStdlib`, +`GoSyntheticTestmainHelpers`, the `GoCompilePkg` action that produces +`~testmain.a`, and `GoLink`. The verifier compares both action keys and output +bytes. Different bytes fail the check; equal bytes with a different key +identify an unnecessary rebuild. #### Plain mode @@ -497,6 +507,8 @@ determinism phase to that same temporary consumer. actually enabled Test Optimization. 8. Preserve the real test execution proving that the woven stdlib can be consumed. +9. Exercise CGO so the woven stdlib, compiler random seed, relative input + paths, synthetic source trim paths, and copied helper build IDs are covered. The implementation may use two isolated Bazel output roots or an explicit action-cache invalidation mechanism. It must not compare one execution with a @@ -539,7 +551,7 @@ If a generator modifies a base source file, stop and understand why before continuing; the base source implementation must remain the reviewed source of truth. -## Validation Matrix +## Validation matrix The minimum acceptance matrix is: @@ -548,6 +560,7 @@ The minimum acceptance matrix is: | v0.60.0 | required | required | two isolated runs | required | | v0.61.1 | required | required | two isolated runs | required | | v0.62.0 | required | required | two isolated runs | required | +| v0.63.0 | required | required | two isolated runs | required | For every row: @@ -594,10 +607,10 @@ consumer repository. Enable the existing local overrides for: - `datadog-rules-test-optimization-go`; - the selected `rules_go` base tree. -Exercise all three upstreams using the existing fixture support: +Exercise all four upstreams using the existing fixture support: ```bash -for rules_go_upstream in v0_60_0 v0_61_1 v0_62_0; do +for rules_go_upstream in v0_60_0 v0_61_1 v0_62_0 v0_63_0; do RULES_GO_UPSTREAM="$rules_go_upstream" RTO_LOCAL_ARCHIVE=1 \ ./fixtures/bzlmod-go/runtests done @@ -619,7 +632,7 @@ discrepancy, and clear the stable Orchestrion cache when the test specifically needs a cold Orchestrion execution. Do not clear caches between the first and second runs whose purpose is to prove cache reuse. -## Cross-Repository Rollout +## Cross-repository rollout ### R-1: Publish `rules_test_optimization` @@ -669,7 +682,7 @@ For the Test Optimization run, require test and telemetry payload counts and successful doctor/enrichment/upload stages. A Reprise success from a plain Go target alone does not prove the Orchestrion path. -## Failure Handling and Rollback +## Failure handling and rollback ### During Rule implementation @@ -693,15 +706,15 @@ target alone does not prove the Orchestrion path. than adding another overlay before determining whether the failure is in Rule generation, consumer integration, or dd-source composition. -## Risks and Mitigations +## Risks and mitigations ### Silent loss of instrumentation **Risk:** compile and link succeed while downstream actions read plain stdlib archives. -**Mitigation:** require real payload emission plus doctor/enrichment validation, -not only builder unit tests or `aquery` output. +**Mitigation:** require real payload emission plus doctor/enrichment validation; +builder unit tests and `aquery` output alone are insufficient. ### A hidden declared-cache writer remains @@ -752,11 +765,11 @@ changes already present in generated profiles. **Mitigation:** modify base sources first, keep existing user changes intact, then run the canonical generators once and inspect the combined generated diff. -## Completion Criteria +## Completion criteria The work is complete only when: -- all three base profiles implement the same cache ownership contract; +- all four base profiles implement the same cache ownership contract; - no live Go or Orchestrion subprocess receives the declared cache as `GOCACHE`; - plain Go declared cache outputs are empty and reproducible; @@ -766,8 +779,8 @@ The work is complete only when: inventories; - builder unit tests, profile materialization, profile verification, release archive checks, and the full rules suite pass; -- local and published consumer fixtures pass for `v0.60.0`, `v0.61.1`, and - `v0.62.0`; +- local and published consumer fixtures pass for `v0.60.0`, `v0.61.1`, + `v0.62.0`, and `v0.63.0`; - real Test Optimization tests emit valid test and telemetry payloads; - an unchanged second consumer run demonstrates Bazel cache reuse; - PR #108 is pinned to the final rules SHA and passes its CI matrix; @@ -776,22 +789,22 @@ The work is complete only when: - no temporary overrides, scratch artifacts, or unrelated generated changes remain in any of the three repositories. -## Execution Checklist - -- [ ] Confirm the active Rule branch and preserve all existing review changes. -- [ ] Recompare relevant builder sources across all three bases. -- [ ] Implement private scratch and separate declared-cache setup. -- [ ] Remove the Orchestrion `GOCACHE=env.stdlibCache` override. -- [ ] Implement safe deterministic archive publication. -- [ ] Audit every declared-cache read and write call site. -- [ ] Add cache ownership and command-environment tests. -- [ ] Add deterministic publisher and path-safety tests. -- [ ] Run focused builder tests. -- [ ] Propagate and compare the implementation across all upstreams. -- [ ] Extend generated-profile plain and Test Optimization determinism smoke. -- [ ] Regenerate profiles, metadata, and changed-files reports. -- [ ] Run materialization, verifier, release, and full-suite checks. -- [ ] Validate local consumer fixtures across all versions and both modes. +## Execution checklist + +- [x] Confirm the active Rule branch and preserve all existing review changes. +- [x] Recompare relevant builder sources across all four bases. +- [x] Implement private scratch and separate declared-cache setup. +- [x] Remove the Orchestrion `GOCACHE=env.stdlibCache` override. +- [x] Implement safe deterministic archive publication. +- [x] Audit every declared-cache read and write call site. +- [x] Add cache ownership and command-environment tests. +- [x] Add deterministic publisher and path-safety tests. +- [x] Run focused builder tests. +- [x] Propagate and compare the implementation across all upstreams. +- [x] Extend generated-profile verification to the four Reprise action families. +- [x] Regenerate profiles, metadata, and changed-files reports. +- [x] Run materialization, verifier, release, and full-suite checks. +- [x] Validate local consumer fixtures across all versions and both modes. - [ ] Publish the Rule commit when explicitly authorized. - [ ] Repin and validate PR #108 from the published commit. - [ ] Repin dd-source and regenerate its composed rules_go patch. @@ -799,4 +812,3 @@ The work is complete only when: - [ ] Validate dd-source plain, Test Optimization, Reprise, load, and cache hydration flows. - [ ] Inspect final diffs and confirm all temporary local wiring is removed. - diff --git a/docs/rules_go_variant_maintenance_guide.md b/docs/rules_go_variant_maintenance_guide.md index 838d3486..c93605f0 100644 --- a/docs/rules_go_variant_maintenance_guide.md +++ b/docs/rules_go_variant_maintenance_guide.md @@ -6,7 +6,7 @@ This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. --> -# rules_go Orchestrion Support Folder Guide +# rules_go Orchestrion support folder guide This guide describes the maintained folder layout for the vendored Orchestrion-enabled `rules_go` support lines and public consumer patch profiles. @@ -17,7 +17,7 @@ Orchestrion-enabled `rules_go` support lines and public consumer patch profiles. The public base tree for one supported upstream. For example, the current default upstream uses `third_party/rgo/v0_60_0/base`, and the additional -support lines use `third_party/rgo/v0_61_1/base` and +support lines use `third_party/rgo/v0_61_1/base`, `third_party/rgo/v0_62_0/base`, and `third_party/rgo/v0_63_0/base`. Each tree contains clean upstream `rules_go` plus the generic Orchestrion support maintained by this repository. Bugs in our integration are fixed in the @@ -64,9 +64,12 @@ Use `tools/dev/materialize_rules_go_fork.py check --all` to verify that patch series recreate the checked-in trees. Use `tools/dev/verify_rules_go_profiles.py --public-denylist tools/dev/private_leak_public_denylist.txt` to verify that public consumer patch profiles round-trip against clean upstream -`rules_go` without leaking private-only strings. +`rules_go` without leaking private-only strings. The verifier also rebuilds +ordinary and Test Optimization consumers in isolated output roots and compares +action keys and bytes for `GoStdlib`, synthetic helpers, synthetic +`~testmain.a`, and `GoLink`. Install `zstd` before running this check locally. -## Adding A rules_go Upstream Release +## Adding a rules_go upstream release Use this sequence when adding support for a new upstream `rules_go` release. Replace the example `v0_62_0` and `v0.62.0` values with the requested release. @@ -242,7 +245,8 @@ Replace the example `v0_62_0` and `v0.62.0` values with the requested release. 9. Run the migration validation checklist before calling the support line done. Build success alone is not enough; at least one runtime lane must prove CI Visibility startup, JSON payload generation, doctor success, and upload or - dry-run upload behavior. + dry-run upload behavior. Profile verification must also prove deterministic + outputs across the four action families in ordinary and instrumented modes. ## Consumer Contract diff --git a/tools/agent-skills/go-test-optimization-onboarding/SKILL.md b/tools/agent-skills/go-test-optimization-onboarding/SKILL.md index 067e75e4..ea9b2232 100644 --- a/tools/agent-skills/go-test-optimization-onboarding/SKILL.md +++ b/tools/agent-skills/go-test-optimization-onboarding/SKILL.md @@ -12,14 +12,14 @@ This product includes software developed at Datadog --> -# Datadog Go Test Optimization Onboarding +# Datadog Go Test Optimization onboarding Use this skill when you need to instrument a Bazel Go repository with Datadog Test Optimization. This skill is intentionally project-neutral: it is stored in the repository as a Codex-compatible skill, but any agent can read it as a normal implementation guide. -## Non-Negotiable Contract +## Non-negotiable contract Keep the RFC contract intact: @@ -27,6 +27,18 @@ Keep the RFC contract intact: - Bazel collects those files under `bazel-testlogs//test.outputs/`. - The doctor validates local files after `bazel test`. - The uploader runs after the doctor with `bazel run`. +- Use the default Python 3.10+ uploader unless a temporary rollback explicitly + requires `use_python_uploader = False`. Its coordinator prepares shared + CODEOWNERS, contexts, schemas, freshness, and telemetry once, then starts up + to eight independent file workers by default. Each worker owns enrichment, + validation, preventive splitting, retries, and cleanup for one test, + coverage, or telemetry source file. +- Run one uploader process. Use `--dry-run --validate-enrichment` to prepare + requests without HTTP or deletion, `--debug` only for verbose redacted + diagnostics, and review the final file/type/split/request/cleanup totals. +- Test bodies are split before HTTP when they exceed `4_718_592` bytes. HTTP + `413` is terminal and must not trigger a retry or adaptive split; coverage + and telemetry are not split. - Do not add payload proxies or msgpack-only handoff paths. - Do not pass `DD_GIT_*` through `--test_env`; use `--repo_env` for sync metadata. - Do not pass uploader endpoints or credentials into the test sandbox. @@ -55,7 +67,7 @@ Keep the RFC contract intact: `reports/doctor-report.json`, optional uploader reports, and `command/flags.json` in that order. -## First Actions +## First actions 1. Read the consumer repository's Bazel shape before editing: - Does it use `MODULE.bazel`, `WORKSPACE`, or both? @@ -82,14 +94,14 @@ Keep the RFC contract intact: sync only when that command can expand exact labels and derive service/runtime contexts. Do not create a checked-in target/service map. -## Implementation Paths +## Implementation paths - **WORKSPACE consumers:** follow [workspace-onboarding.md](references/workspace-onboarding.md). - **Bzlmod consumers:** follow [bzlmod-onboarding.md](references/bzlmod-onboarding.md). - **Validation:** follow [validation-checklist.md](references/validation-checklist.md). - **Debugging:** follow [troubleshooting.md](references/troubleshooting.md). -## Universal Shape +## Universal shape Every successful Go onboarding should end with these pieces: @@ -108,6 +120,10 @@ Every successful Go onboarding should end with these pieces: - Go tests use one central repo-local wrapper that delegates to `dd_topt_go_test`. The named config, not a different BUILD macro, selects enabled behavior. +- A `.topt` clone preserves the source test's execution policy. Keep + `no-remote-exec` on the actual `TestRunner`, but do not let it constrain + deterministic stdlib, synthetic helper, compile, or link actions; those must + remain cacheable and remote-capable. - The workspace has exactly one `dd_test_optimization_doctor` target and one `dd_upload_payloads` target. Root is acceptable for small repositories; use a lightweight package such as `//tools/test_optimization` in monorepos. @@ -136,7 +152,9 @@ Every successful Go onboarding should end with these pieces: Prefer the wrapper support bundle for full CI escalation; use the doctor-only support bundle for the simplest initial customer request. Keep individual reports for local inspection and manual fallback flows. -- Real upload processes available fresh valid payloads after doctor and dry-run attempts, while preserving any earlier failure. +- A real upload processes every available fresh valid payload after validation + attempts. The wrapper preserves the earliest test, doctor, or uploader exit + code; uploader errors never replace an earlier test result. For automatic managed Go/Python monorepos, the universal shape has these additional constraints: @@ -192,7 +210,7 @@ Go wrapper, treat it as a consumer-specific integration: the doctor and one enrichment-validating uploader pass; use dry-run only when real upload is disabled. -## Branch And PR Hygiene +## Branch and PR hygiene Before making changes in a real repository, confirm whether to use the current branch or create a new branch from the latest default branch. Keep onboarding @@ -207,7 +225,7 @@ changes reviewable: - If an issue requires changing this rule repository, add matching fixture coverage in `rules_test_optimization_tests` before declaring it solved. -## Stop Conditions +## Stop conditions Stop and escalate instead of guessing when: diff --git a/tools/agent-skills/go-test-optimization-onboarding/references/validation-checklist.md b/tools/agent-skills/go-test-optimization-onboarding/references/validation-checklist.md index 58bb192e..47ff6895 100644 --- a/tools/agent-skills/go-test-optimization-onboarding/references/validation-checklist.md +++ b/tools/agent-skills/go-test-optimization-onboarding/references/validation-checklist.md @@ -6,7 +6,7 @@ This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. --> -# Validation Checklist +# Validation checklist Use this checklist before calling a Go onboarding complete. @@ -63,7 +63,7 @@ module graph requires the documented escape hatch. ## Managed Manifest Checks -For the automatic managed path, additionally prove: +For the automatic managed path, also prove: - no committed target/service mapping, generated example registry, Gazelle extension, or ownership gate was added; @@ -199,7 +199,7 @@ ls -la "$(bazel info output_base)/external/test_optimization_data_/ cat "$(bazel info output_base)/external/test_optimization_data_/export.bzl" ``` -## Test, Doctor, Dry-Run, Upload +## Test, doctor, dry-run, and upload For the simplest customer troubleshooting request after tests have run, use `bazel run --config=test-optimization //:dd_test_optimization_doctor -- --support-bundle=` with any @@ -256,7 +256,30 @@ done Do not run the real upload unless credentials are intentionally available and the user or CI environment expects data to be sent. -## Payload Inspection +For a safe end-to-end check, add `--dry-run` to the uploader invocation and +keep `--validate-enrichment`. Confirm that its final statistics show the +expected configured and peak workers, per-type file counts, prepared chunks, +zero attempted requests, and zero deleted files. The default runtime uses eight +Python workers; do not simulate concurrency by launching multiple uploader +processes. Use `--debug` only when the normal report lacks enough redacted +detail to diagnose a failure. + +If the source test carries `no-remote-exec`, inspect the transitioned target's +actions: + +```bash +bazel aquery --config=test-optimization \ + 'mnemonic("TestRunner", deps(//path/to:pilot_test.topt))' \ + --output=jsonproto +bazel aquery --config=test-optimization \ + 'mnemonic("GoStdlib|GoSyntheticTestmainHelpers|GoCompilePkg|GoLink", deps(//path/to:pilot_test.topt))' \ + --output=jsonproto +``` + +The `TestRunner` must retain `no-remote-exec`. Deterministic build actions must +not inherit it, so they remain cacheable and remote-capable. + +## Payload inspection After tests, inspect `bazel-testlogs`: @@ -309,7 +332,7 @@ on the default allowlist: - Do not list `.build_test` or build-only controls in `expected_targets` because they do not run instrumented test code. -## Remote Execution +## Remote execution If tests use remote execution or remote cache, make sure the test config uses: @@ -353,7 +376,7 @@ Artifact mode choices: | BEP references bytestream/CAS/custom-auth `test.outputs` or `outputs.zip` artifacts | Add `--artifact-source=bep --remote-artifacts=download --bep-artifact-downloader=/path/to/downloader`; use `--remote-artifacts=required` for strict all-or-nothing validation | | Mixed migration where local outputs may be stale but BEP can stage fresh carriers | Use `--artifact-source=auto --remote-artifacts=download` so staged BEP outputs win for matching output keys | -## Final Consumer Checks +## Final consumer checks Before opening or finishing a consumer PR: diff --git a/tools/agent-skills/java-test-optimization-onboarding/SKILL.md b/tools/agent-skills/java-test-optimization-onboarding/SKILL.md index 8d9612ab..e42f7f5b 100644 --- a/tools/agent-skills/java-test-optimization-onboarding/SKILL.md +++ b/tools/agent-skills/java-test-optimization-onboarding/SKILL.md @@ -11,7 +11,7 @@ This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. --> -# Datadog Java Test Optimization Onboarding +# Datadog Java Test Optimization onboarding Use this skill when you need to instrument a Bazel Java repository with Datadog Test Optimization. The skill is intentionally project-neutral: it is stored in @@ -24,7 +24,7 @@ multi-service contracts in this skill. Do not enroll Java targets in `test_optimization_manifest_sync` or describe the managed Go/Python command as a Java onboarding path. -## Non-Negotiable Contract +## Non-negotiable contract Keep the RFC contract intact: @@ -32,6 +32,18 @@ Keep the RFC contract intact: - Bazel collects those files under `bazel-testlogs//test.outputs/`. - The doctor validates local files after `bazel test`. - The uploader runs after the doctor with `bazel run`. +- Use the default Python 3.10+ uploader unless a temporary rollback explicitly + requires `use_python_uploader = False`. Its coordinator prepares shared + CODEOWNERS, contexts, schemas, freshness, and telemetry once, then starts up + to eight independent file workers by default. Each worker owns enrichment, + validation, preventive splitting, retries, and cleanup for one test, + coverage, or telemetry source file. +- Run one uploader process. Use `--dry-run --validate-enrichment` to prepare + requests without HTTP or deletion, `--debug` only for verbose redacted + diagnostics, and review the final file/type/split/request/cleanup totals. +- Test bodies are split before HTTP when they exceed `4_718_592` bytes. HTTP + `413` is terminal and must not trigger a retry or adaptive split; coverage + and telemetry are not split. - Do not add payload proxies or upload-from-test-sandbox paths. - Do not manually set manifest or payload-in-files environment variables in consumer test rules; `dd_topt_java_test` owns that wiring. @@ -63,7 +75,7 @@ Keep the RFC contract intact: `reports/doctor-report.json`, optional uploader reports, and `command/flags.json` in that order. -## First Actions +## First actions 1. Read the consumer repository's Bazel shape before editing: - Does it use `MODULE.bazel`, `WORKSPACE`, or both? @@ -96,7 +108,7 @@ Keep the RFC contract intact: [validation-checklist.md](references/validation-checklist.md) and [troubleshooting.md](references/troubleshooting.md). -## Universal Shape +## Universal shape Every successful Java onboarding should end with these pieces: @@ -132,13 +144,15 @@ Every successful Java onboarding should end with these pieces: - `FETCH_SALT` is used only for a separate, explicit `bazel sync --only= --repo_env=FETCH_SALT="$(date +%s)"` refresh, never as part of normal test, doctor, or uploader commands. -- Real upload processes available fresh valid payloads after doctor and dry-run attempts, while preserving any earlier failure. +- A real upload processes every available fresh valid payload after validation + attempts. The wrapper preserves the earliest test, doctor, or uploader exit + code; uploader errors never replace an earlier test result. Use the consumer's existing Bazel entrypoint in all commands. Do not switch a repository from `bzl` or `bazelw` to raw `bazel` just because examples use the generic binary name. -## Branch And PR Hygiene +## Branch and PR hygiene Before making changes in a real repository, confirm whether to use the current branch or create a new branch from the latest default branch. Keep onboarding @@ -151,7 +165,7 @@ changes reviewable: - If an issue requires changing this rule repository, add matching fixture coverage in `rules_test_optimization_tests` before declaring it solved. -## Stop Conditions +## Stop conditions Stop and escalate instead of guessing when: diff --git a/tools/agent-skills/java-test-optimization-onboarding/references/validation-checklist.md b/tools/agent-skills/java-test-optimization-onboarding/references/validation-checklist.md index 5fc23ab7..0091ed96 100644 --- a/tools/agent-skills/java-test-optimization-onboarding/references/validation-checklist.md +++ b/tools/agent-skills/java-test-optimization-onboarding/references/validation-checklist.md @@ -6,7 +6,7 @@ This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. --> -# Java Validation Checklist +# Java validation checklist Replace `bazel` in examples with the consumer repository's real Bazel entrypoint such as `bzl` or `./bazelw`. @@ -86,7 +86,7 @@ bazel sync --enable_workspace --config=test-optimization \ --repo_env=FETCH_SALT="$(date +%s)" ``` -## Test, Doctor, Dry-Run, Upload +## Test, doctor, dry-run, and upload For the simplest customer troubleshooting request after tests have run, use `bazel run //:dd_test_optimization_doctor -- --support-bundle=` with any @@ -145,7 +145,15 @@ done Do not run the real upload unless credentials are intentionally available and the user or CI environment expects data to be sent. -## Expected Outputs +For a safe end-to-end check, add `--dry-run` to the uploader invocation and +keep `--validate-enrichment`. Confirm that its final statistics show the +expected configured and peak workers, per-type file counts, prepared chunks, +zero attempted requests, and zero deleted files. The default runtime uses eight +Python workers; do not simulate concurrency by launching multiple uploader +processes. Use `--debug` only when the normal report lacks enough redacted +detail to diagnose a failure. + +## Expected outputs After tests: @@ -164,7 +172,7 @@ After tests: Do not list build-only or analysis-only targets in doctor `expected_targets`; they do not run instrumented test code. -## Remote Execution +## Remote execution If tests use remote execution or remote cache, the test config must include: diff --git a/tools/agent-skills/python-test-optimization-onboarding/SKILL.md b/tools/agent-skills/python-test-optimization-onboarding/SKILL.md index 849e0f73..4f42aeaf 100644 --- a/tools/agent-skills/python-test-optimization-onboarding/SKILL.md +++ b/tools/agent-skills/python-test-optimization-onboarding/SKILL.md @@ -12,14 +12,14 @@ This product includes software developed at Datadog --> -# Datadog Python Test Optimization Onboarding +# Datadog Python Test Optimization onboarding Use this skill when you need to instrument a Bazel Python repository with Datadog Test Optimization. The skill is intentionally project-neutral: it is stored in this repository as a Codex-compatible skill, but any agent can read it as a normal implementation guide. -## Non-Negotiable Contract +## Non-negotiable contract Keep the RFC contract intact: @@ -27,6 +27,18 @@ Keep the RFC contract intact: - Bazel collects those files under `bazel-testlogs//test.outputs/`. - The doctor validates local files after `bazel test`. - The uploader runs after the doctor with `bazel run`. +- Use the default Python 3.10+ uploader unless a temporary rollback explicitly + requires `use_python_uploader = False`. Its coordinator prepares shared + CODEOWNERS, contexts, schemas, freshness, and telemetry once, then starts up + to eight independent file workers by default. Each worker owns enrichment, + validation, preventive splitting, retries, and cleanup for one test, + coverage, or telemetry source file. +- Run one uploader process. Use `--dry-run --validate-enrichment` to prepare + requests without HTTP or deletion, `--debug` only for verbose redacted + diagnostics, and review the final file/type/split/request/cleanup totals. +- Test bodies are split before HTTP when they exceed `4_718_592` bytes. HTTP + `413` is terminal and must not trigger a retry or adaptive split; coverage + and telemetry are not split. - Do not add payload proxies or upload-from-test-sandbox paths. - Do not pass `DD_GIT_*` through `--test_env`; use `--repo_env` for sync metadata. @@ -56,7 +68,7 @@ Keep the RFC contract intact: `reports/doctor-report.json`, optional uploader reports, and `command/flags.json` in that order. -## First Actions +## First actions 1. Read the consumer repository's Bazel shape before editing: - Does it use `MODULE.bazel`, `WORKSPACE`, or both? @@ -91,7 +103,7 @@ Keep the RFC contract intact: [validation-checklist.md](references/validation-checklist.md) and [troubleshooting.md](references/troubleshooting.md). -## Universal Shape +## Universal shape Every successful Python onboarding should end with these pieces: @@ -128,7 +140,9 @@ Every successful Python onboarding should end with these pieces: - `FETCH_SALT` is used only for a separate, explicit `bazel sync --config=test-optimization --only= --repo_env=FETCH_SALT="$(date +%s)"` refresh, never as part of normal test, doctor, or uploader commands. -- Real upload processes available fresh valid payloads after doctor and dry-run attempts, while preserving any earlier failure. +- A real upload processes every available fresh valid payload after validation + attempts. The wrapper preserves the earliest test, doctor, or uploader exit + code; uploader errors never replace an earlier test result. For automatic managed Go/Python monorepos: @@ -146,7 +160,7 @@ Use the consumer's existing Bazel entrypoint in all commands. Do not switch a repository from `bzl` or `bazelw` to raw `bazel` just because examples use the generic binary name. -## Branch And PR Hygiene +## Branch and PR hygiene Before making changes in a real repository, confirm whether to use the current branch or create a new branch from the latest default branch. Keep onboarding @@ -161,7 +175,7 @@ changes reviewable: - If an issue requires changing this rule repository, add matching fixture coverage in `rules_test_optimization_tests` before declaring it solved. -## Stop Conditions +## Stop conditions Stop and escalate instead of guessing when: diff --git a/tools/agent-skills/python-test-optimization-onboarding/references/validation-checklist.md b/tools/agent-skills/python-test-optimization-onboarding/references/validation-checklist.md index a89f55c0..5e57e833 100644 --- a/tools/agent-skills/python-test-optimization-onboarding/references/validation-checklist.md +++ b/tools/agent-skills/python-test-optimization-onboarding/references/validation-checklist.md @@ -6,7 +6,7 @@ This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. --> -# Python Validation Checklist +# Python validation checklist Replace `bazel` in examples with the consumer repository's real Bazel entrypoint such as `bzl` or `./bazelw`. @@ -46,7 +46,7 @@ For WORKSPACE consumers, also confirm: ## Managed Manifest Checks -For automatic managed onboarding, additionally prove: +For automatic managed onboarding, also prove: - there is no committed target/service mapping or Gazelle/ownership machinery; - target patterns are expanded to exact canonical labels before sync; @@ -158,7 +158,7 @@ bazel sync --enable_workspace --config=test-optimization \ --repo_env=FETCH_SALT="$(date +%s)" ``` -## Test, Doctor, Dry-Run, Upload +## Test, doctor, dry-run, and upload For the simplest customer troubleshooting request after tests have run, use `bazel run --config=test-optimization //:dd_test_optimization_doctor -- --support-bundle=` with any @@ -217,7 +217,15 @@ done Do not run the real upload unless credentials are intentionally available and the user or CI environment expects data to be sent. -## Expected Outputs +For a safe end-to-end check, add `--dry-run` to the uploader invocation and +keep `--validate-enrichment`. Confirm that its final statistics show the +expected configured and peak workers, per-type file counts, prepared chunks, +zero attempted requests, and zero deleted files. The default runtime uses eight +Python workers; do not simulate concurrency by launching multiple uploader +processes. Use `--debug` only when the normal report lacks enough redacted +detail to diagnose a failure. + +## Expected outputs After tests: @@ -232,7 +240,7 @@ After tests: Do not list build-only or analysis-only targets in doctor `expected_targets`; they do not run instrumented test code. -## Remote Execution +## Remote execution If tests use remote execution or remote cache, the test config must include: diff --git a/tools/agent-skills/rules-go-orchestrion-upstream-migration/SKILL.md b/tools/agent-skills/rules-go-orchestrion-upstream-migration/SKILL.md index 039d2513..1f0da789 100644 --- a/tools/agent-skills/rules-go-orchestrion-upstream-migration/SKILL.md +++ b/tools/agent-skills/rules-go-orchestrion-upstream-migration/SKILL.md @@ -12,7 +12,7 @@ This product includes software developed at Datadog --> -# Datadog rules_go Orchestrion Upstream Migration +# Datadog rules_go Orchestrion upstream migration Use this skill when you need to move the vendored Orchestrion-enabled `rules_go` fork in this repository from the currently pinned upstream @@ -28,7 +28,7 @@ Bazel-managed Go SDK bootstrap, and the existing public Orchestrion aliases; it must not add target discovery, service naming, or manifest policy to `rules_go`. -## Non-Negotiable Contract +## Non-negotiable contract Preserve the maintained fork contract: @@ -47,8 +47,13 @@ Preserve the maintained fork contract: - Do not call the migration complete until materialization checks, profile verification, changed-file regeneration, and relevant smoke or integration validation have run. +- Preserve deterministic action outputs for `GoStdlib`, + `GoSyntheticTestmainHelpers`, synthetic `GoCompilePkg` (`~testmain.a`), and + `GoLink`. An ordinary `GoStdlib` action must not publish or consume the + instrumented stdlib cache. A Test Optimization build must publish stable + woven archives across two isolated output roots, including CGO mode. -## First Actions +## First actions 1. Confirm the requested target upstream: - exact upstream `rules_go` tag or commit @@ -71,14 +76,16 @@ Preserve the maintained fork contract: - `python3 tools/dev/generate_rules_go_fork_maps.py --check` - `python3 tools/dev/materialize_rules_go_fork.py check --all` - `python3 tools/dev/verify_rules_go_profiles.py --public-denylist tools/dev/private_leak_public_denylist.txt` +5. Install `zstd` before running the local generated-profile verifier; it reads + Bazel's compact execution logs. -## Implementation Path +## Implementation path - **Migration workflow:** follow [migration-workflow.md](references/migration-workflow.md). - **Validation:** follow [validation-checklist.md](references/validation-checklist.md). - **Troubleshooting:** follow [troubleshooting.md](references/troubleshooting.md). -## Sensitive Surfaces +## Sensitive surfaces Expect conflicts or semantic drift around these paths first: @@ -115,7 +122,7 @@ Pay special attention to: - Bzlmod and WORKSPACE extension entrypoints - Bazel transitions that must preserve Orchestrion settings -## Done Criteria +## Done criteria A migration is done only when all of these are true: @@ -127,6 +134,10 @@ A migration is done only when all of these are true: `tools/dev/diff_rules_go_fork.py`. - `python3 tools/dev/materialize_rules_go_fork.py check --all` passes. - `python3 tools/dev/verify_rules_go_profiles.py --public-denylist tools/dev/private_leak_public_denylist.txt` passes. +- Profile verification executes two isolated builds for both ordinary and Test + Optimization modes and compares the four required action families. The + ordinary declared stdlib cache is empty; the instrumented CGO stdlib contains + Test Optimization markers and produces identical bytes. - The selected validation lanes in [validation-checklist.md](references/validation-checklist.md) pass, or every skipped lane has a concrete reason. @@ -135,7 +146,7 @@ A migration is done only when all of these are true: - The final report names the target upstream, changed-path counts, validation results, and any remaining external blockers. -## Test Optimization Alias Contract +## Test Optimization alias contract When validating a consumer that uses Test Optimization, preserve the stable alias contract from the vendored base tree: @@ -149,7 +160,7 @@ targets enable Orchestrion through their transition. Omitting the config must leave metadata bootstrap disabled for public Go extension repositories and for low-level repositories explicitly configured with `enabled_by_env = True`. -## Stop Conditions +## Stop conditions Stop and escalate instead of guessing when: diff --git a/tools/agent-skills/rules-go-orchestrion-upstream-migration/references/migration-workflow.md b/tools/agent-skills/rules-go-orchestrion-upstream-migration/references/migration-workflow.md index 3d0555f2..e8e645ce 100644 --- a/tools/agent-skills/rules-go-orchestrion-upstream-migration/references/migration-workflow.md +++ b/tools/agent-skills/rules-go-orchestrion-upstream-migration/references/migration-workflow.md @@ -6,7 +6,7 @@ This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. --> -# Migration Workflow +# Migration workflow Use this workflow to port the vendored Orchestrion-enabled `rules_go` fork to a new upstream `rules_go` tag or commit. @@ -19,7 +19,7 @@ The public fork has one supported variant: `base`. Do not reintroduce `complete`, `third_party/rules_go_orchestrion/versions/...`, or consumer-specific public patch bundles while adding a new upstream. -## Mechanical Release Recipe +## Mechanical release recipe Use this recipe as the default algorithm for a new upstream release. Replace `v0_62_0`, `0.62.0`, and `v0.62.0` with the requested release. @@ -149,7 +149,7 @@ truth. The source of truth remains the registry entry, the `base.series` patch stack, the materialized `third_party/rgo/$NEW_UPSTREAM/base` tree, and the profile JSON. -## 1. Capture The Current Baseline +## 1. Capture the current baseline Start from a clean understanding of the current fork: @@ -169,7 +169,7 @@ Review the current patch series and checked-in reports before editing: - the current upstream's `*.METADATA.json` - the current upstream's `*.CHANGED_FILES.md` -## 2. Materialize The New Upstream Tree +## 2. Materialize the new upstream tree Add the new upstream to `third_party/rules_go_orchestrion/registry.json` using the shape from the mechanical recipe. Create the metadata file from the same @@ -208,7 +208,7 @@ After the base tree is coherent, regenerate `base.series` and `materialize_rules_go_fork.py check --upstream --variant base` recreates it exactly. -## 4. Verify Consumer Patch Profiles +## 4. Verify consumer patch profiles Run the public profile generator against the migrated base tree. The generated patch must apply to clean upstream `rules_go`, preserve included file modes and @@ -218,7 +218,7 @@ labels in public profiles. Generate profile artifacts into `/tmp` or another throwaway directory unless a release process explicitly asks for checked-in derived artifacts. -## 5. Regenerate Reports +## 5. Regenerate reports Regenerate the target upstream delta report: @@ -233,7 +233,7 @@ python3 tools/dev/check_release_archive_contents.py Read the regenerated reports. The changed-path counts may change, but every new or removed path should be explainable by the upstream migration. -## 6. Validate Behavior +## 6. Validate behavior Run the required lanes from [validation-checklist.md](validation-checklist.md). For Orchestrion migrations, build success alone is not enough. Runtime @@ -243,7 +243,7 @@ payload files. If validation fails, use [troubleshooting.md](troubleshooting.md). Do not hide failures by weakening tests or deleting variant differences from metadata. -## 7. Final Report +## 7. Final report The final report for a migration PR must include: diff --git a/tools/agent-skills/rules-go-orchestrion-upstream-migration/references/troubleshooting.md b/tools/agent-skills/rules-go-orchestrion-upstream-migration/references/troubleshooting.md index 26c0913a..8642771a 100644 --- a/tools/agent-skills/rules-go-orchestrion-upstream-migration/references/troubleshooting.md +++ b/tools/agent-skills/rules-go-orchestrion-upstream-migration/references/troubleshooting.md @@ -22,7 +22,7 @@ Symptoms: Checks: - Compare old upstream, old `base`, new upstream, and the partial migrated tree. -- Identify the behavior being carried, not only the line-level patch. +- Identify the behavior being carried rather than copying the line-level patch. - Read the new upstream implementation before deciding where to insert the Orchestrion behavior. - Re-run the changed-files diff after each coherent chunk. @@ -90,7 +90,8 @@ Checks: - Verify Orchestrion `toolexec` is present in compile and stdlib actions. - Verify the offline module proxy is available as an action input. - Verify `dd_trace_go_versions.json` is present and read by builder actions. -- Run the consumer integration harnesses, not only vendored `rules_go` tests. +- Run both the consumer integration harnesses and the vendored `rules_go` + tests. Build success alone is not proof for stdlib or runtime weaving changes. diff --git a/tools/agent-skills/rules-go-orchestrion-upstream-migration/references/validation-checklist.md b/tools/agent-skills/rules-go-orchestrion-upstream-migration/references/validation-checklist.md index a3684480..8ace924b 100644 --- a/tools/agent-skills/rules-go-orchestrion-upstream-migration/references/validation-checklist.md +++ b/tools/agent-skills/rules-go-orchestrion-upstream-migration/references/validation-checklist.md @@ -6,7 +6,7 @@ This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. --> -# Validation Checklist +# Validation checklist Use this checklist before calling a `rules_go` upstream migration complete. When the Rule also supports manifest-managed Go targets, verify the selected @@ -14,7 +14,7 @@ upstream still passes no-host-Go disabled/enabled bootstrap coverage. Manifest schema, service derivation, and target discovery remain outside this migration and must not be modified here. -## Metadata And Inventory +## Metadata and inventory Run these checks after editing any `rules_go` support line: @@ -38,8 +38,17 @@ Expected: - diff commands report the same counts as the regenerated reports - generated reports name the new upstream tag or commit - no generated report was edited manually +- the verifier observes and compares `GoStdlib`, + `GoSyntheticTestmainHelpers`, synthetic `GoCompilePkg` (`~testmain.a`), and + `GoLink` across two isolated output roots +- ordinary `GoStdlib` leaves its declared Orchestrion cache empty +- Test Optimization CGO stdlib output is woven and byte-identical between runs -## Fast Variant Smoke +The local compact execution-log check requires `zstd`. A changed action key +with unchanged bytes is still useful evidence of avoidable rebuilding; changed +bytes are a reproducibility failure. + +## Fast variant smoke Run the published base variant: @@ -53,7 +62,7 @@ If the migration changes slow or platform-sensitive areas, also run: RULES_GO_UPSTREAM= RULES_GO_VARIANT=base tools/dev/run_rules_go_variant_extended.sh ``` -## Go Consumer Integration +## Go consumer integration Run both WORKSPACE and Bzlmod integration harnesses when the migration touches Orchestrion wiring, module proxy handling, stdlib behavior, transitions, or @@ -79,7 +88,7 @@ Expected: - payload files are written under `bazel-testlogs` - payload metadata does not show unexpected fallback states -## Repository-Level Regression +## Repository-level regression Run root or focused repository tests when changed files overlap repository helpers or companion integration: @@ -97,7 +106,7 @@ workspace wiring: ./bazelw build //examples/... ``` -## Cross-Repository Fixture +## Cross-repository fixture If the migration is intended to validate consumer-style behavior before a PR is called done, run the sibling fixture repository with local overrides: @@ -118,7 +127,7 @@ cd ../rules_test_optimization_tests Restore the fixture repository to its pinned overrides before committing or pushing changes there. -## Runtime Correctness Checks +## Runtime correctness checks For stdlib, synthetic `testmain`, module proxy, or tool-version changes, do not stop at build success. Inspect runtime behavior: @@ -136,7 +145,7 @@ Expected: - No `.msgpack` or `.msgpack.gz` payloads are emitted. - No unexpected `full_bundle_no_match` state appears. -## Completion Gate +## Completion gate Before final response or PR handoff, record: