feat(speculation): fold predictor into Scorer

submitqueue/extension/speculation/generator/bestfirst/bestfirst.go

@@ -121,7 +121,7 @@ func (g *bestFirst) score(ctx context.Context, ids []string, batchByID map[strin
 			probabilityByID[id] = defaultProbability
 			continue
 		}
-		probability, err := g.scorer.Score(ctx, batch)
+		probability, err := g.scorer.Score(ctx, batch, entity.SpeculationPathSet{})
 		if err != nil {
 			// A scorer that failed because the caller went away has not found
 			// an unpriceable dependency — it has found a dead ctx, which ends

submitqueue/extension/speculation/generator/bestfirst/bestfirst_test.go

@@ -38,7 +38,7 @@ type stubScorer struct {
 	scores map[string]float64
 }
 
-func (s stubScorer) Score(_ context.Context, b entity.Batch) (float64, error) {
+func (s stubScorer) Score(_ context.Context, b entity.Batch, _ entity.SpeculationPathSet) (float64, error) {
 	if v, ok := s.scores[b.ID]; ok {
 		return v, nil
 	}
@@ -136,7 +136,7 @@ func newCountingScorer(scores map[string]float64) *countingScorer {
 	return &countingScorer{scores: scores, calls: map[string]int{}}
 }
 
-func (c *countingScorer) Score(_ context.Context, b entity.Batch) (float64, error) {
+func (c *countingScorer) Score(_ context.Context, b entity.Batch, _ entity.SpeculationPathSet) (float64, error) {
 	c.calls[b.ID]++
 	c.total++
 	if v, ok := c.scores[b.ID]; ok {
@@ -148,14 +148,14 @@ func (c *countingScorer) Score(_ context.Context, b entity.Batch) (float64, erro
 // errScorer always fails, to exercise error propagation from scoring.
 type errScorer struct{}
 
-func (errScorer) Score(context.Context, entity.Batch) (float64, error) {
+func (errScorer) Score(context.Context, entity.Batch, entity.SpeculationPathSet) (float64, error) {
 	return 0, assert.AnError
 }
 
 // constScorer scores every batch identically, regardless of ID.
 type constScorer struct{ v float64 }
 
-func (c constScorer) Score(context.Context, entity.Batch) (float64, error) { return c.v, nil }
+func (c constScorer) Score(context.Context, entity.Batch, entity.SpeculationPathSet) (float64, error) { return c.v, nil }
 
 // wideHead builds one Speculating head over n unresolved dependencies, each at a
 // distinct score so no two combinations tie.
@@ -839,7 +839,7 @@ func TestBestFirst_HonorsCancelledContext(t *testing.T) {
 // own call was cancelled would.
 type cancellingScorer struct{ cancel context.CancelFunc }
 
-func (s cancellingScorer) Score(context.Context, entity.Batch) (float64, error) {
+func (s cancellingScorer) Score(context.Context, entity.Batch, entity.SpeculationPathSet) (float64, error) {
 	s.cancel()
 	return 0, context.Canceled
 }

submitqueue/extension/speculation/scorer/README.md

@@ -1,17 +1,23 @@
 # scorer
 
-A `Scorer` returns the probability that a batch ultimately succeeds — reaches its terminal `Succeeded` state with its changes landed, not merely a passing build — as a number between 0.0 and 1.0. It is handed the batch identity and resolves the batch's changes itself through an injected `changeset.Resolver`, so callers pass an `entity.Batch` and nothing more.
+A `Scorer` returns how likely a batch is to reach `Succeeded` with its changes landed, as a number between 0.0 and 1.0. `Score(ctx, batch, paths)` is handed the batch identity and that batch's own `SpeculationPathSet` — zero-valued when nothing has speculated on it yet. Callers pass a snapshot they already hold; a scorer must not load the path-set store.
 
 Callers may score every batch a queue is waiting on, so implementations should be cheap. A speculation run scores each batch at most once, but it does not carry results across runs; anything expensive belongs behind the implementation's own cache.
 
+The default `bestfirst` generator ranks on this number. The default scorer is **evidence** wrapping a **base**: heuristic or composite prices the change and ignores `paths`; evidence revises that price from the path set and batch state.
+
 Like the other extensions, a `Scorer` is selected **per queue** by the wiring layer through the `Config` (queue name) and `Factory` interface.
 
+See [doc/rfc/submitqueue/outcome-predictor.md](../../../../doc/rfc/submitqueue/outcome-predictor.md) for the GLM, factor contract, evidence rules, and configuration shape.
+
 ## Implementations
 
-**`heuristic`** scores a batch by extracting one number from its changes and matching that against ordered buckets, each mapping a `[Min, Max]` range to a probability. The extraction is a caller-supplied `ValueFunc` over the resolved `entity.BatchChanges`, so the same bucketing works for files touched, lines changed, or any other metric.
+**`evidence`** revises a nested base scorer with YAML-configured factors for `pathPassed`, `pathFailed`, `merging`, and `cancelling`. A factor of `1` leaves the base price alone; every factor defaults to `1` until someone sets one. Only paths that assume every dependency succeeds count as path evidence.
+
+**`heuristic`** scores a batch by extracting one number from its changes and matching that against ordered buckets, each mapping a `[Min, Max]` range to a probability. The extraction is a caller-supplied `ValueFunc` over the resolved `entity.BatchChanges`, so the same bucketing works for files touched, lines changed, or any other metric. It ignores `paths`.
 
-**`composite`** runs several named scorers and reduces their scores to one. The reduce function receives the scores keyed by scorer name, so it can weigh sources differently rather than treating them as interchangeable; `Min`, `Max`, and `Avg` are provided.
+**`composite`** runs several named scorers and reduces their scores to one. The reduce function receives the scores keyed by scorer name, so it can weigh sources differently rather than treating them as interchangeable; `Min`, `Max`, and `Avg` are provided. It ignores `paths` except to forward them to children.
 
 ## Adding a backend
 
-Create a package under `scorer/<backend>/` whose `New(...)` returns a `scorer.Scorer`, injecting whatever it needs at construction — a `changeset.Resolver` to reach the batch's changes, a metrics scope, any client. Do not add a `Config` or `Factory` implementation here; per-queue routing and the factory adapter live in the wiring layer.
+Create a package under `scorer/<backend>/` whose `New(...)` returns a `scorer.Scorer`, injecting whatever it needs at construction — a nested `Scorer` for evidence, a `changeset.Resolver` to reach the batch's changes, a metrics scope, any client. Do not add a `Config` or `Factory` implementation here; per-queue routing and the factory adapter live in the wiring layer.

submitqueue/extension/speculation/scorer/composite/scorer.go

@@ -94,13 +94,13 @@ func New(cfg scorer.Config, scorers map[string]scorer.Scorer, reduce ReduceFunc,
 
 // Score evaluates all child scorers on the batch and combines their results using the
 // reduce function. If any child scorer returns an error, that error is returned immediately.
-func (c *compositeScorer) Score(ctx context.Context, batch entity.Batch) (ret float64, retErr error) {
+func (c *compositeScorer) Score(ctx context.Context, batch entity.Batch, paths entity.SpeculationPathSet) (ret float64, retErr error) {
 	op := metrics.Begin(c.scope, "score", metrics.FastLatencyBuckets)
 	defer func() { op.Complete(retErr) }()
 
 	scores := make(map[string]float64, len(c.scorers))
 	for name, s := range c.scorers {
-		score, err := s.Score(ctx, batch)
+		score, err := s.Score(ctx, batch, paths)
 		if err != nil {
 			return 0, err
 		}

submitqueue/extension/speculation/scorer/composite/scorer_test.go

@@ -34,14 +34,14 @@ type fixedScorer struct {
 	score float64
 }
 
-func (f *fixedScorer) Score(_ context.Context, _ entity.Batch) (float64, error) {
+func (f *fixedScorer) Score(_ context.Context, _ entity.Batch, _ entity.SpeculationPathSet) (float64, error) {
 	return f.score, nil
 }
 
 // errorScorer always returns an error.
 type errorScorer struct{}
 
-func (e *errorScorer) Score(_ context.Context, _ entity.Batch) (float64, error) {
+func (e *errorScorer) Score(_ context.Context, _ entity.Batch, _ entity.SpeculationPathSet) (float64, error) {
 	return 0, fmt.Errorf("scorer failed")
 }
 
@@ -102,7 +102,7 @@ func TestScorer_Score(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			s := New(testCfg, tt.scorers, tt.reduce, tally.NoopScope)
-			got, err := s.Score(context.Background(), entity.Batch{})
+			got, err := s.Score(context.Background(), entity.Batch{}, entity.SpeculationPathSet{})
 			require.NoError(t, err)
 			assert.InDelta(t, tt.want, got, 1e-9)
 		})
@@ -114,7 +114,7 @@ func TestScorer_Score_ChildError(t *testing.T) {
 		"error": &errorScorer{},
 		"files": &fixedScorer{0.9},
 	}, Min, tally.NoopScope)
-	_, err := s.Score(context.Background(), entity.Batch{})
+	_, err := s.Score(context.Background(), entity.Batch{}, entity.SpeculationPathSet{})
 	require.Error(t, err)
 }
 
@@ -143,7 +143,7 @@ func TestReduceFunc_ReceivesNames(t *testing.T) {
 		"files": &fixedScorer{0.9},
 		"deps":  &fixedScorer{0.95},
 	}, custom, tally.NoopScope)
-	got, err := s.Score(context.Background(), entity.Batch{})
+	got, err := s.Score(context.Background(), entity.Batch{}, entity.SpeculationPathSet{})
 	require.NoError(t, err)
 	assert.Equal(t, 0.9, got)
 	assert.ElementsMatch(t, []string{"files", "deps"}, receivedNames)

submitqueue/extension/speculation/scorer/evidence/BUILD.bazel

@@ -0,0 +1,27 @@
+load("@rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["evidence.go"],
+    importpath = "github.com/uber/submitqueue/submitqueue/extension/speculation/scorer/evidence",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//platform/metrics:go_default_library",
+        "//submitqueue/entity:go_default_library",
+        "//submitqueue/extension/speculation/scorer:go_default_library",
+        "@com_github_uber_go_tally//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["evidence_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//submitqueue/entity:go_default_library",
+        "//submitqueue/extension/speculation/scorer:go_default_library",
+        "@com_github_stretchr_testify//assert:go_default_library",
+        "@com_github_stretchr_testify//require:go_default_library",
+        "@com_github_uber_go_tally//:go_default_library",
+    ],
+)

submitqueue/extension/speculation/scorer/evidence/evidence.go

@@ -0,0 +1,167 @@
+// Copyright (c) 2025 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package evidence revises a base Scorer's price with factors for observed batch
+// progress. See doc/rfc/submitqueue/outcome-predictor.md.
+package evidence
+
+import (
+	"fmt"
+	"math"
+
+	"context"
+
+	"github.com/uber-go/tally"
+	"github.com/uber/submitqueue/platform/metrics"
+	"github.com/uber/submitqueue/submitqueue/entity"
+	"github.com/uber/submitqueue/submitqueue/extension/speculation/scorer"
+)
+
+// Factors revise the base price, one per piece of evidence. A factor of 1
+// leaves the price alone. Named fields make unknown evidence fail to compile.
+type Factors struct {
+	// PathPassed applies once when a build has passed on the batch's
+	// all-succeed path.
+	PathPassed float64
+	// PathFailed applies once when the all-succeed path has failed.
+	PathFailed float64
+	// Merging applies while the batch is merging.
+	Merging float64
+	// Cancelling applies while the batch is cancelling.
+	Cancelling float64
+}
+
+// AllOnes is the neutral set: Score returns the base price.
+func AllOnes() Factors {
+	return Factors{PathPassed: 1, PathFailed: 1, Merging: 1, Cancelling: 1}
+}
+
+// epsilon keeps exact certainty revisable while remaining close to the scorer.
+const epsilon = 1e-6
+
+// evidence is a scorer.Scorer that revises a base scorer's price.
+type evidence struct {
+	// cfg is the per-queue identity this scorer was built for.
+	cfg scorer.Config
+	// base prices the batch's change; its price is what the factors revise.
+	base scorer.Scorer
+	// factors revise the base price with observed evidence.
+	factors Factors
+	// scope is the tally scope for emitting metrics.
+	scope tally.Scope
+}
+
+// New creates an evidence scorer bound to the queue named in cfg, revising
+// base's price by factors.
+//
+// It rejects a nil base and factors that are non-finite or not positive.
+func New(cfg scorer.Config, base scorer.Scorer, factors Factors, scope tally.Scope) (scorer.Scorer, error) {
+	if base == nil {
+		return nil, fmt.Errorf("evidence.New: base must not be nil")
+	}
+	for name, factor := range map[string]float64{
+		"PathPassed": factors.PathPassed,
+		"PathFailed": factors.PathFailed,
+		"Merging":    factors.Merging,
+		"Cancelling": factors.Cancelling,
+	} {
+		// Zero would permanently pin matching batches to 0; negatives cannot
+		// represent either direction in the factor contract.
+		if !(factor > 0) || math.IsInf(factor, 0) {
+			return nil, fmt.Errorf("evidence.New: factor %s must be finite and positive, got %v", name, factor)
+		}
+	}
+	return &evidence{cfg: cfg, base: base, factors: factors, scope: scope}, nil
+}
+
+// Score prices the batch's change, combines its evidence factors, and revises
+// the base price with the result.
+func (r *evidence) Score(ctx context.Context, batch entity.Batch, paths entity.SpeculationPathSet) (ret float64, retErr error) {
+	op := metrics.Begin(r.scope, "score", metrics.FastLatencyBuckets)
+	defer func() { op.Complete(retErr) }()
+
+	price, err := r.base.Score(ctx, batch, paths)
+	if err != nil {
+		return 0, err
+	}
+	// A price that is not a probability is a broken scorer, not a low opinion of
+	// the batch. Saying so leaves the caller to fall back on its own default,
+	// where clamping would hand back a number that looks deliberate.
+	if !(price >= 0 && price <= 1) {
+		return 0, fmt.Errorf("base scorer returned %v, which is not a probability", price)
+	}
+
+	factor := 1.0
+	if hasPassedAllSucceedPath(paths) {
+		factor *= r.factors.PathPassed
+	}
+	if hasFailedAllSucceedPath(paths) {
+		factor *= r.factors.PathFailed
+	}
+	switch batch.State {
+	case entity.BatchStateMerging:
+		factor *= r.factors.Merging
+	case entity.BatchStateCancelling:
+		factor *= r.factors.Cancelling
+	}
+	if factor == 1 {
+		return price, nil
+	}
+	return revise(math.Min(math.Max(price, epsilon), 1-epsilon), factor), nil
+}
+
+// revise applies the combined factor while keeping the result a probability.
+func revise(price, factor float64) float64 {
+	if math.IsInf(factor, 1) {
+		return 1 - epsilon
+	}
+	revised := price * factor / (1 - price + price*factor)
+	return math.Min(math.Max(revised, epsilon), 1-epsilon)
+}
+
+// hasPassedAllSucceedPath reports a passed build on the batch's all-succeed
+// path. Only that path counts: one built without a dependency's changes says
+// nothing about a candidate that assumes the dependency lands.
+func hasPassedAllSucceedPath(paths entity.SpeculationPathSet) bool {
+	for _, entry := range paths.Paths {
+		if entry.Status != entity.SpeculationPathStatusPassed {
+			continue
+		}
+		if assumesAllSucceed(entry.Path) {
+			return true
+		}
+	}
+	return false
+}
+
+// assumesAllSucceed reports whether every dependency is assumed to succeed.
+func assumesAllSucceed(path entity.SpeculationPath) bool {
+	for _, dep := range path.Dependencies {
+		if dep.Assumption != entity.DependencyAssumptionSucceeds {
+			return false
+		}
+	}
+	return true
+}
+
+// hasFailedAllSucceedPath reports a failed build on the batch's all-succeed
+// path. Flip-subset failures were built under different assumptions.
+func hasFailedAllSucceedPath(paths entity.SpeculationPathSet) bool {
+	for _, entry := range paths.Paths {
+		if entry.Status == entity.SpeculationPathStatusFailed && assumesAllSucceed(entry.Path) {
+			return true
+		}
+	}
+	return false
+}