From cfe8cdda6a4abaaaec75e702571b26ebdaf5f72b Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 30 Jun 2026 15:41:55 -0400 Subject: [PATCH 01/19] design: phase classes Move phase functions out of the bootstrapper into their own classes. Signed-off-by: Doug Hellmann --- docs/proposals/phase-classes-plan.md | 320 +++++++++++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 docs/proposals/phase-classes-plan.md diff --git a/docs/proposals/phase-classes-plan.md b/docs/proposals/phase-classes-plan.md new file mode 100644 index 00000000..9efc7009 --- /dev/null +++ b/docs/proposals/phase-classes-plan.md @@ -0,0 +1,320 @@ +# Plan: Refactor Bootstrapper Phase Handlers into PhaseItem Class Hierarchy + +## Context + +`Bootstrapper` in `src/fromager/bootstrapper.py` has grown large (~2268 lines). The seven `_phase_*` methods that implement the bootstrap loop's processing logic are co-mingled with orchestration, state tracking, and utility methods. The goal is to move each phase handler into its own class (`PhaseItem` subclass) that lives on the stack, wrapping a `WorkItem` data container. This makes each phase self-contained and easier to understand, test, and extend. + +## Design + +### New: `PhaseItem` abstract base class + +Each object pushed onto the bootstrap stack is a `PhaseItem`. It wraps a `WorkItem` (the accumulated per-package state) and implements the logic for one phase. + +```python +class PhaseItem(abc.ABC): + phase: typing.ClassVar[BootstrapPhase] # which phase this item represents + tracks_why: typing.ClassVar[bool] = True # whether to push onto why stack + + def __init__(self, work_item: WorkItem) -> None: + self.work_item = work_item + self.bg_future: concurrent.futures.Future[typing.Any] | None = None + + @abc.abstractmethod + def run(self, bt: Bootstrapper) -> list[PhaseItem]: ... + + def background_work(self, bt: Bootstrapper) -> typing.Callable[[], typing.Any] | None: + """Return a zero-argument callable for background I/O, or None. + Override in subclasses that need background prefetching. + ``bt`` is provided so subclasses can capture Bootstrapper state + (e.g. resolver, ctx) into the returned closure without storing + a circular reference on the item itself.""" + return None + + def __str__(self) -> str: + """Human-readable representation for logging. + Default: ``"()"``. Subclasses may override.""" + wi = self.work_item + return f"{type(self).__name__}({wi.req})" + + def as_json(self) -> dict[str, typing.Any]: + """Return a JSON-serialisable dict for stack-state recording. + The base implementation covers all common ``WorkItem`` fields. + Subclasses may override to add phase-specific entries.""" + wi = self.work_item + return { + "req": str(wi.req), + "req_type": str(wi.req_type), + "phase": str(self.phase), + "resolved_version": str(wi.resolved_version) if wi.resolved_version is not None else None, + "source_url": wi.source_url, + "build_sdist_only": wi.build_sdist_only, + "why": [ + {"req_type": str(rt), "req": str(r), "version": str(v)} + for rt, r, v in wi.why_snapshot + ], + "parent": ( + {"req": str(wi.parent[0]), "version": str(wi.parent[1])} + if wi.parent else None + ), + "build_system_deps": sorted(str(r) for r in wi.build_system_deps), + "build_backend_deps": sorted(str(r) for r in wi.build_backend_deps), + "build_sdist_deps": sorted(str(r) for r in wi.build_sdist_deps), + } +``` + +### New: 7 concrete `PhaseItem` subclasses + +| Class | phase | tracks_why | overrides background_work | +| -- | -- | -- | -- | +| `ResolveItem` | RESOLVE | False | Yes (calls `_bg_resolve`) | +| `StartItem` | START | False | No | +| `PrepareSourceItem` | PREPARE_SOURCE | True | Yes (calls `_bg_prepare_source` or `_bg_prepare_prebuilt`) | +| `PrepareBuildItem` | PREPARE_BUILD | True | No | +| `BuildItem` | BUILD | True | No | +| `ProcessInstallDepsItem` | PROCESS_INSTALL_DEPS | True | No | +| `CompleteItem` | COMPLETE | True | No | + +Each `run(self, bt: Bootstrapper) -> list[PhaseItem]` method contains the body of the corresponding `_phase_*` method. References to the old `self` (Bootstrapper) become `bt`; references to `item.*` become `self.work_item.*`. + +#### Phase advancement pattern + +The current handlers advance an item to the next phase by mutating `item.phase` then returning `[item]`. With `PhaseItem` classes the class encodes the phase, so mutation is impossible. Instead, each `run()` that continues the same package constructs a **new** `PhaseItem` subclass wrapping the same `work_item`: + +```python +# Old: mutation +item.phase = BootstrapPhase.PREPARE_BUILD +return [item] + dep_items + +# New: construction +return [PrepareBuildItem(self.work_item)] + dep_items +``` + +The expected return types per class are: + +| Class | Returns | +| -- | -- | +| `ResolveItem` | `list[StartItem]` (one per resolved version) | +| `StartItem` | `[]` (already seen) or `[PrepareSourceItem]` | +| `PrepareSourceItem` | `[ProcessInstallDepsItem]` (prebuilt) or `[PrepareBuildItem] + dep_items` (source) | +| `PrepareBuildItem` | `[BuildItem] + dep_items` | +| `BuildItem` | `[ProcessInstallDepsItem]` | +| `ProcessInstallDepsItem` | `[CompleteItem] + dep_items` | +| `CompleteItem` | `[]` | + +Assertions such as `assert item.bg_future is not None` in the current `_phase_resolve` and `_phase_prepare_source` bodies become `assert self.bg_future is not None` and must be retained in the corresponding `run()` methods. + +#### `background_work()` and module-level helpers + +`_bg_resolve`, `_bg_prepare_source`, and `_bg_prepare_prebuilt` are already **module-level functions** (not `Bootstrapper` methods). Closures built inside `background_work(self, bt)` capture them directly from module scope — `bt` is provided only to access `Bootstrapper` attributes (e.g. `bt._resolver`, `bt.ctx`) needed to construct the closure, not to store a reference to `bt` on the item. + +### Changes to `BootstrapPhase` + +The `BootstrapPhase` enum is **retained** — it is still referenced by `PhaseItem.phase: ClassVar[BootstrapPhase]` and used for its string values in `as_json()`, log messages, and error output. + +- **Remove** the `tracks_why` property — it becomes dead code once `tracks_why` moves to `PhaseItem` class variables. + +### Changes to `WorkItem` + +- **Remove** `phase: BootstrapPhase` field — phase is now encoded in the `PhaseItem` subclass type +- **Remove** `bg_future` field — moved to `PhaseItem` base class +- All other fields remain unchanged (they accumulate state across phases via `work_item`) + +### Changes to `Bootstrapper` + +| What | Change | +| -- | -- | +| `_phase_resolve`, `_phase_start`, `_phase_prepare_source`, `_phase_prepare_build`, `_phase_build`, `_phase_process_install_deps`, `_phase_complete` | **Delete** — logic moves into `PhaseItem` subclasses | +| `_get_background_work()` | **Delete** — replaced by `PhaseItem.background_work()` | +| `_dispatch_phase(item)` | Simplify to `return item.run(self)`; remove the `match/case` block and the `case _: raise ValueError(...)` fallthrough (unreachable once all subclasses are concrete) | +| `_push_items(stack, items)` | Accept `list[PhaseItem]`; retain the `if self._bg_pool is not None` guard; iterate `reversed(items)` (LIFO) to submit the highest-priority item first; call `item.background_work(self)` instead of `self._get_background_work(item)` and if not None, `item.bg_future = self._bg_pool.submit(bg_work)` | +| `_track_why(item)` | Accept `PhaseItem`; use `item.tracks_why` instead of `item.phase.tracks_why`; access `item.work_item.resolved_version`, `item.work_item.req_type`, `item.work_item.req` | +| `_record_stack_state(stack)` | Accept `list[PhaseItem]`; replace `[serialize(item) for item in reversed(stack)]` with `[item.as_json() for item in reversed(stack)]` — remove the inline `serialize()` helper entirely | +| `_handle_phase_error(item, err)` | Replace `item.phase == BootstrapPhase.RESOLVE` checks with `isinstance(item, ResolveItem)` etc.; replace phase-mutation fallback with `ProcessInstallDepsItem(item.work_item)` construction (see Error handling); update all `item.*` attribute accesses to `item.work_item.*` (see Attribute migration below) | +| `_create_unresolved_work_items(...)` | Return `list[ResolveItem]` instead of `list[WorkItem]`; remove `phase=BootstrapPhase.RESOLVE` from `WorkItem(...)` constructor; wrap each result: `ResolveItem(WorkItem(...))` | +| `bootstrap()` | Remove `phase=BootstrapPhase.RESOLVE` from `WorkItem(...)` calls; wrap each in `ResolveItem(WorkItem(...))` | +| `_bootstrap_one()` | Same — remove `phase=` from `WorkItem(...)`, wrap in `ResolveItem(WorkItem(...))` | +| `_run_bootstrap_loop(stack)` | Accept `list[PhaseItem]`; change `it.phase == BootstrapPhase.RESOLVE` progress-bar check to `isinstance(it, ResolveItem)`; change `self.why = list(item.why_snapshot)` to `self.why = list(item.work_item.why_snapshot)`; change `req_ctxvar_context(item.req, item.resolved_version)` to `req_ctxvar_context(item.work_item.req, item.work_item.resolved_version)` | + +#### Attribute migration in `Bootstrapper` methods + +After the refactor, `item` parameters that were `WorkItem` become `PhaseItem`. All attribute accesses must be updated: + +| Old (`WorkItem`) | New (`PhaseItem`) | +| -- | -- | +| `item.phase` | `type(item).phase` (ClassVar — still accessible on instance, but explicit is clearer) | +| `item.req` | `item.work_item.req` | +| `item.req_type` | `item.work_item.req_type` | +| `item.resolved_version` | `item.work_item.resolved_version` | +| `item.source_url` | `item.work_item.source_url` | +| `item.pbi_pre_built` | `item.work_item.pbi_pre_built` | +| `item.build_result` | `item.work_item.build_result` | +| `item.why_snapshot` | `item.work_item.why_snapshot` | +| `item.bg_future` | `item.bg_future` (moved to `PhaseItem` base — no change) | + +This applies to `_handle_phase_error`, `_run_bootstrap_loop`, `_track_why`, and any other `Bootstrapper` method that receives a `PhaseItem`. Inside each `PhaseItem.run()` method, references to the old `self` (Bootstrapper) become `bt` and references to the old `item.*` become `self.work_item.*`. + +#### `StartItem.run()` mutation ordering + +`StartItem.run()` must set `self.work_item.build_sdist_only` and `self.work_item.pbi_pre_built` **before** constructing `PrepareSourceItem(self.work_item)`, because `PrepareSourceItem.background_work(bt)` reads `work_item.pbi_pre_built` to choose between `_bg_prepare_source` and `_bg_prepare_prebuilt`: + +```python +# Inside StartItem.run(): +self.work_item.build_sdist_only = ( + bt.sdist_only and not bt._processing_build_requirement(self.work_item.req_type) +) +self.work_item.pbi_pre_built = bt.ctx.package_build_info(self.work_item.req).pre_built +# Now safe to hand work_item to the next phase: +return [PrepareSourceItem(self.work_item)] +``` + +### Error handling + +`_handle_phase_error` uses `isinstance` checks: + +```python +if isinstance(item, ResolveItem): ... +if isinstance(item, (PrepareSourceItem, PrepareBuildItem, BuildItem)): ... +``` + +The test-mode prebuilt fallback currently mutates `item.phase` before returning: + +```python +# Old: mutation +item.build_result = fallback +item.phase = BootstrapPhase.PROCESS_INSTALL_DEPS +return [item] +``` + +With `PhaseItem` classes this must instead construct a new item: + +```python +# New: construction +item.work_item.build_result = fallback +return [ProcessInstallDepsItem(item.work_item)] +``` + +## Files to Modify + +- `src/fromager/bootstrapper.py` — all changes (primary file) +- `tests/test_bootstrapper.py` — update affected tests + +## Test Changes + +- `test_phase_build_produces_source_build_result`: + + - Replace `WorkItem(phase=BootstrapPhase.BUILD, ...)` construction with `BuildItem(WorkItem(...))` (no `phase=` arg) + - Change `bt._phase_build(item)` → `item.run(bt)` inside the `bt._track_why(item)` context + - Change `result_items[0].phase == BootstrapPhase.PROCESS_INSTALL_DEPS` → `isinstance(result_items[0], ProcessInstallDepsItem)` + +- `_make_resolve_item()` helper: + + - Remove `phase=BootstrapPhase.RESOLVE` from `WorkItem(...)` call + - Wrap result in `ResolveItem`: `return ResolveItem(WorkItem(...))` + - Update return type annotation from `bootstrapper.WorkItem` to `bootstrapper.ResolveItem` + - Update all callers that compare `item.phase` directly to use `type(item).phase` or `isinstance(item, ResolveItem)` + +- `_record_and_load()` helper: + + - Change parameter type annotation from `list[bootstrapper.WorkItem]` to `list[bootstrapper.PhaseItem]` + +- `test_record_stack_state_full_item`: + + - Replace `WorkItem(phase=BootstrapPhase.BUILD, ...)` with `BuildItem(WorkItem(...))` (remove `phase=` arg) + +- `test_record_stack_state_dep_sets_are_sorted`: + + - Replace `WorkItem(phase=BootstrapPhase.BUILD, ...)` with `BuildItem(WorkItem(...))` (remove `phase=` arg) + +- `test_multiple_versions_continues_on_error` and similar tests that check `item.phase in (BootstrapPhase.RESOLVE, BootstrapPhase.START, ...)` inside mock dispatchers: + + - Replace phase equality checks with `isinstance(item, (ResolveItem, StartItem, ...))` + +- Tests for `_handle_phase_error`: + + - Any test that constructs a mid-pipeline `WorkItem` (e.g. with `phase=PREPARE_SOURCE`) and passes it to `_handle_phase_error` must instead construct the corresponding `PhaseItem` subclass (e.g. `PrepareSourceItem(WorkItem(...))`) + - Tests that check the prebuilt fallback return value must assert `isinstance(result[0], ProcessInstallDepsItem)` rather than checking a mutated `WorkItem.phase` + +- All remaining tests that import or reference `WorkItem` with a `phase=` keyword argument must drop the `phase=` argument + +## New Tests + +These test cases do not exist yet and should be added to cover the new class hierarchy and changed method behaviors. + +### `PhaseItem` base class + +- `test_phase_item_str_default` — `str(BuildItem(work_item))` returns `"BuildItem()"` using the default `__str__` implementation. +- `test_phase_item_background_work_returns_none_by_default` — phases that do not override `background_work()` (`PrepareBuildItem`, `BuildItem`, `ProcessInstallDepsItem`, `CompleteItem`) return `None`. + +### Class variable correctness + +- `test_phase_class_variables` — parametrize over all 7 subclasses; assert `MyClass.phase == expected_phase` and `MyClass.tracks_why == expected_bool` as listed in the design table. Verifies that class-variable declarations are not accidentally overridden by instance state. + +### `background_work()` dispatch + +- `test_resolve_item_background_work_returns_callable` — `ResolveItem(work_item).background_work(bt)` returns a non-`None` callable (does not call it; just verifies a callable is produced). +- `test_prepare_source_item_background_work_source` — when `work_item.pbi_pre_built=False`, `PrepareSourceItem.background_work(bt)` returns a callable that wraps `_bg_prepare_source`. +- `test_prepare_source_item_background_work_prebuilt` — when `work_item.pbi_pre_built=True`, `PrepareSourceItem.background_work(bt)` returns a callable that wraps `_bg_prepare_prebuilt`. + +### `StartItem` mutation ordering + +- `test_start_item_sets_pbi_pre_built_before_constructing_prepare_source` — `StartItem.run(bt)` must set `work_item.pbi_pre_built` before the returned `PrepareSourceItem` is constructed, so that `PrepareSourceItem.background_work(bt)` immediately sees the correct value. Verify by checking `result[0].work_item.pbi_pre_built` is set correctly and that calling `result[0].background_work(bt)` (with the same bootstrapper) returns the appropriate callable branch. + +### Phase advancement shares `work_item` identity + +- `test_phase_advancement_preserves_work_item_identity` — when `BuildItem(wi).run(bt)` returns `[ProcessInstallDepsItem(...)]`, assert `result[0].work_item is wi` (same object, not a copy). Applies to any phase that returns the next phase wrapping the same `work_item`. + +### `CompleteItem` + +- `test_complete_item_run_returns_empty_list` — `CompleteItem(work_item).run(bt)` returns `[]`. + +### `_dispatch_phase` delegation + +- `test_dispatch_phase_calls_item_run` — `bt._dispatch_phase(item)` calls `item.run(bt)` and returns its result without a `match/case` block. Patch `item.run` to return a sentinel list and assert the sentinel is returned. + +### `_track_why` with `PhaseItem` + +- `test_track_why_not_pushed_for_no_tracks_why_item` — inside `bt._track_why(resolve_item)`, `bt.why` is not modified (because `ResolveItem.tracks_why` is `False`). +- `test_track_why_pushed_for_tracks_why_item` — inside `bt._track_why(build_item)`, `bt.why` gains one entry while in the context and is restored on exit (because `BuildItem.tracks_why` is `True`). + +### `_push_items` background future submission + +- `test_push_items_sets_bg_future_when_pool_exists` — construct a `Bootstrapper` with a live `ThreadPoolExecutor` as `_bg_pool`; call `_push_items` with a `ResolveItem`; assert `item.bg_future` is not `None` after the call. +- `test_push_items_no_bg_future_when_pool_is_none` — when `bt._bg_pool` is `None`, calling `_push_items` with a `ResolveItem` leaves `item.bg_future` as `None`. + +### `as_json()` phase field per subclass + +- `test_as_json_phase_field_per_subclass` — parametrize over all 7 `(SubclassType, BootstrapPhase)` pairs; construct the subclass wrapping a minimal `WorkItem` and assert `item.as_json()["phase"] == str(expected_phase)`. Guards against a subclass accidentally inheriting the wrong `phase` ClassVar or the serialization using a stale field. + +### `_create_unresolved_work_items` return type + +- `test_create_unresolved_work_items_returns_resolve_items` — the returned items are `ResolveItem` instances (not bare `WorkItem` objects). Each `item.work_item` holds the expected `req` and `req_type`. + +## Verification + +```bash +# After each phase class is extracted: +hatch run mypy:check src/fromager/bootstrapper.py +hatch run test:test tests/test_bootstrapper.py + +# Full check at the end: +hatch run lint:fix src/fromager/bootstrapper.py tests/test_bootstrapper.py +hatch run mypy:check src/fromager/bootstrapper.py +hatch run test:test tests/test_bootstrapper.py +hatch run lint:check src/fromager/bootstrapper.py +``` + +## Implementation Order + +1. Add `abc` import; define `PhaseItem` abstract base class above `Bootstrapper` +2. Create each concrete subclass one at a time, moving `_phase_*` body into `run()`: + a. `ResolveItem` (with `background_work()` override) + b. `StartItem` + c. `PrepareSourceItem` (with `background_work()` override) + d. `PrepareBuildItem` + e. `BuildItem` + f. `ProcessInstallDepsItem` + g. `CompleteItem` +3. Update `WorkItem`: remove `phase` and `bg_future` fields +4. Update `Bootstrapper`: remove deleted methods, simplify `_dispatch_phase`, `_push_items`, `_track_why`, `_record_stack_state`, `_handle_phase_error`, `_create_unresolved_work_items`, `bootstrap()`, `_bootstrap_one()` +5. Update `tests/test_bootstrapper.py` +6. Run full verification From 06830e3383cc4aab8f92e4e2246d7018b46a7adf Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 1 Jul 2026 08:01:25 -0400 Subject: [PATCH 02/19] refactor(bootstrapper): extract PhaseItem class hierarchy from Bootstrapper Move the seven `_phase_*` handler methods out of `Bootstrapper` and into a `PhaseItem` abstract base class hierarchy. Each concrete subclass (`ResolveItem`, `StartItem`, `PrepareSourceItem`, `PrepareBuildItem`, `BuildItem`, `ProcessInstallDepsItem`, `CompleteItem`) encodes one bootstrap phase and implements its own `run()` method, making each phase self-contained. Key changes: - Add `PhaseItem` ABC with `phase`/`tracks_why` ClassVars, `bg_future`, `background_work()`, `__str__()`, and `as_json()` - Add 7 concrete `PhaseItem` subclasses; move `_phase_*` bodies into their `run()` methods - Remove `phase` and `bg_future` fields from `WorkItem` - Remove `tracks_why` property from `BootstrapPhase` enum - Remove `_dispatch_phase` (was a one-liner: `item.run(self)`) and inline the call in `_run_bootstrap_loop` - Remove `_get_background_work`; replace with `PhaseItem.background_work()` - Remove all 7 `_phase_*` methods from `Bootstrapper` - Simplify `_push_items`, `_track_why`, `_record_stack_state`, `_handle_phase_error`, and related methods to use `PhaseItem` - Update tests to use `PhaseItem` subclass instances and `item.run(bt)` Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper.py | 1615 +++++++++++++------------- tests/test_bootstrapper.py | 135 ++- tests/test_bootstrapper_iterative.py | 966 +++++++++------ 3 files changed, 1527 insertions(+), 1189 deletions(-) diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index 38a0bd5f..ca55e126 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -1,5 +1,6 @@ from __future__ import annotations +import abc import concurrent.futures import contextlib import dataclasses @@ -127,18 +128,14 @@ class BootstrapPhase(StrEnum): PROCESS_INSTALL_DEPS = "process-install-deps" COMPLETE = "complete" - @property - def tracks_why(self) -> bool: - """Whether this phase pushes onto the dependency-chain (why) stack.""" - return self not in (BootstrapPhase.RESOLVE, BootstrapPhase.START) - @dataclasses.dataclass class WorkItem: """A unit of work in the iterative bootstrap loop. Carries identity fields set at creation time and accumulated state - populated across phases as processing advances. + populated across phases as processing advances. The current phase is + encoded by the ``PhaseItem`` subclass wrapping this object. Items enter at the RESOLVE phase with only req and req_type set. The RESOLVE phase populates source_url and resolved_version, then @@ -148,7 +145,6 @@ class WorkItem: # Identity (set at creation) req: Requirement req_type: RequirementType - phase: BootstrapPhase why_snapshot: list[tuple[RequirementType, Requirement, Version]] parent: tuple[Requirement, Version] | None = None @@ -169,12 +165,6 @@ class WorkItem: build_backend_deps: set[Requirement] = dataclasses.field(default_factory=set) build_sdist_deps: set[Requirement] = dataclasses.field(default_factory=set) - # Background future: set by _push_items for RESOLVE and PREPARE_SOURCE items. - # None for all other phases or before _push_items has been called. - bg_future: concurrent.futures.Future[typing.Any] | None = dataclasses.field( - default=None, compare=False, repr=False - ) - def _create_unpack_dir( work_dir: pathlib.Path, @@ -428,239 +418,810 @@ def _bg_prepare_prebuilt( return PreparedSourceData(wheel_filename=wheel_filename, unpack_dir=unpack_dir) -class Bootstrapper: - def __init__( - self, - ctx: context.WorkContext, - progressbar: progress.Progressbar | None = None, - prev_graph: DependencyGraph | None = None, - cache_wheel_server_url: str | None = None, - sdist_only: bool = False, - test_mode: bool = False, - multiple_versions: bool = False, - num_bg_threads: int = _DEFAULT_BG_THREADS, - ) -> None: - if test_mode and sdist_only: - raise ValueError( - "--test-mode requires full wheel builds; incompatible with --sdist-only" - ) +class PhaseItem(abc.ABC): + """Abstract base for items pushed onto the bootstrap stack. - self.ctx = ctx - self.progressbar = progressbar or progress.Progressbar(None) - self.prev_graph = prev_graph - self.cache_wheel_server_url = cache_wheel_server_url or ctx.wheel_server_url - self.sdist_only = sdist_only - self.test_mode = test_mode - self.multiple_versions = multiple_versions - self.why: list[tuple[RequirementType, Requirement, Version]] = [] - self._num_bg_threads = max(1, num_bg_threads) - self._bg_pool: concurrent.futures.ThreadPoolExecutor | None = ( - concurrent.futures.ThreadPoolExecutor( - max_workers=self._num_bg_threads, thread_name_prefix="fromager-bg" - ) - ) + Each subclass encodes one phase of the bootstrap pipeline. + Wraps a ``WorkItem`` (accumulated per-package state) and implements + the processing logic for that phase in ``run()``. + """ - # Delegate resolution to BootstrapRequirementResolver - self._resolver = bootstrap_requirement_resolver.BootstrapRequirementResolver( - ctx=ctx, - prev_graph=prev_graph, - multiple_versions=multiple_versions, - cache_wheel_server_url=self.cache_wheel_server_url, - ) - # Push items onto the stack as we start to resolve their - # dependencies so at the end we have a list of items that need to - # be built in order. - self._build_stack: list[typing.Any] = [] - self._build_requirements: set[tuple[NormalizedName, str]] = set() + phase: typing.ClassVar[BootstrapPhase] + tracks_why: typing.ClassVar[bool] = True - # Track requirements we've seen before so we don't resolve the - # same dependencies over and over and so we can break cycles in - # the dependency list. The key is the requirements spec, rather - # than the package, in case we do have multiple rules for the same - # package. - self._seen_requirements: set[SeenKey] = set() + def __init__(self, work_item: WorkItem) -> None: + self.work_item = work_item + self.bg_future: concurrent.futures.Future[typing.Any] | None = None - self._build_order_filename = self.ctx.work_dir / "build-order.json" - self._stack_filename = self.ctx.work_dir / "bootstrap-stack.json" - logger.info("recording bootstrap stack state to %s", self._stack_filename) + @abc.abstractmethod + def run(self, bt: Bootstrapper) -> list[PhaseItem]: ... - # Track failed packages in test mode (list of typed dicts for JSON export) - self.failed_packages: list[FailureRecord] = [] + def background_work( + self, bt: Bootstrapper + ) -> typing.Callable[[], typing.Any] | None: + """Return a zero-argument callable for background I/O, or None. - # Track failed versions in multiple_versions mode - self._failed_versions: dict[tuple[str, str], Exception] = {} + Override in subclasses that need background prefetching. + ``bt`` is provided so subclasses can capture Bootstrapper state + (e.g. resolver, ctx) into the returned closure without storing + a circular reference on the item itself. + """ + return None - def _resolve_and_add_top_level( - self, - req: Requirement, - ) -> tuple[str, Version] | None: - """Resolve a top-level requirement and add it to the dependency graph. + def __str__(self) -> str: + """Human-readable representation: ``"()"``.""" + wi = self.work_item + return f"{type(self).__name__}({wi.req})" + + def as_json(self) -> dict[str, typing.Any]: + """Return a JSON-serialisable dict for stack-state recording.""" + wi = self.work_item + return { + "req": str(wi.req), + "req_type": str(wi.req_type), + "phase": str(self.phase), + "resolved_version": str(wi.resolved_version) + if wi.resolved_version is not None + else None, + "source_url": wi.source_url, + "build_sdist_only": wi.build_sdist_only, + "why": [ + {"req_type": str(rt), "req": str(r), "version": str(v)} + for rt, r, v in wi.why_snapshot + ], + "parent": ( + {"req": str(wi.parent[0]), "version": str(wi.parent[1])} + if wi.parent + else None + ), + "build_system_deps": sorted(str(r) for r in wi.build_system_deps), + "build_backend_deps": sorted(str(r) for r in wi.build_backend_deps), + "build_sdist_deps": sorted(str(r) for r in wi.build_sdist_deps), + } - Private method called only by ``bootstrap()``. - This is the pre-resolution phase before recursive bootstrapping begins. - In test mode, catches resolution errors and records them as failures. +class ResolveItem(PhaseItem): + """RESOLVE phase: resolve versions and expand into StartItems.""" - When multiple_versions is enabled, resolves and adds all matching versions - to the graph, but still returns only the first (highest) version for - backward compatibility. + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.RESOLVE + tracks_why: typing.ClassVar[bool] = False - Args: - req: The top-level requirement to resolve. + def background_work( + self, bt: Bootstrapper + ) -> typing.Callable[[], typing.Any] | None: + """Return closure that calls ``_bg_resolve`` in a thread.""" + bg_resolver = bt._resolver + req = self.work_item.req + req_type = self.work_item.req_type + parent_req = ( + self.work_item.why_snapshot[-1][1] if self.work_item.why_snapshot else None + ) + return_all = bt.multiple_versions - Returns: - Tuple of (source_url, version) if resolution succeeded, None if it - failed in test mode. + def do_resolve() -> list[tuple[str, Version]]: + with req_ctxvar_context(req): + return _bg_resolve(bg_resolver, req, req_type, parent_req, return_all) - Raises: - Exception: In normal mode, re-raises any resolution error. + return do_resolve + + def run(self, bt: Bootstrapper) -> list[PhaseItem]: + """RESOLVE phase: resolve versions and expand into StartItems. + + Centralizes version resolution so all dependencies are expanded + uniformly. In multiple_versions mode, filters out versions that + already failed in this run and versions whose wheels are already + cached to avoid redundant builds and transitive dependency + processing. + + Returns: + One StartItem per resolved version that needs building. + Empty list if all versions are already cached. """ - try: - pbi = self.ctx.package_build_info(req) - results = self.resolve_versions( - req=req, - req_type=RequirementType.TOP_LEVEL, - parent_req=None, - return_all_versions=self.multiple_versions, + assert self.bg_future is not None + # bg_future.result() blocks until the background resolution completes, + # then returns the result or re-raises any exception from the background. + resolved_versions = self.bg_future.result() + if not resolved_versions: + raise RuntimeError( + f"Could not resolve any versions for {self.work_item.req}" ) - if self.multiple_versions: - logger.info(f"resolved {len(results)} version(s) for {req}") - # Add all resolved versions to the graph - for source_url, version in results: - logger.info("%s resolves to %s", req, version) - self.ctx.dependency_graph.add_dependency( - parent_name=None, - parent_version=None, - req_type=RequirementType.TOP_LEVEL, - req=req, - req_version=version, - download_url=source_url, - pre_built=pbi.pre_built, - constraint=self.ctx.constraints.get_constraint(req.name), + if bt.multiple_versions: + pkg_name = canonicalize_name(self.work_item.req.name) + resolved_versions = [ + (url, ver) + for url, ver in resolved_versions + if (pkg_name, str(ver)) not in bt._failed_versions + ] + if not resolved_versions: + raise RuntimeError( + f"Could not resolve any versions for {self.work_item.req}" + f" (all candidates failed previously)" ) - if not results: - if self.multiple_versions: - err = RuntimeError(f"no versions found for {req}") - self._record_failed_version( - req, "unresolved", err, "no versions resolved, skipping" + logger.info( + f"resolved {len(resolved_versions)} version(s) for {self.work_item.req}" + ) + filtered: list[tuple[str, Version]] = [] + for source_url, version in resolved_versions: + cached_wheel, _ = _find_cached_wheel( + bt.ctx, bt.cache_wheel_server_url, self.work_item.req, version + ) + if cached_wheel: + logger.info( + f"{self.work_item.req.name}=={version}: wheel already cached " + f"at {cached_wheel.name}, skipping" ) - if self.test_mode: - self._record_test_mode_failure(req, None, err, "resolution") - return None - raise RuntimeError(f"Could not resolve any versions for {req}") + else: + filtered.append((source_url, version)) + if not filtered: + # Always process the highest version (first in + # resolved_versions) so new transitive dependencies + # are discovered even when every wheel is cached. + logger.info( + f"all versions of {self.work_item.req.name} already cached, " + f"keeping highest version {resolved_versions[0][1]} " + f"for dependency discovery" + ) + filtered.append(resolved_versions[0]) + resolved_versions = filtered - # Return first result for backward compatibility - return results[0] - except Exception as err: - if self.multiple_versions: - self._record_failed_version(req, "unresolved", err, "failed to resolve") - if self.test_mode: - self._record_test_mode_failure(req, None, err, "resolution") - return None - if not self.test_mode: - raise - self._record_test_mode_failure(req, None, err, "resolution") - return None + # Build list so highest version ends up on top of the stack + # (last element after extend) and is processed first. + items: list[PhaseItem] = [] + for source_url, version in reversed(resolved_versions): + items.append( + StartItem( + WorkItem( + req=self.work_item.req, + req_type=self.work_item.req_type, + why_snapshot=list(self.work_item.why_snapshot), + parent=self.work_item.parent, + source_url=source_url, + resolved_version=version, + ) + ) + ) + return items - def resolve_versions( - self, - req: Requirement, - req_type: RequirementType, - parent_req: Requirement | None = None, - return_all_versions: bool = False, - ) -> list[tuple[str, Version]]: - """Resolve version(s) of a requirement. - Returns list of (source URL, version) tuples, sorted by version (highest first). +class StartItem(PhaseItem): + """START phase: add to graph, check if already seen.""" - Git URL resolution stays in Bootstrapper because it requires - build orchestration (BuildEnvironment, build dependencies). - Delegates PyPI/graph resolution to BootstrapRequirementResolver. + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.START + tracks_why: typing.ClassVar[bool] = False - Args: - req: Package requirement to resolve - req_type: Type of requirement - parent_req: Explicit parent requirement from dependency chain. - Callers must pass this explicitly; do not read ``self.why`` here. - return_all_versions: If True, return all matching versions. - If False, return only highest version. + def run(self, bt: Bootstrapper) -> list[PhaseItem]: + """START phase: add to graph, check if already seen. + + _track_why is a no-op for this phase (tracks_why is False), + matching the original behavior where graph addition and + seen-check happen before pushing onto the why stack. Returns: - List of (url, version) tuples. Contains one item when - return_all_versions=False, or all matching versions when True. + Empty list if already seen (nothing to do). + [PrepareSourceItem] if this is new work. """ - if req.url: - if req_type != RequirementType.TOP_LEVEL: - raise ValueError( - f"{req} includes a URL, but is not a top-level dependency" - ) + wi = self.work_item + assert wi.resolved_version is not None + assert wi.source_url is not None - # Check cache first to avoid re-resolving - # Git URLs are always source (not prebuilt) - cached_result = self._resolver.get_matching_versions(req, pre_built=False) - if cached_result: - logger.debug(f"resolved {req} from cache") - return cached_result if return_all_versions else [cached_result[0]] + # Add to graph (skip TOP_LEVEL, already added in _resolve_and_add_top_level) + if wi.req_type != RequirementType.TOP_LEVEL: + bt._add_to_graph( + wi.req, + wi.req_type, + wi.resolved_version, + wi.source_url, + wi.parent, + ) - logger.info("resolving source via URL, ignoring any plugins") - source_url, resolved_version = self._resolve_version_from_git_url(req=req) - # Cache the git URL resolution (always source, not prebuilt) - # Store as list for consistency with cache structure - result = [(source_url, resolved_version)] - self._resolver.extend_known_versions(req, pre_built=False, result=result) - return result # Git URLs always return single version + wi.build_sdist_only = bt.sdist_only and not bt._processing_build_requirement( + wi.req_type + ) - # Delegate to RequirementResolver - return self._resolver.resolve( - req=req, - req_type=req_type, - parent_req=parent_req, - return_all_versions=return_all_versions, + if bt._has_been_seen(wi.req, wi.resolved_version, wi.build_sdist_only): + logger.debug( + f"redundant {wi.req_type} dependency {wi.req} " + f"({wi.resolved_version}, sdist_only={wi.build_sdist_only}) " + f"for {bt._explain}" + ) + return [] + bt._mark_as_seen(wi.req, wi.resolved_version, wi.build_sdist_only) + + logger.info( + f"new {wi.req_type} dependency {wi.req} resolves to {wi.resolved_version}" ) - def bootstrap(self, requirements: list[Requirement]) -> None: - """Bootstrap all top-level requirements and their transitive dependencies. + # Must set pbi_pre_built before constructing PrepareSourceItem so that + # PrepareSourceItem.background_work() immediately sees the correct value. + wi.pbi_pre_built = bt.ctx.package_build_info(wi.req).pre_built + return [PrepareSourceItem(wi)] - .. versionadded:: 0.89 - Replaces the former ``bootstrap(req, req_type)`` signature. - Resolves each requirement, adds it to the dependency graph, and processes - the full dependency tree using an iterative DFS loop. Handles - ``requirement_ctxvar`` context internally; callers do not need to manage it. +class PrepareSourceItem(PhaseItem): + """PREPARE_SOURCE phase: download source or prebuilt, get build system deps.""" - In test mode, records failures and continues instead of raising. In - ``multiple_versions`` mode, processes all matching versions per requirement. + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PREPARE_SOURCE + tracks_why: typing.ClassVar[bool] = True - Args: - requirements: Top-level requirements to resolve and bootstrap. - """ - # Resolve all top-level reqs and build initial stack. - # Use the token pattern (no try/finally) so that if resolution raises - # in normal mode, the context var stays set for the top-level error - # handler in __main__.py to include the package name in its log message. - stack: list[WorkItem] = [] - initial_items: list[WorkItem] = [] - for req in requirements: - token = requirement_ctxvar.set(req) - result = self._resolve_and_add_top_level(req) - requirement_ctxvar.reset(token) - if result is not None: + def background_work( + self, bt: Bootstrapper + ) -> typing.Callable[[], typing.Any] | None: + """Return closure for background source download or prebuilt fetch.""" + wi = self.work_item + assert wi.resolved_version is not None + assert wi.source_url is not None + ctx = bt.ctx + cache_wheel_server_url = bt.cache_wheel_server_url + req = wi.req + req_type = wi.req_type + resolved_version = wi.resolved_version + source_url = wi.source_url + + if wi.pbi_pre_built: + + def do_prepare_prebuilt() -> PreparedSourceData: + with req_ctxvar_context(req, resolved_version): + return _bg_prepare_prebuilt( + ctx, req, req_type, resolved_version, source_url + ) + + return do_prepare_prebuilt + + def do_prepare_source() -> PreparedSourceData: + with req_ctxvar_context(req, resolved_version): + return _bg_prepare_source( + ctx, cache_wheel_server_url, req, resolved_version, source_url + ) + + return do_prepare_source + + def run(self, bt: Bootstrapper) -> list[PhaseItem]: + """PREPARE_SOURCE phase: download source or prebuilt, get build system deps. + + Uses background I/O result from ``self.bg_future`` when available, + falling back to inline I/O otherwise. + + Returns: + Prebuilt: [ProcessInstallDepsItem] (skip build phases). + Source: [PrepareBuildItem, *build_system_dep_items]. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.source_url is not None + + # bg_future is always set for PREPARE_SOURCE items (see _push_items). + # bg_future.result() blocks until done and re-raises any background exception. + assert self.bg_future is not None + prepared: PreparedSourceData = self.bg_future.result() + + constraint = bt.ctx.constraints.get_constraint(wi.req.name) + if constraint: + logger.info( + f"incoming requirement {wi.req} matches constraint " + f"{constraint}. Will apply both." + ) + + if wi.pbi_pre_built: + # Background task already downloaded the prebuilt wheel + assert prepared.wheel_filename is not None + assert prepared.unpack_dir is not None + wi.build_result = SourceBuildResult( + wheel_filename=prepared.wheel_filename, + sdist_filename=None, + unpack_dir=prepared.unpack_dir, + sdist_root_dir=None, + build_env=None, + source_type=SourceType.PREBUILT, + ) + return [ProcessInstallDepsItem(wi)] + + # Source build path: background task already downloaded and prepared the source + assert prepared.sdist_root_dir is not None + sdist_root_dir = prepared.sdist_root_dir + wi.cached_wheel_filename = prepared.cached_wheel_filename + + assert sdist_root_dir is not None + + if sdist_root_dir.parent.parent != bt.ctx.work_dir: + raise ValueError(f"'{sdist_root_dir}/../..' should be {bt.ctx.work_dir}") + wi.sdist_root_dir = sdist_root_dir + wi.unpack_dir = sdist_root_dir.parent + + wi.build_env = bt._create_build_env( + req=wi.req, + resolved_version=wi.resolved_version, + parent_dir=sdist_root_dir.parent, + ) + + # Get build system dependencies + wi.build_system_deps = dependencies.get_build_system_dependencies( + ctx=bt.ctx, + req=wi.req, + version=wi.resolved_version, + sdist_root_dir=sdist_root_dir, + ) + + dep_items = bt._create_unresolved_work_items( + wi.build_system_deps, + RequirementType.BUILD_SYSTEM, + wi.req, + wi.resolved_version, + ) + + return [PrepareBuildItem(wi)] + dep_items + + +class PrepareBuildItem(PhaseItem): + """PREPARE_BUILD phase: install system deps, get backend/sdist deps.""" + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PREPARE_BUILD + tracks_why: typing.ClassVar[bool] = True + + def run(self, bt: Bootstrapper) -> list[PhaseItem]: + """PREPARE_BUILD phase: install system deps, get backend/sdist deps. + + Build-backend and build-sdist dependencies that are already satisfied + by a resolved build-system dependency reuse that version instead of + resolving independently (see :issue:`1194`). + + Returns: + [BuildItem, *backend_dep_items, *sdist_dep_items]. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.build_env is not None + assert wi.sdist_root_dir is not None + + # Install build system deps (their wheels exist from DFS processing) + wi.build_env.install(wi.build_system_deps) + + # Get build backend dependencies + wi.build_backend_deps = dependencies.get_build_backend_dependencies( + ctx=bt.ctx, + req=wi.req, + version=wi.resolved_version, + sdist_root_dir=wi.sdist_root_dir, + build_env=wi.build_env, + ) + + # Get build sdist dependencies + wi.build_sdist_deps = dependencies.get_build_sdist_dependencies( + ctx=bt.ctx, + req=wi.req, + version=wi.resolved_version, + sdist_root_dir=wi.sdist_root_dir, + build_env=wi.build_env, + ) + + # Filter out deps already satisfied by build-system dependencies + # to avoid resolving to a different (typically newer) version. + resolved_build_sys = bt._get_resolved_build_system_versions(wi) + parent = (wi.req, wi.resolved_version) + wi.build_backend_deps = bt._filter_deps_satisfied_by_build_system( + wi.build_backend_deps, + resolved_build_sys, + RequirementType.BUILD_BACKEND, + parent, + ) + wi.build_sdist_deps = bt._filter_deps_satisfied_by_build_system( + wi.build_sdist_deps, + resolved_build_sys, + RequirementType.BUILD_SDIST, + parent, + ) + + backend_items = bt._create_unresolved_work_items( + wi.build_backend_deps, + RequirementType.BUILD_BACKEND, + wi.req, + wi.resolved_version, + ) + sdist_items = bt._create_unresolved_work_items( + wi.build_sdist_deps, + RequirementType.BUILD_SDIST, + wi.req, + wi.resolved_version, + ) + dep_items = backend_items + sdist_items + + return [BuildItem(wi)] + dep_items + + +class BuildItem(PhaseItem): + """BUILD phase: install remaining deps, build wheel/sdist.""" + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.BUILD + tracks_why: typing.ClassVar[bool] = True + + def run(self, bt: Bootstrapper) -> list[PhaseItem]: + """BUILD phase: install remaining deps, build wheel/sdist. + + Returns: + [ProcessInstallDepsItem]. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.build_env is not None + assert wi.sdist_root_dir is not None + + # Drain all in-flight background I/O before an exclusive build starts. + pbi = bt.ctx.package_build_info(wi.req) + if pbi.exclusive_build: + logger.info("%s requires exclusive build, draining background pool", wi.req) + bt._drain_background_pool() + + # Install backend+sdist deps if disjoint from system deps + remaining_deps = wi.build_backend_deps | wi.build_sdist_deps + if remaining_deps.isdisjoint(wi.build_system_deps): + wi.build_env.install(remaining_deps) + + wheel_filename, sdist_filename = bt._do_build( + req=wi.req, + resolved_version=wi.resolved_version, + sdist_root_dir=wi.sdist_root_dir, + build_env=wi.build_env, + build_sdist_only=wi.build_sdist_only, + cached_wheel_filename=wi.cached_wheel_filename, + ) + + source_type = sources.get_source_type(bt.ctx, wi.req) + + wi.build_result = SourceBuildResult( + wheel_filename=wheel_filename, + sdist_filename=sdist_filename, + unpack_dir=wi.sdist_root_dir.parent, + sdist_root_dir=wi.sdist_root_dir, + build_env=wi.build_env, + source_type=source_type, + ) + + return [ProcessInstallDepsItem(wi)] + + +class ProcessInstallDepsItem(PhaseItem): + """PROCESS_INSTALL_DEPS phase: hooks, extract deps, build order.""" + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PROCESS_INSTALL_DEPS + tracks_why: typing.ClassVar[bool] = True + + def run(self, bt: Bootstrapper) -> list[PhaseItem]: + """PROCESS_INSTALL_DEPS phase: hooks, extract deps, build order. + + Returns: + [CompleteItem, *install_dep_items]. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.source_url is not None + assert wi.build_result is not None + + # Run post-bootstrap hooks (non-fatal in test mode) + try: + hooks.run_post_bootstrap_hooks( + ctx=bt.ctx, + req=wi.req, + dist_name=canonicalize_name(wi.req.name), + dist_version=str(wi.resolved_version), + sdist_filename=wi.build_result.sdist_filename, + wheel_filename=wi.build_result.wheel_filename, + ) + except Exception as hook_error: + if not bt.test_mode: + raise + bt._record_test_mode_failure( + wi.req, + str(wi.resolved_version), + hook_error, + "hook", + "warning", + ) + + # Extract install dependencies (non-fatal in test mode) + try: + install_dependencies = bt._get_install_dependencies( + req=wi.req, + resolved_version=wi.resolved_version, + wheel_filename=wi.build_result.wheel_filename, + sdist_filename=wi.build_result.sdist_filename, + sdist_root_dir=wi.build_result.sdist_root_dir, + build_env=wi.build_result.build_env, + unpack_dir=wi.build_result.unpack_dir, + ) + except Exception as dep_error: + if not bt.test_mode: + raise + bt._record_test_mode_failure( + wi.req, + str(wi.resolved_version), + dep_error, + "dependency_extraction", + "warning", + ) + install_dependencies = [] + + logger.debug( + "install dependencies: %s", + ", ".join(sorted(str(r) for r in install_dependencies)), + ) + + pbi = bt.ctx.package_build_info(wi.req) + constraint = bt.ctx.constraints.get_constraint(wi.req.name) + bt._add_to_build_order( + req=wi.req, + version=wi.resolved_version, + source_url=wi.source_url, + source_type=wi.build_result.source_type, + prebuilt=pbi.pre_built, + constraint=constraint, + ) + + dep_items = bt._create_unresolved_work_items( + install_dependencies, + RequirementType.INSTALL, + wi.req, + wi.resolved_version, + ) + + return [CompleteItem(wi)] + dep_items + + +class CompleteItem(PhaseItem): + """COMPLETE phase: clean up build directories.""" + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.COMPLETE + tracks_why: typing.ClassVar[bool] = True + + def run(self, bt: Bootstrapper) -> list[PhaseItem]: + """COMPLETE phase: clean up build directories. + + Returns: + Empty list (processing finished for this item). + """ + wi = self.work_item + if wi.build_result is not None: + bt.ctx.clean_build_dirs( + wi.build_result.sdist_root_dir, + wi.build_result.build_env, + ) + return [] + + +class Bootstrapper: + def __init__( + self, + ctx: context.WorkContext, + progressbar: progress.Progressbar | None = None, + prev_graph: DependencyGraph | None = None, + cache_wheel_server_url: str | None = None, + sdist_only: bool = False, + test_mode: bool = False, + multiple_versions: bool = False, + num_bg_threads: int = _DEFAULT_BG_THREADS, + ) -> None: + if test_mode and sdist_only: + raise ValueError( + "--test-mode requires full wheel builds; incompatible with --sdist-only" + ) + + self.ctx = ctx + self.progressbar = progressbar or progress.Progressbar(None) + self.prev_graph = prev_graph + self.cache_wheel_server_url = cache_wheel_server_url or ctx.wheel_server_url + self.sdist_only = sdist_only + self.test_mode = test_mode + self.multiple_versions = multiple_versions + self.why: list[tuple[RequirementType, Requirement, Version]] = [] + self._num_bg_threads = max(1, num_bg_threads) + self._bg_pool: concurrent.futures.ThreadPoolExecutor | None = ( + concurrent.futures.ThreadPoolExecutor( + max_workers=self._num_bg_threads, thread_name_prefix="fromager-bg" + ) + ) + + # Delegate resolution to BootstrapRequirementResolver + self._resolver = bootstrap_requirement_resolver.BootstrapRequirementResolver( + ctx=ctx, + prev_graph=prev_graph, + multiple_versions=multiple_versions, + cache_wheel_server_url=self.cache_wheel_server_url, + ) + # Push items onto the stack as we start to resolve their + # dependencies so at the end we have a list of items that need to + # be built in order. + self._build_stack: list[typing.Any] = [] + self._build_requirements: set[tuple[NormalizedName, str]] = set() + + # Track requirements we've seen before so we don't resolve the + # same dependencies over and over and so we can break cycles in + # the dependency list. The key is the requirements spec, rather + # than the package, in case we do have multiple rules for the same + # package. + self._seen_requirements: set[SeenKey] = set() + + self._build_order_filename = self.ctx.work_dir / "build-order.json" + self._stack_filename = self.ctx.work_dir / "bootstrap-stack.json" + logger.info("recording bootstrap stack state to %s", self._stack_filename) + + # Track failed packages in test mode (list of typed dicts for JSON export) + self.failed_packages: list[FailureRecord] = [] + + # Track failed versions in multiple_versions mode + self._failed_versions: dict[tuple[str, str], Exception] = {} + + def _resolve_and_add_top_level( + self, + req: Requirement, + ) -> tuple[str, Version] | None: + """Resolve a top-level requirement and add it to the dependency graph. + + Private method called only by ``bootstrap()``. + + This is the pre-resolution phase before recursive bootstrapping begins. + In test mode, catches resolution errors and records them as failures. + + When multiple_versions is enabled, resolves and adds all matching versions + to the graph, but still returns only the first (highest) version for + backward compatibility. + + Args: + req: The top-level requirement to resolve. + + Returns: + Tuple of (source_url, version) if resolution succeeded, None if it + failed in test mode. + + Raises: + Exception: In normal mode, re-raises any resolution error. + """ + try: + pbi = self.ctx.package_build_info(req) + results = self.resolve_versions( + req=req, + req_type=RequirementType.TOP_LEVEL, + parent_req=None, + return_all_versions=self.multiple_versions, + ) + if self.multiple_versions: + logger.info(f"resolved {len(results)} version(s) for {req}") + + # Add all resolved versions to the graph + for source_url, version in results: + logger.info("%s resolves to %s", req, version) + self.ctx.dependency_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=req, + req_version=version, + download_url=source_url, + pre_built=pbi.pre_built, + constraint=self.ctx.constraints.get_constraint(req.name), + ) + + if not results: + if self.multiple_versions: + err = RuntimeError(f"no versions found for {req}") + self._record_failed_version( + req, "unresolved", err, "no versions resolved, skipping" + ) + if self.test_mode: + self._record_test_mode_failure(req, None, err, "resolution") + return None + raise RuntimeError(f"Could not resolve any versions for {req}") + + # Return first result for backward compatibility + return results[0] + except Exception as err: + if self.multiple_versions: + self._record_failed_version(req, "unresolved", err, "failed to resolve") + if self.test_mode: + self._record_test_mode_failure(req, None, err, "resolution") + return None + if not self.test_mode: + raise + self._record_test_mode_failure(req, None, err, "resolution") + return None + + def resolve_versions( + self, + req: Requirement, + req_type: RequirementType, + parent_req: Requirement | None = None, + return_all_versions: bool = False, + ) -> list[tuple[str, Version]]: + """Resolve version(s) of a requirement. + + Returns list of (source URL, version) tuples, sorted by version (highest first). + + Git URL resolution stays in Bootstrapper because it requires + build orchestration (BuildEnvironment, build dependencies). + Delegates PyPI/graph resolution to BootstrapRequirementResolver. + + Args: + req: Package requirement to resolve + req_type: Type of requirement + parent_req: Explicit parent requirement from dependency chain. + Callers must pass this explicitly; do not read ``self.why`` here. + return_all_versions: If True, return all matching versions. + If False, return only highest version. + + Returns: + List of (url, version) tuples. Contains one item when + return_all_versions=False, or all matching versions when True. + """ + if req.url: + if req_type != RequirementType.TOP_LEVEL: + raise ValueError( + f"{req} includes a URL, but is not a top-level dependency" + ) + + # Check cache first to avoid re-resolving + # Git URLs are always source (not prebuilt) + cached_result = self._resolver.get_matching_versions(req, pre_built=False) + if cached_result: + logger.debug(f"resolved {req} from cache") + return cached_result if return_all_versions else [cached_result[0]] + + logger.info("resolving source via URL, ignoring any plugins") + source_url, resolved_version = self._resolve_version_from_git_url(req=req) + # Cache the git URL resolution (always source, not prebuilt) + # Store as list for consistency with cache structure + result = [(source_url, resolved_version)] + self._resolver.extend_known_versions(req, pre_built=False, result=result) + return result # Git URLs always return single version + + # Delegate to RequirementResolver + return self._resolver.resolve( + req=req, + req_type=req_type, + parent_req=parent_req, + return_all_versions=return_all_versions, + ) + + def bootstrap(self, requirements: list[Requirement]) -> None: + """Bootstrap all top-level requirements and their transitive dependencies. + + .. versionadded:: 0.89 + Replaces the former ``bootstrap(req, req_type)`` signature. + + Resolves each requirement, adds it to the dependency graph, and processes + the full dependency tree using an iterative DFS loop. Handles + ``requirement_ctxvar`` context internally; callers do not need to manage it. + + In test mode, records failures and continues instead of raising. In + ``multiple_versions`` mode, processes all matching versions per requirement. + + Args: + requirements: Top-level requirements to resolve and bootstrap. + """ + # Resolve all top-level reqs and build initial stack. + # Use the token pattern (no try/finally) so that if resolution raises + # in normal mode, the context var stays set for the top-level error + # handler in __main__.py to include the package name in its log message. + stack: list[PhaseItem] = [] + initial_items: list[PhaseItem] = [] + for req in requirements: + token = requirement_ctxvar.set(req) + result = self._resolve_and_add_top_level(req) + requirement_ctxvar.reset(token) + if result is not None: initial_items.append( - WorkItem( - req=req, - req_type=RequirementType.TOP_LEVEL, - phase=BootstrapPhase.RESOLVE, - why_snapshot=[], - parent=None, + ResolveItem( + WorkItem( + req=req, + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + parent=None, + ) ) ) self._push_items(stack, initial_items) self._run_bootstrap_loop(stack) - def _run_bootstrap_loop(self, stack: list[WorkItem]) -> None: + def _run_bootstrap_loop(self, stack: list[PhaseItem]) -> None: """Run the iterative DFS bootstrap loop over a pre-built work stack. Pops items one at a time, dispatches each phase, and pushes any @@ -668,26 +1229,24 @@ def _run_bootstrap_loop(self, stack: list[WorkItem]) -> None: stack. Updates the progress bar as items complete. Args: - stack: Initial list of ``WorkItem`` objects to process. Modified + stack: Initial list of ``PhaseItem`` objects to process. Modified in-place; empty on return. """ while stack: self._record_stack_state(stack) item = stack.pop() - self.why = list(item.why_snapshot) + self.why = list(item.work_item.why_snapshot) with ( - req_ctxvar_context(item.req, item.resolved_version), + req_ctxvar_context(item.work_item.req, item.work_item.resolved_version), self._track_why(item), ): try: - new_items = self._dispatch_phase(item) + new_items = item.run(self) except Exception as err: new_items = self._handle_phase_error(item, err) - new_dep_count = sum( - 1 for it in new_items if it.phase == BootstrapPhase.RESOLVE - ) + new_dep_count = sum(1 for it in new_items if isinstance(it, ResolveItem)) if new_dep_count > 0: self.progressbar.update_total(new_dep_count) if not new_items: @@ -750,15 +1309,16 @@ def _bootstrap_one(self, req: Requirement, req_type: RequirementType) -> None: saved_why = list(self.why) # Single RESOLVE item — resolution, version expansion, and error - # handling all happen inside the loop via _phase_resolve. - initial_item = WorkItem( - req=req, - req_type=req_type, - phase=BootstrapPhase.RESOLVE, - why_snapshot=list(self.why), - parent=parent, + # handling all happen inside the loop via ResolveItem.run(). + initial_item = ResolveItem( + WorkItem( + req=req, + req_type=req_type, + why_snapshot=list(self.why), + parent=parent, + ) ) - stack: list[WorkItem] = [] + stack: list[PhaseItem] = [] self._push_items(stack, [initial_item]) self._run_bootstrap_loop(stack) @@ -773,7 +1333,7 @@ def _bootstrap_one(self, req: Requirement, req_type: RequirementType) -> None: @contextlib.contextmanager def _track_why( self, - item: WorkItem, + item: PhaseItem, ) -> typing.Generator[None, None, None]: """Context manager to track dependency chain in self.why stack. @@ -781,11 +1341,12 @@ def _track_why( For all other phases, pushes the item onto the why stack and ensures it is popped even if an exception occurs. """ - if not item.phase.tracks_why: + if not item.tracks_why: yield return - assert item.resolved_version is not None - self.why.append((item.req_type, item.req, item.resolved_version)) + wi = item.work_item + assert wi.resolved_version is not None + self.why.append((wi.req_type, wi.req, wi.resolved_version)) try: yield finally: @@ -1444,111 +2005,31 @@ def _add_to_build_order( # converted to JSON without help. json.dump(self._build_stack, f, indent=2, default=str) - def _record_stack_state(self, stack: list[WorkItem]) -> None: + def _record_stack_state(self, stack: list[PhaseItem]) -> None: """Write the current bootstrap stack to `self._stack_filename`. Index 0 in the output corresponds to `stack[-1]`, the next item to be processed. Overwrites the file on each call. """ - - def serialize(item: WorkItem) -> dict[str, typing.Any]: - return { - "req": str(item.req), - "req_type": str(item.req_type), - "phase": str(item.phase), - "resolved_version": str(item.resolved_version) - if item.resolved_version is not None - else None, - "source_url": item.source_url, - "build_sdist_only": item.build_sdist_only, - "why": [ - {"req_type": str(rt), "req": str(r), "version": str(v)} - for rt, r, v in item.why_snapshot - ], - "parent": ( - {"req": str(item.parent[0]), "version": str(item.parent[1])} - if item.parent - else None - ), - "build_system_deps": sorted(str(r) for r in item.build_system_deps), - "build_backend_deps": sorted(str(r) for r in item.build_backend_deps), - "build_sdist_deps": sorted(str(r) for r in item.build_sdist_deps), - } - - records = [serialize(item) for item in reversed(stack)] + records = [item.as_json() for item in reversed(stack)] with open(self._stack_filename, "w") as f: json.dump(records, f, indent=2, default=str) - # ---- Iterative bootstrap: phase handlers and helpers ---- - - def _push_items(self, stack: list[WorkItem], items: list[WorkItem]) -> None: - """Push items onto the stack and submit background tasks in LIFO order. - - Submits the item that will be processed first (top of stack) to the - background pool first, maximising overlap between background I/O and - main-thread serial work. - """ - stack.extend(items) - if self._bg_pool is not None: - for item in reversed(items): - bg_work = self._get_background_work(item) - if bg_work is not None: - item.bg_future = self._bg_pool.submit(bg_work) - - def _get_background_work( - self, item: WorkItem - ) -> typing.Callable[[], typing.Any] | None: - """Return a zero-argument callable for background submission, or None. - - Uses closures that capture local variables rather than ``self``, so - the returned callable cannot access mutable ``Bootstrapper`` state. - Each closure sets up ``req_ctxvar_context`` so log messages include - the package name (and version when known). - """ - if item.phase == BootstrapPhase.RESOLVE: - bg_resolver = self._resolver - req = item.req - req_type = item.req_type - parent_req = item.why_snapshot[-1][1] if item.why_snapshot else None - return_all = self.multiple_versions - - def do_resolve() -> list[tuple[str, Version]]: - with req_ctxvar_context(req): - return _bg_resolve( - bg_resolver, req, req_type, parent_req, return_all - ) - - return do_resolve - - if item.phase == BootstrapPhase.PREPARE_SOURCE: - assert item.resolved_version is not None - assert item.source_url is not None - ctx = self.ctx - cache_wheel_server_url = self.cache_wheel_server_url - req = item.req - req_type = item.req_type - resolved_version = item.resolved_version - source_url = item.source_url - - if item.pbi_pre_built: - - def do_prepare_prebuilt() -> PreparedSourceData: - with req_ctxvar_context(req, resolved_version): - return _bg_prepare_prebuilt( - ctx, req, req_type, resolved_version, source_url - ) - - return do_prepare_prebuilt - - def do_prepare_source() -> PreparedSourceData: - with req_ctxvar_context(req, resolved_version): - return _bg_prepare_source( - ctx, cache_wheel_server_url, req, resolved_version, source_url - ) + # ---- Iterative bootstrap: phase handlers and helpers ---- - return do_prepare_source + def _push_items(self, stack: list[PhaseItem], items: list[PhaseItem]) -> None: + """Push items onto the stack and submit background tasks in LIFO order. - return None + Submits the item that will be processed first (top of stack) to the + background pool first, maximising overlap between background I/O and + main-thread serial work. + """ + stack.extend(items) + if self._bg_pool is not None: + for item in reversed(items): + bg_work = item.background_work(self) + if bg_work is not None: + item.bg_future = self._bg_pool.submit(bg_work) def _drain_background_pool(self) -> None: """Drain all in-flight background tasks and recreate the pool. @@ -1569,7 +2050,7 @@ def _create_unresolved_work_items( dep_req_type: RequirementType, parent_req: Requirement, parent_version: Version, - ) -> list[WorkItem]: + ) -> list[PhaseItem]: """Create RESOLVE-phase work items for dependencies. Called inside a parent's _track_why context so that why_snapshot @@ -1577,212 +2058,16 @@ def _create_unresolved_work_items( handling happen later when each item's RESOLVE phase runs. """ return [ - WorkItem( - req=dep, - req_type=dep_req_type, - phase=BootstrapPhase.RESOLVE, - why_snapshot=list(self.why), - parent=(parent_req, parent_version), - ) - for dep in self._sort_requirements(deps) - ] - - def _phase_resolve(self, item: WorkItem) -> list[WorkItem]: - """RESOLVE phase: resolve versions and expand into START-phase items. - - Centralizes version resolution so all dependencies are expanded - uniformly. In multiple_versions mode, filters out versions that - already failed in this run and versions whose wheels are already - cached to avoid redundant builds and transitive dependency - processing. - - Returns: - One START-phase item per resolved version that needs building. - Empty list if all versions are already cached. - """ - assert item.bg_future is not None - # bg_future.result() blocks until the background resolution completes, - # then returns the result or re-raises any exception from the background. - resolved_versions = item.bg_future.result() - if not resolved_versions: - raise RuntimeError(f"Could not resolve any versions for {item.req}") - - if self.multiple_versions: - pkg_name = canonicalize_name(item.req.name) - resolved_versions = [ - (url, ver) - for url, ver in resolved_versions - if (pkg_name, str(ver)) not in self._failed_versions - ] - if not resolved_versions: - raise RuntimeError( - f"Could not resolve any versions for {item.req}" - f" (all candidates failed previously)" - ) - - logger.info(f"resolved {len(resolved_versions)} version(s) for {item.req}") - filtered: list[tuple[str, Version]] = [] - for source_url, version in resolved_versions: - cached_wheel, _ = _find_cached_wheel( - self.ctx, self.cache_wheel_server_url, item.req, version - ) - if cached_wheel: - logger.info( - f"{item.req.name}=={version}: wheel already cached " - f"at {cached_wheel.name}, skipping" - ) - else: - filtered.append((source_url, version)) - if not filtered: - # Always process the highest version (first in - # resolved_versions) so new transitive dependencies - # are discovered even when every wheel is cached. - logger.info( - f"all versions of {item.req.name} already cached, " - f"keeping highest version {resolved_versions[0][1]} " - f"for dependency discovery" - ) - filtered.append(resolved_versions[0]) - resolved_versions = filtered - - # Build list so highest version ends up on top of the stack - # (last element after extend) and is processed first. - items: list[WorkItem] = [] - for source_url, version in reversed(resolved_versions): - items.append( + ResolveItem( WorkItem( - req=item.req, - req_type=item.req_type, - phase=BootstrapPhase.START, - why_snapshot=list(item.why_snapshot), - parent=item.parent, - source_url=source_url, - resolved_version=version, + req=dep, + req_type=dep_req_type, + why_snapshot=list(self.why), + parent=(parent_req, parent_version), ) ) - return items - - def _phase_start(self, item: WorkItem) -> list[WorkItem]: - """START phase: add to graph, check if already seen. - - _track_why is a no-op for this phase (tracks_why is False), - matching the original behavior where graph addition and - seen-check happen before pushing onto the why stack. - - Returns: - Empty list if already seen (nothing to do). - [item] advanced to PREPARE_SOURCE if this is new work. - """ - assert item.resolved_version is not None - assert item.source_url is not None - - # Add to graph (skip TOP_LEVEL, already added in _resolve_and_add_top_level) - if item.req_type != RequirementType.TOP_LEVEL: - self._add_to_graph( - item.req, - item.req_type, - item.resolved_version, - item.source_url, - item.parent, - ) - - item.build_sdist_only = ( - self.sdist_only and not self._processing_build_requirement(item.req_type) - ) - - if self._has_been_seen(item.req, item.resolved_version, item.build_sdist_only): - logger.debug( - f"redundant {item.req_type} dependency {item.req} " - f"({item.resolved_version}, sdist_only={item.build_sdist_only}) " - f"for {self._explain}" - ) - return [] - self._mark_as_seen(item.req, item.resolved_version, item.build_sdist_only) - - logger.info( - f"new {item.req_type} dependency {item.req} " - f"resolves to {item.resolved_version}" - ) - - item.pbi_pre_built = self.ctx.package_build_info(item.req).pre_built - item.phase = BootstrapPhase.PREPARE_SOURCE - return [item] - - def _phase_prepare_source(self, item: WorkItem) -> list[WorkItem]: - """PREPARE_SOURCE phase: download source or prebuilt, get build system deps. - - Uses background I/O result from ``item.bg_future`` when available, - falling back to inline I/O otherwise. - - Returns: - Prebuilt: [item] advanced to PROCESS_INSTALL_DEPS (skip build phases). - Source: [item advanced to PREPARE_BUILD, *build_system_dep_items]. - """ - assert item.resolved_version is not None - assert item.source_url is not None - - # bg_future is always set for PREPARE_SOURCE items (see _push_items / _get_background_work). - # bg_future.result() blocks until done and re-raises any background exception. - assert item.bg_future is not None - prepared: PreparedSourceData = item.bg_future.result() - - constraint = self.ctx.constraints.get_constraint(item.req.name) - if constraint: - logger.info( - f"incoming requirement {item.req} matches constraint " - f"{constraint}. Will apply both." - ) - - if item.pbi_pre_built: - # Background task already downloaded the prebuilt wheel - assert prepared.wheel_filename is not None - assert prepared.unpack_dir is not None - item.build_result = SourceBuildResult( - wheel_filename=prepared.wheel_filename, - sdist_filename=None, - unpack_dir=prepared.unpack_dir, - sdist_root_dir=None, - build_env=None, - source_type=SourceType.PREBUILT, - ) - item.phase = BootstrapPhase.PROCESS_INSTALL_DEPS - return [item] - - # Source build path: background task already downloaded and prepared the source - assert prepared.sdist_root_dir is not None - sdist_root_dir = prepared.sdist_root_dir - item.cached_wheel_filename = prepared.cached_wheel_filename - - assert sdist_root_dir is not None - - if sdist_root_dir.parent.parent != self.ctx.work_dir: - raise ValueError(f"'{sdist_root_dir}/../..' should be {self.ctx.work_dir}") - item.sdist_root_dir = sdist_root_dir - item.unpack_dir = sdist_root_dir.parent - - item.build_env = self._create_build_env( - req=item.req, - resolved_version=item.resolved_version, - parent_dir=sdist_root_dir.parent, - ) - - # Get build system dependencies - item.build_system_deps = dependencies.get_build_system_dependencies( - ctx=self.ctx, - req=item.req, - version=item.resolved_version, - sdist_root_dir=sdist_root_dir, - ) - - dep_items = self._create_unresolved_work_items( - item.build_system_deps, - RequirementType.BUILD_SYSTEM, - item.req, - item.resolved_version, - ) - - item.phase = BootstrapPhase.PREPARE_BUILD - return [item] + dep_items + for dep in self._sort_requirements(deps) + ] def _resolve_build_system_versions_by_name( self, @@ -1887,261 +2172,35 @@ def _filter_deps_satisfied_by_build_system( unsatisfied.add(dep) return unsatisfied - def _phase_prepare_build(self, item: WorkItem) -> list[WorkItem]: - """PREPARE_BUILD phase: install system deps, get backend/sdist deps. - - Build-backend and build-sdist dependencies that are already satisfied - by a resolved build-system dependency reuse that version instead of - resolving independently (see :issue:`1194`). - - Returns: - [item advanced to BUILD, *backend_dep_items, *sdist_dep_items]. - """ - assert item.resolved_version is not None - assert item.build_env is not None - assert item.sdist_root_dir is not None - - # Install build system deps (their wheels exist from DFS processing) - item.build_env.install(item.build_system_deps) - - # Get build backend dependencies - item.build_backend_deps = dependencies.get_build_backend_dependencies( - ctx=self.ctx, - req=item.req, - version=item.resolved_version, - sdist_root_dir=item.sdist_root_dir, - build_env=item.build_env, - ) - - # Get build sdist dependencies - item.build_sdist_deps = dependencies.get_build_sdist_dependencies( - ctx=self.ctx, - req=item.req, - version=item.resolved_version, - sdist_root_dir=item.sdist_root_dir, - build_env=item.build_env, - ) - - # Filter out deps already satisfied by build-system dependencies - # to avoid resolving to a different (typically newer) version. - resolved_build_sys = self._get_resolved_build_system_versions(item) - parent = (item.req, item.resolved_version) - item.build_backend_deps = self._filter_deps_satisfied_by_build_system( - item.build_backend_deps, - resolved_build_sys, - RequirementType.BUILD_BACKEND, - parent, - ) - item.build_sdist_deps = self._filter_deps_satisfied_by_build_system( - item.build_sdist_deps, - resolved_build_sys, - RequirementType.BUILD_SDIST, - parent, - ) - - backend_items = self._create_unresolved_work_items( - item.build_backend_deps, - RequirementType.BUILD_BACKEND, - item.req, - item.resolved_version, - ) - sdist_items = self._create_unresolved_work_items( - item.build_sdist_deps, - RequirementType.BUILD_SDIST, - item.req, - item.resolved_version, - ) - dep_items = backend_items + sdist_items - - item.phase = BootstrapPhase.BUILD - return [item] + dep_items - - def _phase_build(self, item: WorkItem) -> list[WorkItem]: - """BUILD phase: install remaining deps, build wheel/sdist. - - Returns: - [item] advanced to PROCESS_INSTALL_DEPS. - """ - assert item.resolved_version is not None - assert item.build_env is not None - assert item.sdist_root_dir is not None - - # Drain all in-flight background I/O before an exclusive build starts. - # This ensures no parallel background tasks interfere with a build that - # must run without resource contention. - pbi = self.ctx.package_build_info(item.req) - if pbi.exclusive_build: - logger.info( - "%s requires exclusive build, draining background pool", item.req - ) - self._drain_background_pool() - - # Install backend+sdist deps if disjoint from system deps - remaining_deps = item.build_backend_deps | item.build_sdist_deps - if remaining_deps.isdisjoint(item.build_system_deps): - item.build_env.install(remaining_deps) - - wheel_filename, sdist_filename = self._do_build( - req=item.req, - resolved_version=item.resolved_version, - sdist_root_dir=item.sdist_root_dir, - build_env=item.build_env, - build_sdist_only=item.build_sdist_only, - cached_wheel_filename=item.cached_wheel_filename, - ) - - source_type = sources.get_source_type(self.ctx, item.req) - - item.build_result = SourceBuildResult( - wheel_filename=wheel_filename, - sdist_filename=sdist_filename, - unpack_dir=item.sdist_root_dir.parent, - sdist_root_dir=item.sdist_root_dir, - build_env=item.build_env, - source_type=source_type, - ) - - item.phase = BootstrapPhase.PROCESS_INSTALL_DEPS - return [item] - - def _phase_process_install_deps(self, item: WorkItem) -> list[WorkItem]: - """PROCESS_INSTALL_DEPS phase: hooks, extract deps, build order. - - Returns: - [item advanced to COMPLETE, *install_dep_items]. - """ - assert item.resolved_version is not None - assert item.source_url is not None - assert item.build_result is not None - - # Run post-bootstrap hooks (non-fatal in test mode) - try: - hooks.run_post_bootstrap_hooks( - ctx=self.ctx, - req=item.req, - dist_name=canonicalize_name(item.req.name), - dist_version=str(item.resolved_version), - sdist_filename=item.build_result.sdist_filename, - wheel_filename=item.build_result.wheel_filename, - ) - except Exception as hook_error: - if not self.test_mode: - raise - self._record_test_mode_failure( - item.req, - str(item.resolved_version), - hook_error, - "hook", - "warning", - ) - - # Extract install dependencies (non-fatal in test mode) - try: - install_dependencies = self._get_install_dependencies( - req=item.req, - resolved_version=item.resolved_version, - wheel_filename=item.build_result.wheel_filename, - sdist_filename=item.build_result.sdist_filename, - sdist_root_dir=item.build_result.sdist_root_dir, - build_env=item.build_result.build_env, - unpack_dir=item.build_result.unpack_dir, - ) - except Exception as dep_error: - if not self.test_mode: - raise - self._record_test_mode_failure( - item.req, - str(item.resolved_version), - dep_error, - "dependency_extraction", - "warning", - ) - install_dependencies = [] - - logger.debug( - "install dependencies: %s", - ", ".join(sorted(str(r) for r in install_dependencies)), - ) - - pbi = self.ctx.package_build_info(item.req) - constraint = self.ctx.constraints.get_constraint(item.req.name) - self._add_to_build_order( - req=item.req, - version=item.resolved_version, - source_url=item.source_url, - source_type=item.build_result.source_type, - prebuilt=pbi.pre_built, - constraint=constraint, - ) - - dep_items = self._create_unresolved_work_items( - install_dependencies, - RequirementType.INSTALL, - item.req, - item.resolved_version, - ) - - item.phase = BootstrapPhase.COMPLETE - return [item] + dep_items - - def _phase_complete(self, item: WorkItem) -> list[WorkItem]: - """COMPLETE phase: clean up build directories. - - Returns: - Empty list (processing finished for this item). - """ - if item.build_result is not None: - self.ctx.clean_build_dirs( - item.build_result.sdist_root_dir, - item.build_result.build_env, - ) - return [] - - def _dispatch_phase(self, item: WorkItem) -> list[WorkItem]: - """Route a work item to the appropriate phase handler.""" - match item.phase: - case BootstrapPhase.RESOLVE: - return self._phase_resolve(item) - case BootstrapPhase.START: - return self._phase_start(item) - case BootstrapPhase.PREPARE_SOURCE: - return self._phase_prepare_source(item) - case BootstrapPhase.PREPARE_BUILD: - return self._phase_prepare_build(item) - case BootstrapPhase.BUILD: - return self._phase_build(item) - case BootstrapPhase.PROCESS_INSTALL_DEPS: - return self._phase_process_install_deps(item) - case BootstrapPhase.COMPLETE: - return self._phase_complete(item) - case _: - raise ValueError(f"unexpected phase: {item.phase}") - def _handle_phase_error( self, - item: WorkItem, + item: PhaseItem, err: Exception, - ) -> list[WorkItem]: + ) -> list[PhaseItem]: """Handle errors from phase processing. Returns work items to continue processing (e.g. prebuilt fallback), or empty list to skip this item. Raises in normal mode (fail-fast). """ + wi = item.work_item # Resolution failures: recoverable in test mode and multiple versions mode - if item.phase == BootstrapPhase.RESOLVE: + if isinstance(item, ResolveItem): if self.test_mode: - self._record_test_mode_failure(item.req, None, err, "resolution") + self._record_test_mode_failure(wi.req, None, err, "resolution") if self.multiple_versions: self._record_failed_version( - item.req, + wi.req, "unresolved", err, - f"failed during {item.phase} phase", + f"failed during {type(item).phase} phase", ) return [] if self.multiple_versions: self._record_failed_version( - item.req, "unresolved", err, f"failed during {item.phase} phase" + wi.req, + "unresolved", + err, + f"failed during {type(item).phase} phase", ) return [] raise @@ -2149,46 +2208,40 @@ def _handle_phase_error( # Test mode: try prebuilt fallback for build-related phases if self.test_mode: if ( - item.phase - in ( - BootstrapPhase.PREPARE_SOURCE, - BootstrapPhase.PREPARE_BUILD, - BootstrapPhase.BUILD, - ) - and not item.pbi_pre_built + isinstance(item, PrepareSourceItem | PrepareBuildItem | BuildItem) + and not wi.pbi_pre_built ): - assert item.resolved_version is not None + assert wi.resolved_version is not None fallback = self._handle_test_mode_failure( - req=item.req, - resolved_version=item.resolved_version, - req_type=item.req_type, + req=wi.req, + resolved_version=wi.resolved_version, + req_type=wi.req_type, build_error=err, ) if fallback is not None: - item.build_result = fallback - item.phase = BootstrapPhase.PROCESS_INSTALL_DEPS - return [item] + wi.build_result = fallback + return [ProcessInstallDepsItem(wi)] self._record_test_mode_failure( - item.req, str(item.resolved_version), err, "bootstrap" + wi.req, str(wi.resolved_version), err, "bootstrap" ) return [] # Multiple versions mode: record failure, remove from graph, continue if self.multiple_versions: - assert item.resolved_version is not None - pkg_name = canonicalize_name(item.req.name) + assert wi.resolved_version is not None + pkg_name = canonicalize_name(wi.req.name) self._record_failed_version( - item.req, - str(item.resolved_version), + wi.req, + str(wi.resolved_version), err, - f"failed during {item.phase} phase", + f"failed during {type(item).phase} phase", ) - self.ctx.dependency_graph.remove_dependency(pkg_name, item.resolved_version) + self.ctx.dependency_graph.remove_dependency(pkg_name, wi.resolved_version) self._seen_requirements.discard( - self._resolved_key(item.req, item.resolved_version, "sdist") + self._resolved_key(wi.req, wi.resolved_version, "sdist") ) self._seen_requirements.discard( - self._resolved_key(item.req, item.resolved_version, "wheel") + self._resolved_key(wi.req, wi.resolved_version, "wheel") ) self.ctx.write_to_graph_to_file() return [] diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index bc55f2f1..6c3ee483 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -265,19 +265,18 @@ def test_get_install_dependencies_returns_list( def test_phase_build_produces_source_build_result(tmp_context: WorkContext) -> None: - """Verify _phase_build produces a SourceBuildResult with correct values.""" + """Verify BuildItem.run() produces a SourceBuildResult with correct values.""" bt = bootstrapper.Bootstrapper(tmp_context) mock_sdist_root = tmp_context.work_dir / "package-1.0.0" / "package-1.0.0" mock_sdist_root.parent.mkdir(parents=True, exist_ok=True) mock_wheel = tmp_context.work_dir / "package-1.0.0-py3-none-any.whl" - item = bootstrapper.WorkItem( + wi = bootstrapper.WorkItem( req=Requirement("test-package"), req_type=RequirementType.TOP_LEVEL, source_url="https://pypi.org/simple/test-package", resolved_version=Version("1.0.0"), - phase=bootstrapper.BootstrapPhase.BUILD, why_snapshot=[], sdist_root_dir=mock_sdist_root, unpack_dir=mock_sdist_root.parent, @@ -286,6 +285,7 @@ def test_phase_build_produces_source_build_result(tmp_context: WorkContext) -> N build_backend_deps=set(), build_sdist_deps=set(), ) + item = bootstrapper.BuildItem(wi) # Set up why stack so _track_why works bt.why = [] @@ -295,12 +295,12 @@ def test_phase_build_produces_source_build_result(tmp_context: WorkContext) -> N patch.object(bt, "_build_wheel", return_value=(mock_wheel, None)), ): with bt._track_why(item): - result_items = bt._phase_build(item) + result_items = item.run(bt) assert len(result_items) == 1 - assert result_items[0].phase == bootstrapper.BootstrapPhase.PROCESS_INSTALL_DEPS + assert isinstance(result_items[0], bootstrapper.ProcessInstallDepsItem) - result = result_items[0].build_result + result = result_items[0].work_item.build_result assert isinstance(result, bootstrapper.SourceBuildResult) assert result.wheel_filename == mock_wheel assert result.sdist_filename is None @@ -323,25 +323,22 @@ def test_multiple_versions_continues_on_error(tmp_context: WorkContext) -> None: ("https://pypi.org/testpkg-1.0.tar.gz", Version("1.0")), ], ): - # Mock _dispatch_phase to let RESOLVE and START run normally - # but fail for version 1.5 in build phases. - original_dispatch = bt._dispatch_phase + # Let RESOLVE and START run normally, but intercept PrepareSource onwards. + # Fail for version 1.5; count all interceptions. build_phase_count = {"count": 0} - def mock_dispatch(item: bootstrapper.WorkItem) -> list[bootstrapper.WorkItem]: - if item.phase in ( - bootstrapper.BootstrapPhase.RESOLVE, - bootstrapper.BootstrapPhase.START, - ): - return original_dispatch(item) + def prepare_source_run( + self: bootstrapper.PrepareSourceItem, + bt_arg: bootstrapper.Bootstrapper, + ) -> list[bootstrapper.PhaseItem]: build_phase_count["count"] += 1 - if str(item.resolved_version) == "1.5": + if str(self.work_item.resolved_version) == "1.5": raise ValueError("Simulated failure for version 1.5") return [] req = Requirement("testpkg>=1.0") - with patch.object(bt, "_dispatch_phase", side_effect=mock_dispatch): + with patch.object(bootstrapper.PrepareSourceItem, "run", prepare_source_run): with patch.object(bt, "_has_been_seen", return_value=False): bt._bootstrap_one( req=req, @@ -567,18 +564,19 @@ def _make_resolve_item( req_type: RequirementType = RequirementType.TOP_LEVEL, why_snapshot: list[tuple[RequirementType, Requirement, Version]] | None = None, parent: tuple[Requirement, Version] | None = None, -) -> bootstrapper.WorkItem: - return bootstrapper.WorkItem( - req=Requirement(req), - req_type=req_type, - phase=bootstrapper.BootstrapPhase.RESOLVE, - why_snapshot=why_snapshot or [], - parent=parent, +) -> bootstrapper.ResolveItem: + return bootstrapper.ResolveItem( + bootstrapper.WorkItem( + req=Requirement(req), + req_type=req_type, + why_snapshot=why_snapshot or [], + parent=parent, + ) ) def _record_and_load( - bt: bootstrapper.Bootstrapper, stack: list[bootstrapper.WorkItem] + bt: bootstrapper.Bootstrapper, stack: list[bootstrapper.PhaseItem] ) -> list[typing.Any]: bt._record_stack_state(stack) return typing.cast(list[typing.Any], json.loads(bt._stack_filename.read_text())) @@ -610,18 +608,19 @@ def test_record_stack_state_full_item(tmp_context: WorkContext) -> None: parent_version = Version("2.0") why_snapshot = [(RequirementType.INSTALL, parent_req, parent_version)] - item = bootstrapper.WorkItem( - req=Requirement("child-pkg>=1.0"), - req_type=RequirementType.INSTALL, - phase=bootstrapper.BootstrapPhase.BUILD, - why_snapshot=why_snapshot, - parent=(parent_req, parent_version), - resolved_version=Version("1.5"), - source_url="https://pypi.test/child-pkg-1.5.tar.gz", - build_sdist_only=True, - build_system_deps={Requirement("setuptools")}, - build_backend_deps={Requirement("wheel")}, - build_sdist_deps={Requirement("flit-core")}, + item = bootstrapper.BuildItem( + bootstrapper.WorkItem( + req=Requirement("child-pkg>=1.0"), + req_type=RequirementType.INSTALL, + why_snapshot=why_snapshot, + parent=(parent_req, parent_version), + resolved_version=Version("1.5"), + source_url="https://pypi.test/child-pkg-1.5.tar.gz", + build_sdist_only=True, + build_system_deps={Requirement("setuptools")}, + build_backend_deps={Requirement("wheel")}, + build_sdist_deps={Requirement("flit-core")}, + ) ) contents = _record_and_load(bt, [item]) @@ -646,12 +645,17 @@ def test_record_stack_state_full_item(tmp_context: WorkContext) -> None: def test_record_stack_state_dep_sets_are_sorted(tmp_context: WorkContext) -> None: """Mixed-order dep sets come out alphabetically sorted.""" bt = bootstrapper.Bootstrapper(tmp_context) - item = bootstrapper.WorkItem( - req=Requirement("mypkg"), - req_type=RequirementType.TOP_LEVEL, - phase=bootstrapper.BootstrapPhase.BUILD, - why_snapshot=[], - build_system_deps={Requirement("zzz"), Requirement("aaa"), Requirement("mmm")}, + item = bootstrapper.BuildItem( + bootstrapper.WorkItem( + req=Requirement("mypkg"), + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + build_system_deps={ + Requirement("zzz"), + Requirement("aaa"), + Requirement("mmm"), + }, + ) ) contents = _record_and_load(bt, [item]) @@ -661,7 +665,10 @@ def test_record_stack_state_dep_sets_are_sorted(tmp_context: WorkContext) -> Non def test_record_stack_state_writes_file(tmp_context: WorkContext) -> None: """File is created; list length matches stack size.""" bt = bootstrapper.Bootstrapper(tmp_context) - stack = [_make_resolve_item("pkga"), _make_resolve_item("pkgb")] + stack: list[bootstrapper.PhaseItem] = [ + _make_resolve_item("pkga"), + _make_resolve_item("pkgb"), + ] bt._record_stack_state(stack) @@ -674,7 +681,7 @@ def test_record_stack_state_writes_file(tmp_context: WorkContext) -> None: def test_record_stack_state_ordering(tmp_context: WorkContext) -> None: """Index 0 = stack[-1] (next to pop); last index = stack[0].""" bt = bootstrapper.Bootstrapper(tmp_context) - stack = [ + stack: list[bootstrapper.PhaseItem] = [ _make_resolve_item("pkga"), _make_resolve_item("pkgb"), _make_resolve_item("pkgc"), @@ -709,7 +716,7 @@ def test_bootstrap_calls_record_stack_state(tmp_context: WorkContext) -> None: original = bt._record_stack_state - def counting_record(stack: list[bootstrapper.WorkItem]) -> None: + def counting_record(stack: list[bootstrapper.PhaseItem]) -> None: call_count["n"] += 1 original(stack) @@ -722,7 +729,7 @@ def counting_record(stack: list[bootstrapper.WorkItem]) -> None: "resolve", return_value=[("https://pypi.test/testpkg-1.0.tar.gz", Version("1.0"))], ), - patch.object(bt, "_phase_start", return_value=[]), + patch.object(bootstrapper.StartItem, "run", return_value=[]), ): bt._bootstrap_one(req=req, req_type=RequirementType.TOP_LEVEL) @@ -732,15 +739,22 @@ def counting_record(stack: list[bootstrapper.WorkItem]) -> None: def test_bootstrap_with_empty_list(tmp_context: WorkContext) -> None: """bootstrap([]) completes without error and runs no phases.""" bt = bootstrapper.Bootstrapper(tmp_context) - with patch.object(bt, "_dispatch_phase") as mock_dispatch: + with patch.object(bootstrapper.ResolveItem, "run") as mock_run: bt.bootstrap([]) - mock_dispatch.assert_not_called() + mock_run.assert_not_called() def test_bootstrap_with_single_requirement(tmp_context: WorkContext) -> None: """bootstrap([req]) resolves and processes the requirement.""" bt = bootstrapper.Bootstrapper(tmp_context) req = Requirement("testpkg==1.0") + captured: list[bootstrapper.ResolveItem] = [] + + def capture_run( + self: bootstrapper.ResolveItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[bootstrapper.PhaseItem]: + captured.append(self) + return [] with ( patch.object( @@ -748,16 +762,15 @@ def test_bootstrap_with_single_requirement(tmp_context: WorkContext) -> None: "_resolve_and_add_top_level", return_value=("http://example.test/testpkg-1.0.tar.gz", Version("1.0")), ), - patch.object(bt, "_dispatch_phase", return_value=[]) as mock_dispatch, + patch.object(bootstrapper.ResolveItem, "run", capture_run), patch.object(bt, "_record_stack_state"), ): bt.bootstrap([req]) - mock_dispatch.assert_called_once() - item = mock_dispatch.call_args[0][0] - assert item.req == req - assert item.req_type == RequirementType.TOP_LEVEL - assert item.phase == bootstrapper.BootstrapPhase.RESOLVE + assert len(captured) == 1 + assert isinstance(captured[0], bootstrapper.ResolveItem) + assert captured[0].work_item.req == req + assert captured[0].work_item.req_type == RequirementType.TOP_LEVEL def test_bootstrap_skips_failed_resolution(tmp_context: WorkContext) -> None: @@ -767,12 +780,12 @@ def test_bootstrap_skips_failed_resolution(tmp_context: WorkContext) -> None: with ( patch.object(bt, "_resolve_and_add_top_level", return_value=None), - patch.object(bt, "_dispatch_phase") as mock_dispatch, + patch.object(bootstrapper.ResolveItem, "run") as mock_run, patch.object(bt, "_record_stack_state"), ): bt.bootstrap([req]) - mock_dispatch.assert_not_called() + mock_run.assert_not_called() def test_bootstrap_two_requirements_both_processed(tmp_context: WorkContext) -> None: @@ -783,8 +796,10 @@ def test_bootstrap_two_requirements_both_processed(tmp_context: WorkContext) -> dispatch_calls: list = [] - def fake_dispatch(item: bootstrapper.WorkItem) -> list: - dispatch_calls.append(item.req.name) + def capture_run( + self: bootstrapper.ResolveItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[bootstrapper.PhaseItem]: + dispatch_calls.append(self.work_item.req.name) return [] with ( @@ -793,7 +808,7 @@ def fake_dispatch(item: bootstrapper.WorkItem) -> list: "_resolve_and_add_top_level", return_value=("http://example.test/pkg-1.0.tar.gz", Version("1.0")), ), - patch.object(bt, "_dispatch_phase", side_effect=fake_dispatch), + patch.object(bootstrapper.ResolveItem, "run", capture_run), patch.object(bt, "_record_stack_state"), ): bt.bootstrap([req1, req2]) diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index d4e75c52..de4840a8 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -1,18 +1,17 @@ """Tests for the iterative bootstrap implementation. Tests cover: -- BootstrapPhase enum and tracks_why property +- PhaseItem class hierarchy: class variables and base class behavior - WorkItem dataclass defaults and state accumulation - _track_why context manager behavior - _create_unresolved_work_items helper -- _phase_resolve version expansion -- _phase_start graph addition and seen-check -- _phase_prepare_source all branches (prebuilt, source, cached, bad path) -- _phase_prepare_build dep installation and extraction -- _phase_build conditional install and result construction -- _phase_process_install_deps hooks, dep extraction, error modes -- _phase_complete cleanup -- _dispatch_phase routing +- ResolveItem.run() version expansion +- StartItem.run() graph addition and seen-check +- PrepareSourceItem.run() all branches (prebuilt, source, cached, bad path) +- PrepareBuildItem.run() dep installation and extraction +- BuildItem.run() conditional install and result construction +- ProcessInstallDepsItem.run() hooks, dep extraction, error modes +- CompleteItem.run() cleanup - _handle_phase_error for all three error modes - End-to-end iterative loop with LIFO ordering """ @@ -30,7 +29,19 @@ from packaging.version import Version from fromager import bootstrapper, build_environment -from fromager.bootstrapper import BootstrapPhase, SourceBuildResult, WorkItem +from fromager.bootstrapper import ( + BootstrapPhase, + BuildItem, + CompleteItem, + PhaseItem, + PrepareBuildItem, + PrepareSourceItem, + ProcessInstallDepsItem, + ResolveItem, + SourceBuildResult, + StartItem, + WorkItem, +) from fromager.context import WorkContext from fromager.requirements_file import RequirementType, SourceType @@ -44,18 +55,59 @@ def _make_resolved_future( return future -def _make_resolve_item( +def _make_work_item( req: str = "testpkg", req_type: RequirementType = RequirementType.INSTALL, why_snapshot: list | None = None, parent: tuple | None = None, + source_url: str | None = None, + version: str | None = None, + build_env: build_environment.BuildEnvironment | None = None, + sdist_root_dir: pathlib.Path | None = None, + unpack_dir: pathlib.Path | None = None, + build_result: SourceBuildResult | None = None, + build_system_deps: set[Requirement] | None = None, + build_backend_deps: set[Requirement] | None = None, + build_sdist_deps: set[Requirement] | None = None, + pbi_pre_built: bool = False, + cached_wheel_filename: pathlib.Path | None = None, + build_sdist_only: bool = False, ) -> WorkItem: return WorkItem( req=Requirement(req), req_type=req_type, - phase=BootstrapPhase.RESOLVE, why_snapshot=why_snapshot or [], parent=parent, + source_url=source_url, + resolved_version=Version(version) if version else None, + build_env=build_env, + sdist_root_dir=sdist_root_dir, + unpack_dir=unpack_dir, + build_result=build_result, + build_system_deps=build_system_deps if build_system_deps is not None else set(), + build_backend_deps=build_backend_deps + if build_backend_deps is not None + else set(), + build_sdist_deps=build_sdist_deps if build_sdist_deps is not None else set(), + pbi_pre_built=pbi_pre_built, + cached_wheel_filename=cached_wheel_filename, + build_sdist_only=build_sdist_only, + ) + + +def _make_resolve_item( + req: str = "testpkg", + req_type: RequirementType = RequirementType.INSTALL, + why_snapshot: list | None = None, + parent: tuple | None = None, +) -> ResolveItem: + return ResolveItem( + WorkItem( + req=Requirement(req), + req_type=req_type, + why_snapshot=why_snapshot or [], + parent=parent, + ) ) @@ -66,18 +118,30 @@ def _make_start_item( version: str = "1.0", why_snapshot: list | None = None, parent: tuple | None = None, -) -> WorkItem: - return WorkItem( - req=Requirement(req), - req_type=req_type, - phase=BootstrapPhase.START, - why_snapshot=why_snapshot or [], - parent=parent, - source_url=source_url, - resolved_version=Version(version), +) -> StartItem: + return StartItem( + WorkItem( + req=Requirement(req), + req_type=req_type, + why_snapshot=why_snapshot or [], + parent=parent, + source_url=source_url, + resolved_version=Version(version), + ) ) +_PHASE_TO_CLASS: dict[BootstrapPhase, type[PhaseItem]] = { + BootstrapPhase.RESOLVE: ResolveItem, + BootstrapPhase.START: StartItem, + BootstrapPhase.PREPARE_SOURCE: PrepareSourceItem, + BootstrapPhase.PREPARE_BUILD: PrepareBuildItem, + BootstrapPhase.BUILD: BuildItem, + BootstrapPhase.PROCESS_INSTALL_DEPS: ProcessInstallDepsItem, + BootstrapPhase.COMPLETE: CompleteItem, +} + + def _make_build_item( req: str = "testpkg", version: str = "1.0", @@ -93,11 +157,10 @@ def _make_build_item( pbi_pre_built: bool = False, cached_wheel_filename: pathlib.Path | None = None, build_sdist_only: bool = False, -) -> WorkItem: - return WorkItem( +) -> PhaseItem: + wi = WorkItem( req=Requirement(req), req_type=RequirementType.INSTALL, - phase=phase, why_snapshot=[], source_url=source_url, resolved_version=Version(version), @@ -114,48 +177,75 @@ def _make_build_item( cached_wheel_filename=cached_wheel_filename, build_sdist_only=build_sdist_only, ) + return _PHASE_TO_CLASS[phase](wi) -class TestBootstrapPhase: - def test_tracks_why_false_for_resolve(self) -> None: - assert BootstrapPhase.RESOLVE.tracks_why is False +class TestPhaseItemClassVariables: + """Verify class-variable declarations on each PhaseItem subclass.""" - def test_tracks_why_false_for_start(self) -> None: - assert BootstrapPhase.START.tracks_why is False + @pytest.mark.parametrize( + "cls, expected_phase, expected_tracks_why", + [ + (ResolveItem, BootstrapPhase.RESOLVE, False), + (StartItem, BootstrapPhase.START, False), + (PrepareSourceItem, BootstrapPhase.PREPARE_SOURCE, True), + (PrepareBuildItem, BootstrapPhase.PREPARE_BUILD, True), + (BuildItem, BootstrapPhase.BUILD, True), + (ProcessInstallDepsItem, BootstrapPhase.PROCESS_INSTALL_DEPS, True), + (CompleteItem, BootstrapPhase.COMPLETE, True), + ], + ) + def test_class_variables( + self, + tmp_context: WorkContext, + cls: type[PhaseItem], + expected_phase: BootstrapPhase, + expected_tracks_why: bool, + ) -> None: + assert cls.phase == expected_phase + assert cls.tracks_why == expected_tracks_why - def test_tracks_why_true_for_build_phases(self) -> None: - for phase in ( - BootstrapPhase.PREPARE_SOURCE, - BootstrapPhase.PREPARE_BUILD, - BootstrapPhase.BUILD, - BootstrapPhase.PROCESS_INSTALL_DEPS, - BootstrapPhase.COMPLETE, - ): - assert phase.tracks_why is True, f"{phase} should track why" + def test_str_default(self) -> None: + wi = _make_work_item(req="mypkg") + item = BuildItem(wi) + assert str(item) == "BuildItem(mypkg)" + + def test_background_work_returns_none_by_default( + self, tmp_context: WorkContext + ) -> None: + bt = bootstrapper.Bootstrapper(tmp_context) + for cls in (PrepareBuildItem, BuildItem, ProcessInstallDepsItem, CompleteItem): + wi = _make_work_item(req="testpkg", version="1.0") + item = cls(wi) + assert item.background_work(bt) is None, ( + f"{cls.__name__}.background_work() should return None" + ) class TestWorkItem: def test_defaults_for_resolve_item(self) -> None: item = _make_resolve_item() - assert item.source_url is None - assert item.resolved_version is None - assert item.build_sdist_only is False - assert item.build_env is None - assert item.build_result is None - assert item.pbi_pre_built is False - assert item.build_system_deps == set() - assert item.build_backend_deps == set() - assert item.build_sdist_deps == set() + wi = item.work_item + assert wi.source_url is None + assert wi.resolved_version is None + assert wi.build_sdist_only is False + assert wi.build_env is None + assert wi.build_result is None + assert wi.pbi_pre_built is False + assert wi.build_system_deps == set() + assert wi.build_backend_deps == set() + assert wi.build_sdist_deps == set() def test_state_accumulation(self) -> None: item = _make_start_item() - item.pbi_pre_built = True - item.build_sdist_only = True + wi = item.work_item + wi.pbi_pre_built = True + wi.build_sdist_only = True mock_env = Mock() - item.build_env = mock_env - assert item.pbi_pre_built is True - assert item.build_sdist_only is True - assert item.build_env is mock_env + wi.build_env = mock_env + assert wi.pbi_pre_built is True + assert wi.build_sdist_only is True + assert wi.build_env is mock_env class TestTrackWhy: @@ -183,11 +273,12 @@ def test_pushes_and_pops_for_build_phase(self, tmp_context: WorkContext) -> None bt = bootstrapper.Bootstrapper(tmp_context) bt.why = [] item = _make_build_item(phase=BootstrapPhase.PREPARE_SOURCE) + wi = item.work_item with bt._track_why(item): assert len(bt.why) == 1 - assert bt.why[0][1] == item.req - assert bt.why[0][2] == item.resolved_version + assert bt.why[0][1] == wi.req + assert bt.why[0][2] == wi.resolved_version assert len(bt.why) == 0 @@ -215,11 +306,12 @@ def test_creates_resolve_phase_items(self, tmp_context: WorkContext) -> None: assert len(items) == 2 for item in items: + assert isinstance(item, ResolveItem) assert item.phase == BootstrapPhase.RESOLVE - assert item.req_type == RequirementType.BUILD_SYSTEM - assert item.parent == (Requirement("parent"), Version("1.0")) - assert item.source_url is None - assert item.resolved_version is None + assert item.work_item.req_type == RequirementType.BUILD_SYSTEM + assert item.work_item.parent == (Requirement("parent"), Version("1.0")) + assert item.work_item.source_url is None + assert item.work_item.resolved_version is None def test_captures_why_snapshot(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) @@ -233,10 +325,10 @@ def test_captures_why_snapshot(self, tmp_context: WorkContext) -> None: ) assert len(items) == 1 - assert items[0].why_snapshot == bt.why + assert items[0].work_item.why_snapshot == bt.why # Verify it's a copy, not a reference bt.why.append((RequirementType.INSTALL, Requirement("other"), Version("3.0"))) - assert len(items[0].why_snapshot) == 1 + assert len(items[0].work_item.why_snapshot) == 1 def test_sorts_by_name(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) @@ -246,7 +338,7 @@ def test_sorts_by_name(self, tmp_context: WorkContext) -> None: deps, RequirementType.INSTALL, Requirement("p"), Version("1.0") ) - names = [str(item.req.name) for item in items] + names = [str(item.work_item.req.name) for item in items] assert names == ["alpha", "middle", "zebra"] def test_empty_deps(self, tmp_context: WorkContext) -> None: @@ -261,19 +353,19 @@ class TestPhaseResolve: def test_single_version(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) item = _make_resolve_item() - parent = (Requirement("parent"), Version("2.0")) - item.parent = parent + wi = item.work_item + wi.parent = (Requirement("parent"), Version("2.0")) item.bg_future = _make_resolved_future( [("https://pypi.org/testpkg-1.0.tar.gz", Version("1.0"))] ) - result = bt._phase_resolve(item) + result = item.run(bt) assert len(result) == 1 - assert result[0].phase == BootstrapPhase.START - assert result[0].source_url == "https://pypi.org/testpkg-1.0.tar.gz" - assert result[0].resolved_version == Version("1.0") - assert result[0].parent == parent + assert isinstance(result[0], StartItem) + assert result[0].work_item.source_url == "https://pypi.org/testpkg-1.0.tar.gz" + assert result[0].work_item.resolved_version == Version("1.0") + assert result[0].work_item.parent == wi.parent def test_multiple_versions(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) @@ -285,12 +377,12 @@ def test_multiple_versions(self, tmp_context: WorkContext) -> None: ] ) - result = bt._phase_resolve(item) + result = item.run(bt) assert len(result) == 2 # Reversed so highest version ends up on top of stack (last element) - assert result[0].resolved_version == Version("1.0") - assert result[1].resolved_version == Version("2.0") + assert result[0].work_item.resolved_version == Version("1.0") + assert result[1].work_item.resolved_version == Version("2.0") def test_empty_resolution_raises(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) @@ -298,7 +390,7 @@ def test_empty_resolution_raises(self, tmp_context: WorkContext) -> None: item.bg_future = _make_resolved_future([]) with pytest.raises(RuntimeError, match="Could not resolve"): - bt._phase_resolve(item) + item.run(bt) def test_preserves_why_snapshot(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) @@ -306,14 +398,14 @@ def test_preserves_why_snapshot(self, tmp_context: WorkContext) -> None: item = _make_resolve_item(why_snapshot=list(snapshot)) item.bg_future = _make_resolved_future([("url", Version("1.0"))]) - result = bt._phase_resolve(item) + result = item.run(bt) - assert result[0].why_snapshot == snapshot + assert result[0].work_item.why_snapshot == snapshot def test_filters_cached_versions_in_multiple_versions_mode( self, tmp_context: WorkContext ) -> None: - """Cached versions are filtered out before creating START items.""" + """Cached versions are filtered out before creating StartItems.""" bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) item = _make_resolve_item() item.bg_future = _make_resolved_future( @@ -332,10 +424,10 @@ def mock_cache( return (None, None) with patch("fromager.bootstrapper._find_cached_wheel", side_effect=mock_cache): - result = bt._phase_resolve(item) + result = item.run(bt) assert len(result) == 2 - versions = {str(it.resolved_version) for it in result} + versions = {str(it.work_item.resolved_version) for it in result} assert versions == {"1.0", "3.0"} def test_all_cached_keeps_highest_version(self, tmp_context: WorkContext) -> None: @@ -354,10 +446,10 @@ def test_all_cached_keeps_highest_version(self, tmp_context: WorkContext) -> Non "fromager.bootstrapper._find_cached_wheel", return_value=(tmp_context.work_dir / "cached.whl", None), ): - result = bt._phase_resolve(item) + result = item.run(bt) assert len(result) == 1 - assert result[0].resolved_version == Version("3.0") + assert result[0].work_item.resolved_version == Version("3.0") def test_no_filtering_in_single_version_mode( self, tmp_context: WorkContext @@ -368,7 +460,7 @@ def test_no_filtering_in_single_version_mode( item.bg_future = _make_resolved_future([("url-1.0", Version("1.0"))]) with patch("fromager.bootstrapper._find_cached_wheel") as mock_cache: - result = bt._phase_resolve(item) + result = item.run(bt) assert len(result) == 1 mock_cache.assert_not_called() @@ -383,12 +475,12 @@ def test_empty_resolution_raises_runtime_error( item.bg_future = _make_resolved_future([]) with pytest.raises(RuntimeError, match="Could not resolve"): - bt._phase_resolve(item) + item.run(bt) def test_filters_failed_versions_in_multiple_versions_mode( self, tmp_context: WorkContext ) -> None: - """Previously failed versions are excluded before creating START items.""" + """Previously failed versions are excluded before creating StartItems.""" bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) item = _make_resolve_item() @@ -406,9 +498,9 @@ def test_filters_failed_versions_in_multiple_versions_mode( with patch( "fromager.bootstrapper._find_cached_wheel", return_value=(None, None) ): - result = bt._phase_resolve(item) + result = item.run(bt) - versions = {str(it.resolved_version) for it in result} + versions = {str(it.work_item.resolved_version) for it in result} assert versions == {"1.0", "3.0"} def test_failed_version_filter_does_not_apply_in_single_version_mode( @@ -423,10 +515,10 @@ def test_failed_version_filter_does_not_apply_in_single_version_mode( ) item.bg_future = _make_resolved_future([("url-1.0", Version("1.0"))]) - result = bt._phase_resolve(item) + result = item.run(bt) assert len(result) == 1 - assert result[0].resolved_version == Version("1.0") + assert result[0].work_item.resolved_version == Version("1.0") def test_all_versions_failed_raises_runtime_error( self, tmp_context: WorkContext @@ -441,10 +533,10 @@ def test_all_versions_failed_raises_runtime_error( item.bg_future = _make_resolved_future([("url-1.0", Version("1.0"))]) with pytest.raises(RuntimeError, match="failed previously"): - bt._phase_resolve(item) + item.run(bt) def test_bg_future_exception_propagates(self, tmp_context: WorkContext) -> None: - """Exceptions from the background resolver thread are surfaced by _phase_resolve.""" + """Exceptions from the background resolver thread are surfaced by run().""" bt = bootstrapper.Bootstrapper(tmp_context) item = _make_resolve_item() future: concurrent.futures.Future[list[tuple[str, Version]]] = ( @@ -454,7 +546,7 @@ def test_bg_future_exception_propagates(self, tmp_context: WorkContext) -> None: item.bg_future = future with pytest.raises(ValueError, match="resolver exploded"): - bt._phase_resolve(item) + item.run(bt) class TestPhaseStart: @@ -465,22 +557,23 @@ def test_new_item_advances_to_prepare_source( bt.why = [] item = _make_start_item() - result = bt._phase_start(item) + result = item.run(bt) assert len(result) == 1 - assert result[0].phase == BootstrapPhase.PREPARE_SOURCE - assert result[0] is item + assert isinstance(result[0], PrepareSourceItem) + assert result[0].work_item is item.work_item def test_already_seen_returns_empty(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) bt.why = [] item = _make_start_item() + wi = item.work_item # Mark as seen first - assert item.resolved_version is not None - bt._mark_as_seen(item.req, item.resolved_version) + assert wi.resolved_version is not None + bt._mark_as_seen(wi.req, wi.resolved_version) - result = bt._phase_start(item) + result = item.run(bt) assert result == [] @@ -489,7 +582,7 @@ def test_adds_to_graph_for_non_toplevel(self, tmp_context: WorkContext) -> None: bt.why = [] item = _make_start_item(req_type=RequirementType.INSTALL) - bt._phase_start(item) + item.run(bt) key = f"{canonicalize_name('testpkg')}==1.0" assert key in tmp_context.dependency_graph.nodes @@ -499,7 +592,7 @@ def test_skips_graph_for_toplevel(self, tmp_context: WorkContext) -> None: bt.why = [] item = _make_start_item(req_type=RequirementType.TOP_LEVEL) - bt._phase_start(item) + item.run(bt) key = f"{canonicalize_name('testpkg')}==1.0" assert key not in tmp_context.dependency_graph.nodes @@ -511,9 +604,9 @@ def test_sdist_only_set_for_non_build_requirement( bt.why = [] item = _make_start_item(req_type=RequirementType.INSTALL) - bt._phase_start(item) + item.run(bt) - assert item.build_sdist_only is True + assert item.work_item.build_sdist_only is True def test_sdist_only_not_set_for_build_requirement( self, tmp_context: WorkContext @@ -522,22 +615,42 @@ def test_sdist_only_not_set_for_build_requirement( bt.why = [] item = _make_start_item(req_type=RequirementType.BUILD_SYSTEM) - bt._phase_start(item) + item.run(bt) - assert item.build_sdist_only is False + assert item.work_item.build_sdist_only is False def test_marks_as_seen(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) bt.why = [] item = _make_start_item() - assert item.resolved_version is not None + wi = item.work_item + assert wi.resolved_version is not None - assert not bt._has_been_seen(item.req, item.resolved_version) - bt._phase_start(item) - assert bt._has_been_seen(item.req, item.resolved_version) + assert not bt._has_been_seen(wi.req, wi.resolved_version) + item.run(bt) + assert bt._has_been_seen(wi.req, wi.resolved_version) + def test_sets_pbi_pre_built_before_prepare_source( + self, tmp_context: WorkContext + ) -> None: + """pbi_pre_built is set on work_item before PrepareSourceItem is constructed.""" + bt = bootstrapper.Bootstrapper(tmp_context) + bt.why = [] + item = _make_start_item() + + with patch.object( + tmp_context, + "package_build_info", + return_value=Mock(pre_built=True), + ): + result = item.run(bt) + + assert len(result) == 1 + assert isinstance(result[0], PrepareSourceItem) + assert result[0].work_item.pbi_pre_built is True -class TestPhaseComplete: + +class TestCompleteItem: def test_calls_clean_build_dirs(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) mock_sdist_root = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" @@ -551,10 +664,10 @@ def test_calls_clean_build_dirs(self, tmp_context: WorkContext) -> None: source_type=SourceType.SDIST, ) item = _make_build_item(phase=BootstrapPhase.COMPLETE) - item.build_result = build_result + item.work_item.build_result = build_result with patch.object(tmp_context, "clean_build_dirs") as mock_clean: - result = bt._phase_complete(item) + result = item.run(bt) assert result == [] mock_clean.assert_called_once_with(mock_sdist_root, mock_env) @@ -562,40 +675,18 @@ def test_calls_clean_build_dirs(self, tmp_context: WorkContext) -> None: def test_no_build_result_skips_cleanup(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) item = _make_build_item(phase=BootstrapPhase.COMPLETE) - item.build_result = None + item.work_item.build_result = None with patch.object(tmp_context, "clean_build_dirs") as mock_clean: - result = bt._phase_complete(item) + result = item.run(bt) assert result == [] mock_clean.assert_not_called() - -class TestDispatchPhase: - @pytest.mark.parametrize( - "phase,method_name", - [ - (BootstrapPhase.RESOLVE, "_phase_resolve"), - (BootstrapPhase.START, "_phase_start"), - (BootstrapPhase.PREPARE_SOURCE, "_phase_prepare_source"), - (BootstrapPhase.PREPARE_BUILD, "_phase_prepare_build"), - (BootstrapPhase.BUILD, "_phase_build"), - (BootstrapPhase.PROCESS_INSTALL_DEPS, "_phase_process_install_deps"), - (BootstrapPhase.COMPLETE, "_phase_complete"), - ], - ) - def test_routes_to_correct_handler( - self, tmp_context: WorkContext, phase: BootstrapPhase, method_name: str - ) -> None: + def test_returns_empty_list(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) - item = _make_build_item(phase=phase) - expected = [item] - - with patch.object(bt, method_name, return_value=expected) as mock_method: - result = bt._dispatch_phase(item) - - assert result is expected - mock_method.assert_called_once_with(item) + item = _make_build_item(phase=BootstrapPhase.COMPLETE) + assert item.run(bt) == [] class TestHandlePhaseError: @@ -654,7 +745,7 @@ def test_build_phase_test_mode_fallback_success( ) -> None: bt = bootstrapper.Bootstrapper(tmp_context, test_mode=True) item = _make_build_item(phase=BootstrapPhase.PREPARE_SOURCE) - item.pbi_pre_built = False + item.work_item.pbi_pre_built = False err = RuntimeError("build failed") mock_fallback = Mock(spec=SourceBuildResult) @@ -662,9 +753,9 @@ def test_build_phase_test_mode_fallback_success( result = bt._handle_phase_error(item, err) assert len(result) == 1 - assert result[0] is item - assert item.build_result is mock_fallback - assert item.phase == BootstrapPhase.PROCESS_INSTALL_DEPS + assert isinstance(result[0], ProcessInstallDepsItem) + assert result[0].work_item is item.work_item + assert result[0].work_item.build_result is mock_fallback assert len(bt.failed_packages) == 0 def test_build_phase_test_mode_fallback_failure( @@ -672,7 +763,7 @@ def test_build_phase_test_mode_fallback_failure( ) -> None: bt = bootstrapper.Bootstrapper(tmp_context, test_mode=True) item = _make_build_item(phase=BootstrapPhase.BUILD) - item.pbi_pre_built = False + item.work_item.pbi_pre_built = False err = RuntimeError("build failed") with patch.object(bt, "_handle_test_mode_failure", return_value=None): @@ -687,7 +778,7 @@ def test_build_phase_test_mode_prebuilt_skips_fallback( ) -> None: bt = bootstrapper.Bootstrapper(tmp_context, test_mode=True) item = _make_build_item(phase=BootstrapPhase.PREPARE_SOURCE) - item.pbi_pre_built = True + item.work_item.pbi_pre_built = True err = RuntimeError("download failed") result = bt._handle_phase_error(item, err) @@ -716,8 +807,9 @@ def test_multiple_versions_records_and_removes_from_graph( ) -> None: bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) item = _make_build_item(phase=BootstrapPhase.BUILD) - assert item.resolved_version is not None - assert item.source_url is not None + wi = item.work_item + assert wi.resolved_version is not None + assert wi.source_url is not None err = ValueError("build failed") # Add to graph first so remove_dependency has something to remove @@ -725,13 +817,13 @@ def test_multiple_versions_records_and_removes_from_graph( parent_name=None, parent_version=None, req_type=RequirementType.TOP_LEVEL, - req=item.req, - req_version=item.resolved_version, - download_url=item.source_url, + req=wi.req, + req_version=wi.resolved_version, + download_url=wi.source_url, pre_built=False, ) # Mark as seen - bt._mark_as_seen(item.req, item.resolved_version) + bt._mark_as_seen(wi.req, wi.resolved_version) result = bt._handle_phase_error(item, err) @@ -743,8 +835,8 @@ def test_multiple_versions_records_and_removes_from_graph( key = f"{canonicalize_name('testpkg')}==1.0" assert key not in tmp_context.dependency_graph.nodes # Seen markers cleared - assert not bt._has_been_seen(item.req, item.resolved_version) - assert not bt._has_been_seen(item.req, item.resolved_version, sdist_only=True) + assert not bt._has_been_seen(wi.req, wi.resolved_version) + assert not bt._has_been_seen(wi.req, wi.resolved_version, sdist_only=True) def test_multiple_versions_logs_phase( self, tmp_context: WorkContext, caplog: pytest.LogCaptureFixture @@ -777,45 +869,72 @@ def test_full_lifecycle_source_package(self, tmp_context: WorkContext) -> None: """Drive a package through RESOLVE -> START -> ... -> COMPLETE.""" bt = bootstrapper.Bootstrapper(tmp_context) - # Track which phases are visited phases_visited: list[BootstrapPhase] = [] - original_dispatch = bt._dispatch_phase - - def tracking_dispatch(item: WorkItem) -> list[WorkItem]: - phases_visited.append(item.phase) - if item.phase == BootstrapPhase.RESOLVE: - return original_dispatch(item) - if item.phase == BootstrapPhase.START: - return original_dispatch(item) - if item.phase == BootstrapPhase.PREPARE_SOURCE: - # Skip actual source download, simulate source build path - item.phase = BootstrapPhase.PREPARE_BUILD - item.build_env = Mock() - item.sdist_root_dir = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" - return [item] - if item.phase == BootstrapPhase.PREPARE_BUILD: - item.phase = BootstrapPhase.BUILD - return [item] - if item.phase == BootstrapPhase.BUILD: - item.build_result = SourceBuildResult( - wheel_filename=None, - sdist_filename=None, - unpack_dir=tmp_context.work_dir, - sdist_root_dir=None, - build_env=None, - source_type=SourceType.SDIST, - ) - item.phase = BootstrapPhase.PROCESS_INSTALL_DEPS - return [item] - if item.phase == BootstrapPhase.PROCESS_INSTALL_DEPS: - item.phase = BootstrapPhase.COMPLETE - return [item] - if item.phase == BootstrapPhase.COMPLETE: - return [] + original_resolve_run = ResolveItem.run + original_start_run = StartItem.run + + def resolve_run( + self: ResolveItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: + phases_visited.append(type(self).phase) + return original_resolve_run(self, bt_arg) + + def start_run( + self: StartItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: + phases_visited.append(type(self).phase) + return original_start_run(self, bt_arg) + + def prepare_source_run( + self: PrepareSourceItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: + phases_visited.append(type(self).phase) + wi = self.work_item + wi.build_env = Mock() + wi.sdist_root_dir = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + return [PrepareBuildItem(wi)] + + def prepare_build_run( + self: PrepareBuildItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: + phases_visited.append(type(self).phase) + return [BuildItem(self.work_item)] + + def build_run( + self: BuildItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: + phases_visited.append(type(self).phase) + wi = self.work_item + wi.build_result = SourceBuildResult( + wheel_filename=None, + sdist_filename=None, + unpack_dir=tmp_context.work_dir, + sdist_root_dir=None, + build_env=None, + source_type=SourceType.SDIST, + ) + return [ProcessInstallDepsItem(wi)] + + def process_install_deps_run( + self: ProcessInstallDepsItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: + phases_visited.append(type(self).phase) + return [CompleteItem(self.work_item)] + + def complete_run( + self: CompleteItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: + phases_visited.append(type(self).phase) return [] with ( - patch.object(bt, "_dispatch_phase", side_effect=tracking_dispatch), + patch.object(ResolveItem, "run", resolve_run), + patch.object(StartItem, "run", start_run), + patch.object(PrepareSourceItem, "run", prepare_source_run), + patch.object(PrepareBuildItem, "run", prepare_build_run), + patch.object(BuildItem, "run", build_run), + patch.object(ProcessInstallDepsItem, "run", process_install_deps_run), + patch.object(CompleteItem, "run", complete_run), patch.object( bt._resolver, "resolve", @@ -841,31 +960,51 @@ def test_lifo_ordering_deps_before_continuation( bt = bootstrapper.Bootstrapper(tmp_context) processing_order: list[tuple[str, str]] = [] - original_dispatch = bt._dispatch_phase + original_resolve_run = ResolveItem.run + original_start_run = StartItem.run + + def resolve_run( + self: ResolveItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: + processing_order.append( + (str(self.work_item.req.name), str(type(self).phase)) + ) + return original_resolve_run(self, bt_arg) - def tracking_dispatch(item: WorkItem) -> list[WorkItem]: - phase_name = str(item.phase) - req_name = str(item.req.name) - processing_order.append((req_name, phase_name)) + def start_run( + self: StartItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: + processing_order.append( + (str(self.work_item.req.name), str(type(self).phase)) + ) + return original_start_run(self, bt_arg) - if item.phase == BootstrapPhase.RESOLVE: - return original_dispatch(item) - if item.phase == BootstrapPhase.START: - return original_dispatch(item) - - # Parent discovers a dep at PREPARE_SOURCE - if req_name == "parent" and item.phase == BootstrapPhase.PREPARE_SOURCE: - assert item.resolved_version is not None - dep_item = WorkItem( - req=Requirement("child"), - req_type=RequirementType.BUILD_SYSTEM, - phase=BootstrapPhase.RESOLVE, - why_snapshot=[], - parent=(item.req, item.resolved_version), + def prepare_source_run( + self: PrepareSourceItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: + phase_name = str(type(self).phase) + req_name = str(self.work_item.req.name) + processing_order.append((req_name, phase_name)) + wi = self.work_item + if req_name == "parent": + assert wi.resolved_version is not None + dep_item = ResolveItem( + WorkItem( + req=Requirement("child"), + req_type=RequirementType.BUILD_SYSTEM, + why_snapshot=[], + parent=(wi.req, wi.resolved_version), + ) ) - item.phase = BootstrapPhase.COMPLETE - return [item, dep_item] + return [CompleteItem(wi), dep_item] + return [] + def complete_run( + self: CompleteItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: + processing_order.append( + (str(self.work_item.req.name), str(type(self).phase)) + ) return [] # Pre-add parent to graph so child can reference it as parent @@ -880,7 +1019,10 @@ def tracking_dispatch(item: WorkItem) -> list[WorkItem]: ) with ( - patch.object(bt, "_dispatch_phase", side_effect=tracking_dispatch), + patch.object(ResolveItem, "run", resolve_run), + patch.object(StartItem, "run", start_run), + patch.object(PrepareSourceItem, "run", prepare_source_run), + patch.object(CompleteItem, "run", complete_run), patch.object( bt._resolver, "resolve", @@ -911,18 +1053,15 @@ def test_multiple_versions_error_isolation(self, tmp_context: WorkContext) -> No """Each version fails independently without crashing the loop.""" bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) - original_dispatch = bt._dispatch_phase - - def mock_dispatch(item: WorkItem) -> list[WorkItem]: - if item.phase in (BootstrapPhase.RESOLVE, BootstrapPhase.START): - return original_dispatch(item) - # Fail version 1.5, succeed for others - if str(item.resolved_version) == "1.5": + def prepare_source_run( + self: PrepareSourceItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: + if str(self.work_item.resolved_version) == "1.5": raise ValueError("1.5 broken") return [] with ( - patch.object(bt, "_dispatch_phase", side_effect=mock_dispatch), + patch.object(PrepareSourceItem, "run", prepare_source_run), patch.object( bt._resolver, "resolve", @@ -948,21 +1087,25 @@ def test_multiple_versions_resolve_failure_continues( """RESOLVE failure for one dependency does not crash the loop.""" bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) - original_dispatch = bt._dispatch_phase + original_resolve_run = ResolveItem.run completed: list[str] = [] - def mock_dispatch(item: WorkItem) -> list[WorkItem]: - if item.phase == BootstrapPhase.START: - return original_dispatch(item) - if item.phase == BootstrapPhase.RESOLVE: - if str(item.req.name) == "bad-dep": - raise RuntimeError("Could not resolve any versions for bad-dep") - return original_dispatch(item) - completed.append(str(item.req.name)) + def resolve_run( + self: ResolveItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: + if str(self.work_item.req.name) == "bad-dep": + raise RuntimeError("Could not resolve any versions for bad-dep") + return original_resolve_run(self, bt_arg) + + def prepare_source_run( + self: PrepareSourceItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: + completed.append(str(self.work_item.req.name)) return [] with ( - patch.object(bt, "_dispatch_phase", side_effect=mock_dispatch), + patch.object(ResolveItem, "run", resolve_run), + patch.object(PrepareSourceItem, "run", prepare_source_run), patch.object( bt._resolver, "resolve", @@ -983,19 +1126,18 @@ def test_test_mode_continues_after_failure(self, tmp_context: WorkContext) -> No """In test mode, failed items are recorded and processing continues.""" bt = bootstrapper.Bootstrapper(tmp_context, test_mode=True) - original_dispatch = bt._dispatch_phase items_completed: list[str] = [] - def mock_dispatch(item: WorkItem) -> list[WorkItem]: - if item.phase in (BootstrapPhase.RESOLVE, BootstrapPhase.START): - return original_dispatch(item) - if str(item.req.name) == "fail-pkg": + def prepare_source_run( + self: PrepareSourceItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: + if str(self.work_item.req.name) == "fail-pkg": raise RuntimeError("build error") - items_completed.append(str(item.req.name)) + items_completed.append(str(self.work_item.req.name)) return [] with ( - patch.object(bt, "_dispatch_phase", side_effect=mock_dispatch), + patch.object(PrepareSourceItem, "run", prepare_source_run), patch.object( bt._resolver, "resolve", @@ -1013,7 +1155,7 @@ def mock_dispatch(item: WorkItem) -> list[WorkItem]: class TestPhasePrepareSource: - """Tests for _phase_prepare_source: prebuilt, source, cache, and error paths.""" + """Tests for PrepareSourceItem.run(): prebuilt, source, cache, and error paths.""" def test_prebuilt_uses_background_result(self, tmp_context: WorkContext) -> None: """Prebuilt package uses bg_future result and advances to PROCESS_INSTALL_DEPS.""" @@ -1033,16 +1175,17 @@ def test_prebuilt_uses_background_result(self, tmp_context: WorkContext) -> None ) with patch.object(tmp_context.constraints, "get_constraint", return_value=None): - result = bt._phase_prepare_source(item) + result = item.run(bt) assert len(result) == 1 - assert result[0] is item - assert item.phase == BootstrapPhase.PROCESS_INSTALL_DEPS - assert item.build_result is not None - assert item.build_result.source_type == SourceType.PREBUILT - assert item.build_result.wheel_filename == mock_wheel - assert item.build_result.sdist_filename is None - assert item.build_result.build_env is None + assert isinstance(result[0], ProcessInstallDepsItem) + assert result[0].work_item is item.work_item + wi = result[0].work_item + assert wi.build_result is not None + assert wi.build_result.source_type == SourceType.PREBUILT + assert wi.build_result.wheel_filename == mock_wheel + assert wi.build_result.sdist_filename is None + assert wi.build_result.build_env is None def test_source_no_cache_uses_background_result( self, tmp_context: WorkContext @@ -1071,21 +1214,22 @@ def test_source_no_cache_uses_background_result( bt, "_create_unresolved_work_items", return_value=[mock_dep_item] ) as mock_create_items, ): - result = bt._phase_prepare_source(item) - - assert item.phase == BootstrapPhase.PREPARE_BUILD - assert item.build_env is mock_env - assert item.sdist_root_dir == sdist_root - assert item.unpack_dir == sdist_root.parent - assert item.cached_wheel_filename is None - assert result[0] is item + result = item.run(bt) + + wi = item.work_item + assert isinstance(result[0], PrepareBuildItem) + assert result[0].work_item is wi + assert wi.build_env is mock_env + assert wi.sdist_root_dir == sdist_root + assert wi.unpack_dir == sdist_root.parent + assert wi.cached_wheel_filename is None assert result[1] is mock_dep_item mock_create_env.assert_called_once() mock_create_items.assert_called_once_with( - item.build_system_deps, + wi.build_system_deps, RequirementType.BUILD_SYSTEM, - item.req, - item.resolved_version, + wi.req, + wi.resolved_version, ) def test_source_cached_wheel_uses_background_result( @@ -1115,11 +1259,12 @@ def test_source_cached_wheel_uses_background_result( ), patch.object(bt, "_create_unresolved_work_items", return_value=[]), ): - result = bt._phase_prepare_source(item) + result = item.run(bt) - assert item.cached_wheel_filename == cached_wheel - assert item.sdist_root_dir == unpacked / unpacked.stem - assert item.phase == BootstrapPhase.PREPARE_BUILD + wi = item.work_item + assert wi.cached_wheel_filename == cached_wheel + assert wi.sdist_root_dir == unpacked / unpacked.stem + assert isinstance(result[0], PrepareBuildItem) assert len(result) == 1 def test_bad_sdist_root_raises_valueerror(self, tmp_context: WorkContext) -> None: @@ -1134,7 +1279,7 @@ def test_bad_sdist_root_raises_valueerror(self, tmp_context: WorkContext) -> Non with patch.object(tmp_context.constraints, "get_constraint", return_value=None): with pytest.raises(ValueError, match="should be"): - bt._phase_prepare_source(item) + item.run(bt) def test_constraint_logged_when_present( self, tmp_context: WorkContext, caplog: pytest.LogCaptureFixture @@ -1162,14 +1307,14 @@ def test_constraint_logged_when_present( ), patch.object(bt, "_create_unresolved_work_items", return_value=[]), ): - bt._phase_prepare_source(item) + result = item.run(bt) - assert item.phase == BootstrapPhase.PREPARE_BUILD + assert isinstance(result[0], PrepareBuildItem) assert "matches constraint" in caplog.text class TestPhasePrepareBuild: - """Tests for _phase_prepare_build: dep installation and extraction.""" + """Tests for PrepareBuildItem.run(): dep installation and extraction.""" def test_installs_system_deps_and_returns_backend_sdist_items( self, tmp_context: WorkContext @@ -1185,6 +1330,7 @@ def test_installs_system_deps_and_returns_backend_sdist_items( sdist_root_dir=sdist_root, build_system_deps=system_deps, ) + wi = item.work_item backend_item = _make_resolve_item(req="wheel") sdist_item = _make_resolve_item(req="flit-core") @@ -1204,13 +1350,14 @@ def test_installs_system_deps_and_returns_backend_sdist_items( side_effect=[[backend_item], [sdist_item]], ), ): - result = bt._phase_prepare_build(item) + result = item.run(bt) - assert item.phase == BootstrapPhase.BUILD - assert item.build_backend_deps == {Requirement("wheel")} - assert item.build_sdist_deps == {Requirement("flit-core")} + assert isinstance(result[0], BuildItem) + assert result[0].work_item is wi + assert wi.build_backend_deps == {Requirement("wheel")} + assert wi.build_sdist_deps == {Requirement("flit-core")} mock_env.install.assert_called_once_with(system_deps) - assert result == [item, backend_item, sdist_item] + assert result == [result[0], backend_item, sdist_item] def test_no_extra_deps_returns_item_only(self, tmp_context: WorkContext) -> None: """When backend and sdist deps are empty, returns only the item.""" @@ -1240,10 +1387,10 @@ def test_no_extra_deps_returns_item_only(self, tmp_context: WorkContext) -> None side_effect=[[], []], ), ): - result = bt._phase_prepare_build(item) + result = item.run(bt) - assert result == [item] - assert item.phase == BootstrapPhase.BUILD + assert len(result) == 1 + assert isinstance(result[0], BuildItem) mock_env.install.assert_called_once_with(system_deps) def test_install_called_once_with_system_deps( @@ -1276,7 +1423,7 @@ def test_install_called_once_with_system_deps( side_effect=[[], []], ), ): - bt._phase_prepare_build(item) + item.run(bt) mock_env.install.assert_called_once_with(system_deps) @@ -1293,6 +1440,7 @@ def test_creates_items_with_correct_requirement_types( sdist_root_dir=sdist_root, build_system_deps={Requirement("setuptools")}, ) + wi = item.work_item with ( patch( @@ -1309,31 +1457,31 @@ def test_creates_items_with_correct_requirement_types( side_effect=[[], []], ) as mock_create_items, ): - bt._phase_prepare_build(item) + item.run(bt) calls = mock_create_items.call_args_list assert calls[0] == call( {Requirement("wheel")}, RequirementType.BUILD_BACKEND, - item.req, - item.resolved_version, + wi.req, + wi.resolved_version, ) assert calls[1] == call( {Requirement("flit-core")}, RequirementType.BUILD_SDIST, - item.req, - item.resolved_version, + wi.req, + wi.resolved_version, ) class TestPhaseBuild: - """Tests for _phase_build: conditional dep install and result construction.""" + """Tests for BuildItem.run(): conditional dep install and result construction.""" - def _make_build_phase_item(self, tmp_context: WorkContext) -> WorkItem: + def _make_build_phase_item(self, tmp_context: WorkContext) -> BuildItem: mock_env = Mock() sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" sdist_root.parent.mkdir(parents=True, exist_ok=True) - return _make_build_item( + item = _make_build_item( phase=BootstrapPhase.BUILD, build_env=mock_env, sdist_root_dir=sdist_root, @@ -1341,6 +1489,8 @@ def _make_build_phase_item(self, tmp_context: WorkContext) -> WorkItem: build_backend_deps={Requirement("wheel")}, build_sdist_deps={Requirement("flit-core")}, ) + assert isinstance(item, BuildItem) + return item def test_disjoint_deps_installs_remaining(self, tmp_context: WorkContext) -> None: """Disjoint backend/sdist deps from system deps triggers install.""" @@ -1362,13 +1512,13 @@ def test_disjoint_deps_installs_remaining(self, tmp_context: WorkContext) -> Non patch.object(bt, "_do_build", return_value=(mock_wheel, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): - result = bt._phase_build(item) + result = item.run(bt) mock_env.install.assert_called_once_with( {Requirement("wheel"), Requirement("flit-core")} ) - assert item.phase == BootstrapPhase.PROCESS_INSTALL_DEPS assert len(result) == 1 + assert isinstance(result[0], ProcessInstallDepsItem) def test_overlapping_deps_skips_install(self, tmp_context: WorkContext) -> None: """Overlapping backend/sdist deps with system deps skips install.""" @@ -1389,14 +1539,12 @@ def test_overlapping_deps_skips_install(self, tmp_context: WorkContext) -> None: patch.object(bt, "_do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): - bt._phase_build(item) + item.run(bt) mock_env.install.assert_not_called() def test_partial_overlap_deps_skips_install(self, tmp_context: WorkContext) -> None: - """isdisjoint is False on partial overlap, so install is skipped entirely - even for deps not in build_system_deps (here: cython). - """ + """isdisjoint is False on partial overlap, so install is skipped entirely.""" bt = bootstrapper.Bootstrapper(tmp_context) mock_env = Mock() sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" @@ -1414,7 +1562,7 @@ def test_partial_overlap_deps_skips_install(self, tmp_context: WorkContext) -> N patch.object(bt, "_do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): - bt._phase_build(item) + item.run(bt) mock_env.install.assert_not_called() @@ -1432,16 +1580,17 @@ def test_do_build_receives_item_fields(self, tmp_context: WorkContext) -> None: build_sdist_only=True, cached_wheel_filename=cached_wheel, ) + wi = item.work_item with ( patch.object(bt, "_do_build", return_value=(None, None)) as mock_do_build, patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): - bt._phase_build(item) + item.run(bt) mock_do_build.assert_called_once_with( - req=item.req, - resolved_version=item.resolved_version, + req=wi.req, + resolved_version=wi.resolved_version, sdist_root_dir=sdist_root, build_env=mock_env, build_sdist_only=True, @@ -1451,18 +1600,19 @@ def test_do_build_receives_item_fields(self, tmp_context: WorkContext) -> None: def test_build_result_references_item_build_env( self, tmp_context: WorkContext ) -> None: - """build_result.build_env is the same object as item.build_env.""" + """build_result.build_env is the same object as item.work_item.build_env.""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_build_phase_item(tmp_context) + wi = item.work_item with ( patch.object(bt, "_do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): - result = bt._phase_build(item) + item.run(bt) - assert result[0].build_result is not None - assert result[0].build_result.build_env is item.build_env + assert wi.build_result is not None + assert wi.build_result.build_env is wi.build_env def test_build_result_uses_source_type_from_sources( self, tmp_context: WorkContext @@ -1470,20 +1620,21 @@ def test_build_result_uses_source_type_from_sources( """source_type comes from sources.get_source_type, not hardcoded.""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_build_phase_item(tmp_context) + wi = item.work_item with ( patch.object(bt, "_do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.PREBUILT), ): - bt._phase_build(item) + item.run(bt) - assert item.build_result is not None - assert item.build_result.source_type == SourceType.PREBUILT + assert wi.build_result is not None + assert wi.build_result.source_type == SourceType.PREBUILT - def test_returns_single_item_at_process_install_deps( + def test_returns_single_process_install_deps_item( self, tmp_context: WorkContext ) -> None: - """_phase_build returns exactly [item] with phase PROCESS_INSTALL_DEPS.""" + """BuildItem.run() returns exactly [ProcessInstallDepsItem].""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_build_phase_item(tmp_context) @@ -1491,17 +1642,33 @@ def test_returns_single_item_at_process_install_deps( patch.object(bt, "_do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): - result = bt._phase_build(item) + result = item.run(bt) assert len(result) == 1 - assert result[0] is item - assert item.phase == BootstrapPhase.PROCESS_INSTALL_DEPS + assert isinstance(result[0], ProcessInstallDepsItem) + assert result[0].work_item is item.work_item + + def test_phase_advancement_preserves_work_item_identity( + self, tmp_context: WorkContext + ) -> None: + """When BuildItem.run() returns ProcessInstallDepsItem, work_item is same obj.""" + bt = bootstrapper.Bootstrapper(tmp_context) + item = self._make_build_phase_item(tmp_context) + wi = item.work_item + + with ( + patch.object(bt, "_do_build", return_value=(None, None)), + patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), + ): + result = item.run(bt) + + assert result[0].work_item is wi class TestPhaseProcessInstallDeps: - """Tests for _phase_process_install_deps: hooks, dep extraction, error modes.""" + """Tests for ProcessInstallDepsItem.run(): hooks, dep extraction, error modes.""" - def _make_process_item(self, tmp_context: WorkContext) -> WorkItem: + def _make_process_item(self, tmp_context: WorkContext) -> ProcessInstallDepsItem: build_result = SourceBuildResult( wheel_filename=tmp_context.work_dir / "testpkg-1.0-py3-none-any.whl", sdist_filename=tmp_context.work_dir / "testpkg-1.0.tar.gz", @@ -1510,18 +1677,21 @@ def _make_process_item(self, tmp_context: WorkContext) -> WorkItem: build_env=Mock(), source_type=SourceType.SDIST, ) - return _make_build_item( + item = _make_build_item( phase=BootstrapPhase.PROCESS_INSTALL_DEPS, build_result=build_result, source_url="https://pkg.test/testpkg-1.0.tar.gz", ) + assert isinstance(item, ProcessInstallDepsItem) + return item def test_normal_path_returns_item_and_dep_items( self, tmp_context: WorkContext ) -> None: - """Normal path: hooks, deps, build order, returns [item, *dep_items].""" + """Normal path: hooks, deps, build order, returns [CompleteItem, *dep_items].""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_process_item(tmp_context) + wi = item.work_item dep_item = _make_resolve_item(req="dep-a") with ( @@ -1542,16 +1712,17 @@ def test_normal_path_returns_item_and_dep_items( bt, "_create_unresolved_work_items", return_value=[dep_item] ) as mock_create_items, ): - result = bt._phase_process_install_deps(item) + result = item.run(bt) - assert item.phase == BootstrapPhase.COMPLETE - assert result == [item, dep_item] + assert isinstance(result[0], CompleteItem) + assert result[0].work_item is wi + assert result[1] is dep_item mock_build_order.assert_called_once() mock_create_items.assert_called_once_with( [Requirement("dep-a")], RequirementType.INSTALL, - item.req, - item.resolved_version, + wi.req, + wi.resolved_version, ) def test_hook_error_test_mode_records_and_continues( @@ -1578,10 +1749,9 @@ def test_hook_error_test_mode_records_and_continues( patch.object(bt, "_add_to_build_order") as mock_build_order, patch.object(bt, "_create_unresolved_work_items", return_value=[]), ): - result = bt._phase_process_install_deps(item) + result = item.run(bt) - assert item.phase == BootstrapPhase.COMPLETE - assert result == [item] + assert isinstance(result[0], CompleteItem) assert len(bt.failed_packages) == 1 assert bt.failed_packages[0]["failure_type"] == "hook" mock_get_deps.assert_called_once() @@ -1599,7 +1769,7 @@ def test_hook_error_normal_mode_raises(self, tmp_context: WorkContext) -> None: ), ): with pytest.raises(RuntimeError, match="hook failed"): - bt._phase_process_install_deps(item) + item.run(bt) def test_dep_extraction_error_test_mode_uses_empty_deps( self, tmp_context: WorkContext @@ -1607,6 +1777,7 @@ def test_dep_extraction_error_test_mode_uses_empty_deps( """Dep extraction error in test mode uses empty dep list, still writes build order.""" bt = bootstrapper.Bootstrapper(tmp_context, test_mode=True) item = self._make_process_item(tmp_context) + wi = item.work_item with ( patch("fromager.hooks.run_post_bootstrap_hooks"), @@ -1626,18 +1797,17 @@ def test_dep_extraction_error_test_mode_uses_empty_deps( bt, "_create_unresolved_work_items", return_value=[] ) as mock_create_items, ): - result = bt._phase_process_install_deps(item) + result = item.run(bt) - assert item.phase == BootstrapPhase.COMPLETE - assert result == [item] + assert isinstance(result[0], CompleteItem) assert len(bt.failed_packages) == 1 assert bt.failed_packages[0]["failure_type"] == "dependency_extraction" mock_build_order.assert_called_once() mock_create_items.assert_called_once_with( [], RequirementType.INSTALL, - item.req, - item.resolved_version, + wi.req, + wi.resolved_version, ) def test_dep_extraction_error_normal_mode_raises( @@ -1656,10 +1826,10 @@ def test_dep_extraction_error_normal_mode_raises( ), ): with pytest.raises(RuntimeError, match="dep failed"): - bt._phase_process_install_deps(item) + item.run(bt) def test_no_install_deps_returns_item_only(self, tmp_context: WorkContext) -> None: - """When no install deps, returns [item] at COMPLETE phase.""" + """When no install deps, returns [CompleteItem].""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_process_item(tmp_context) @@ -1675,10 +1845,10 @@ def test_no_install_deps_returns_item_only(self, tmp_context: WorkContext) -> No patch.object(bt, "_add_to_build_order"), patch.object(bt, "_create_unresolved_work_items", return_value=[]), ): - result = bt._phase_process_install_deps(item) + result = item.run(bt) - assert result == [item] - assert item.phase == BootstrapPhase.COMPLETE + assert len(result) == 1 + assert isinstance(result[0], CompleteItem) def test_build_order_called_with_correct_args( self, tmp_context: WorkContext @@ -1686,6 +1856,7 @@ def test_build_order_called_with_correct_args( """_add_to_build_order receives correct source_type, prebuilt, constraint.""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_process_item(tmp_context) + wi = item.work_item constraint = Requirement("testpkg>=1.0") with ( @@ -1704,12 +1875,12 @@ def test_build_order_called_with_correct_args( patch.object(bt, "_add_to_build_order") as mock_build_order, patch.object(bt, "_create_unresolved_work_items", return_value=[]), ): - bt._phase_process_install_deps(item) + item.run(bt) mock_build_order.assert_called_once_with( - req=item.req, - version=item.resolved_version, - source_url=item.source_url, + req=wi.req, + version=wi.resolved_version, + source_url=wi.source_url, source_type=SourceType.SDIST, prebuilt=True, constraint=constraint, @@ -1974,9 +2145,9 @@ def test_returns_build_system_edges(self, tmp_context: WorkContext) -> None: ) bt = bootstrapper.Bootstrapper(tmp_context) - item = _make_build_item(req="biotite", version="1.6.0") + wi = _make_work_item(req="biotite", version="1.6.0") - result = bt._get_resolved_build_system_versions(item) + result = bt._get_resolved_build_system_versions(wi) assert canonicalize_name("hatch-cython") in result assert result[canonicalize_name("hatch-cython")] == ( @@ -1990,15 +2161,15 @@ def test_returns_empty_when_parent_not_in_graph( ) -> None: """Returns empty dict when parent node is not in the graph.""" bt = bootstrapper.Bootstrapper(tmp_context) - item = _make_build_item(req="nonexistent", version="1.0") + wi = _make_work_item(req="nonexistent", version="1.0") - result = bt._get_resolved_build_system_versions(item) + result = bt._get_resolved_build_system_versions(wi) assert result == {} class TestPhasePrepareBuildFiltering: - """Tests for _phase_prepare_build with build-system satisfaction filtering.""" + """Tests for PrepareBuildItem.run() with build-system satisfaction filtering.""" def test_satisfied_backend_dep_skips_resolve( self, tmp_context: WorkContext @@ -2032,6 +2203,7 @@ def test_satisfied_backend_dep_skips_resolve( sdist_root_dir=sdist_root, build_system_deps={Requirement("hatch-cython==0.5")}, ) + wi = item.work_item with ( patch( @@ -2048,14 +2220,14 @@ def test_satisfied_backend_dep_skips_resolve( return_value=[], ) as mock_create, ): - result = bt._phase_prepare_build(item) + result = item.run(bt) - assert item.phase == BootstrapPhase.BUILD + assert isinstance(result[0], BuildItem) calls = mock_create.call_args_list assert calls[0][0][0] == set() assert calls[1][0][0] == set() - assert item.build_backend_deps == set() - assert result == [item] + assert wi.build_backend_deps == set() + assert len(result) == 1 def test_extras_dep_not_filtered(self, tmp_context: WorkContext) -> None: """Backend dep with extras passes through even if name matches.""" @@ -2104,12 +2276,12 @@ def test_extras_dep_not_filtered(self, tmp_context: WorkContext) -> None: side_effect=[[resolve_item], []], ) as mock_create, ): - result = bt._phase_prepare_build(item) + result = item.run(bt) calls = mock_create.call_args_list assert calls[0][0][0] == {extras_req} - assert item.build_backend_deps == {extras_req} - assert result == [item, resolve_item] + assert item.work_item.build_backend_deps == {extras_req} + assert result[1] is resolve_item def test_unsatisfied_backend_dep_creates_resolve_item( self, tmp_context: WorkContext @@ -2151,30 +2323,100 @@ def test_unsatisfied_backend_dep_creates_resolve_item( side_effect=[[resolve_item], []], ) as mock_create, ): - result = bt._phase_prepare_build(item) + result = item.run(bt) calls = mock_create.call_args_list assert calls[0][0][0] == {Requirement("wheel")} - assert result == [item, resolve_item] + assert result[1] is resolve_item + + +class TestBackgroundWork: + """Tests for background_work() dispatch on PhaseItem subclasses.""" + + def test_resolve_item_background_work_returns_callable( + self, tmp_context: WorkContext + ) -> None: + """ResolveItem.background_work(bt) returns a non-None callable.""" + bt = bootstrapper.Bootstrapper(tmp_context) + item = _make_resolve_item() + bg = item.background_work(bt) + assert bg is not None + assert callable(bg) + + def test_prepare_source_item_background_work_source( + self, tmp_context: WorkContext + ) -> None: + """When pbi_pre_built=False, background_work() returns a source callable.""" + bt = bootstrapper.Bootstrapper(tmp_context) + wi = _make_work_item( + req="testpkg", + version="1.0", + source_url="https://pypi.test/testpkg-1.0.tar.gz", + ) + wi.pbi_pre_built = False + item = PrepareSourceItem(wi) + bg = item.background_work(bt) + assert bg is not None + assert callable(bg) + + def test_prepare_source_item_background_work_prebuilt( + self, tmp_context: WorkContext + ) -> None: + """When pbi_pre_built=True, background_work() returns a prebuilt callable.""" + bt = bootstrapper.Bootstrapper(tmp_context) + wi = _make_work_item( + req="testpkg", + version="1.0", + source_url="https://pypi.test/testpkg-1.0-py3-none-any.whl", + ) + wi.pbi_pre_built = True + item = PrepareSourceItem(wi) + bg = item.background_work(bt) + assert bg is not None + assert callable(bg) + + +class TestAsJson: + """Tests for PhaseItem.as_json() serialization.""" + + @pytest.mark.parametrize( + "cls, expected_phase", + [ + (ResolveItem, BootstrapPhase.RESOLVE), + (StartItem, BootstrapPhase.START), + (PrepareSourceItem, BootstrapPhase.PREPARE_SOURCE), + (PrepareBuildItem, BootstrapPhase.PREPARE_BUILD), + (BuildItem, BootstrapPhase.BUILD), + (ProcessInstallDepsItem, BootstrapPhase.PROCESS_INSTALL_DEPS), + (CompleteItem, BootstrapPhase.COMPLETE), + ], + ) + def test_phase_field_per_subclass( + self, cls: type[PhaseItem], expected_phase: BootstrapPhase + ) -> None: + wi = _make_work_item(req="testpkg") + item = cls(wi) + data = item.as_json() + assert data["phase"] == str(expected_phase) class TestThreadPoolSubmission: """Tests that exercise the real ThreadPoolExecutor submission path.""" - def test_push_items_submits_resolve_to_real_thread_pool( + def test_push_items_sets_bg_future_when_pool_exists( self, tmp_context: WorkContext ) -> None: - """_push_items submits RESOLVE work to a real thread pool and result() returns correctly.""" + """_push_items submits RESOLVE work to a real thread pool.""" expected = [("https://pkg.test/testpkg-1.0.tar.gz", Version("1.0"))] with bootstrapper.Bootstrapper(tmp_context, num_bg_threads=1) as bt: with patch.object(bt._resolver, "resolve", return_value=expected): item = _make_resolve_item(req="testpkg") - stack: list[WorkItem] = [] + stack: list[PhaseItem] = [] bt._push_items(stack, [item]) - # bg_future must be a real Future, not pre-set + # bg_future must be set on the PhaseItem, not None assert item.bg_future is not None assert isinstance(item.bg_future, concurrent.futures.Future) @@ -2182,9 +2424,37 @@ def test_push_items_submits_resolve_to_real_thread_pool( result = item.bg_future.result(timeout=10) assert result == expected - # _phase_resolve processes the future correctly - new_items = bt._phase_resolve(item) + # ResolveItem.run() processes the future correctly + new_items = item.run(bt) assert len(new_items) == 1 - assert new_items[0].phase == BootstrapPhase.START - assert new_items[0].resolved_version == Version("1.0") + assert isinstance(new_items[0], StartItem) + assert new_items[0].work_item.resolved_version == Version("1.0") + + def test_push_items_no_bg_future_when_pool_is_none( + self, tmp_context: WorkContext + ) -> None: + """When _bg_pool is None, calling _push_items leaves item.bg_future as None.""" + bt = bootstrapper.Bootstrapper(tmp_context) + bt._bg_pool = None + item = _make_resolve_item(req="testpkg") + stack: list[PhaseItem] = [] + + bt._push_items(stack, [item]) + + assert item.bg_future is None + + +class TestCreateUnresolvedWorkItemsReturnType: + """_create_unresolved_work_items returns ResolveItem instances.""" + + def test_returns_resolve_items(self, tmp_context: WorkContext) -> None: + bt = bootstrapper.Bootstrapper(tmp_context) + deps = [Requirement("dep-a")] + items = bt._create_unresolved_work_items( + deps, RequirementType.INSTALL, Requirement("parent"), Version("1.0") + ) + assert len(items) == 1 + assert isinstance(items[0], ResolveItem) + assert items[0].work_item.req == Requirement("dep-a") + assert items[0].work_item.req_type == RequirementType.INSTALL From 4f90f4ba0196d960d6f54525a14c4adf07eba1c8 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 1 Jul 2026 09:23:20 -0400 Subject: [PATCH 03/19] refactor(bootstrapper): expose _resolver as public property Add a public `resolver` property to the Bootstrapper class to provide a public API for accessing the version resolver. This allows ResolveItem.background_work() to access the resolver through the public property instead of directly touching the private _resolver attribute, maintaining the boundary between PhaseItem and Bootstrapper. Co-Authored-By: Claude Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index ca55e126..d53ff9b5 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -490,7 +490,7 @@ def background_work( self, bt: Bootstrapper ) -> typing.Callable[[], typing.Any] | None: """Return closure that calls ``_bg_resolve`` in a thread.""" - bg_resolver = bt._resolver + bg_resolver = bt.resolver req = self.work_item.req req_type = self.work_item.req_type parent_req = ( @@ -1050,6 +1050,11 @@ def __init__( # Track failed versions in multiple_versions mode self._failed_versions: dict[tuple[str, str], Exception] = {} + @property + def resolver(self) -> bootstrap_requirement_resolver.BootstrapRequirementResolver: + """Public accessor for the version resolver.""" + return self._resolver + def _resolve_and_add_top_level( self, req: Requirement, From fd956d334a2ebc7f1e0fb81a8bfaef4c06ece862 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 1 Jul 2026 14:10:09 -0400 Subject: [PATCH 04/19] refactor(bootstrapper): encapsulate _failed_versions as has_failed_version() Replace direct access to _failed_versions dict in ResolveItem.run() with a public has_failed_version() method. This maintains encapsulation of the private dict and provides a clean public API for checking if a package version has previously failed resolution. Co-Authored-By: Claude Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index d53ff9b5..d8632460 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -531,7 +531,7 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: resolved_versions = [ (url, ver) for url, ver in resolved_versions - if (pkg_name, str(ver)) not in bt._failed_versions + if not bt.has_failed_version(pkg_name, ver) ] if not resolved_versions: raise RuntimeError( @@ -1055,6 +1055,10 @@ def resolver(self) -> bootstrap_requirement_resolver.BootstrapRequirementResolve """Public accessor for the version resolver.""" return self._resolver + def has_failed_version(self, pkg_name: NormalizedName, version: Version) -> bool: + """Return True if this (pkg_name, version) has previously failed resolution.""" + return (pkg_name, str(version)) in self._failed_versions + def _resolve_and_add_top_level( self, req: Requirement, From 670825e4f6ecc51ce4fd7d385db7cb94e2b87d13 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 1 Jul 2026 14:16:24 -0400 Subject: [PATCH 05/19] refactor(bootstrapper): make StartItem-accessed members public MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove leading underscores from Bootstrapper methods that are accessed from StartItem.run() to fix cross-boundary private-member access: - _add_to_graph → add_to_graph - _processing_build_requirement → processing_build_requirement - _has_been_seen → has_been_seen - _mark_as_seen → mark_as_seen - _explain → explain Also update internal callers within Bootstrapper: - _filter_deps_satisfied_by_build_system() calls self.add_to_graph() - _build_wheel() calls self.explain Update all references in tests/test_bootstrapper.py and tests/test_bootstrapper_iterative.py. Co-Authored-By: Claude Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper.py | 24 ++++---- tests/test_bootstrapper.py | 86 ++++++++++++++-------------- tests/test_bootstrapper_iterative.py | 30 +++++----- 3 files changed, 70 insertions(+), 70 deletions(-) diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index d8632460..1428a1fb 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -608,7 +608,7 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: # Add to graph (skip TOP_LEVEL, already added in _resolve_and_add_top_level) if wi.req_type != RequirementType.TOP_LEVEL: - bt._add_to_graph( + bt.add_to_graph( wi.req, wi.req_type, wi.resolved_version, @@ -616,18 +616,18 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: wi.parent, ) - wi.build_sdist_only = bt.sdist_only and not bt._processing_build_requirement( + wi.build_sdist_only = bt.sdist_only and not bt.processing_build_requirement( wi.req_type ) - if bt._has_been_seen(wi.req, wi.resolved_version, wi.build_sdist_only): + if bt.has_been_seen(wi.req, wi.resolved_version, wi.build_sdist_only): logger.debug( f"redundant {wi.req_type} dependency {wi.req} " f"({wi.resolved_version}, sdist_only={wi.build_sdist_only}) " - f"for {bt._explain}" + f"for {bt.explain}" ) return [] - bt._mark_as_seen(wi.req, wi.resolved_version, wi.build_sdist_only) + bt.mark_as_seen(wi.req, wi.resolved_version, wi.build_sdist_only) logger.info( f"new {wi.req_type} dependency {wi.req} resolves to {wi.resolved_version}" @@ -1263,7 +1263,7 @@ def _run_bootstrap_loop(self, stack: list[PhaseItem]) -> None: self._push_items(stack, new_items) - def _processing_build_requirement(self, current_req_type: RequirementType) -> bool: + def processing_build_requirement(self, current_req_type: RequirementType) -> bool: """Are we currently processing a build requirement? We determine that a package is a build dependency if its requirement @@ -1396,7 +1396,7 @@ def _record_test_mode_failure( ) @property - def _explain(self) -> str: + def explain(self) -> str: """Return message formatting current version of why stack.""" return " for ".join( f"{req_type} dependency {req} ({resolved_version})" @@ -1437,7 +1437,7 @@ def _build_wheel( req, resolved_version, sdist_root_dir, build_env ) - logger.info(f"starting build of {self._explain} for {self.ctx.variant}") + logger.info(f"starting build of {self.explain} for {self.ctx.variant}") built_filename = wheels.build_wheel( ctx=self.ctx, req=req, @@ -1908,7 +1908,7 @@ def _get_version_from_package_metadata( metadata = dependencies.parse_metadata(metadata_filename, validate=False) return metadata.version - def _add_to_graph( + def add_to_graph( self, req: Requirement, req_type: RequirementType, @@ -1949,7 +1949,7 @@ def _resolved_key( typ, ) - def _mark_as_seen( + def mark_as_seen( self, req: Requirement, version: Version, @@ -1968,7 +1968,7 @@ def _mark_as_seen( # Mark wheel seen only for wheel build self._seen_requirements.add(self._resolved_key(req, version, "wheel")) - def _has_been_seen( + def has_been_seen( self, req: Requirement, version: Version, @@ -2160,7 +2160,7 @@ def _filter_deps_satisfied_by_build_system( version, ) if parent is not None: - self._add_to_graph( + self.add_to_graph( req=dep, req_type=dep_req_type, req_version=version, diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index 6c3ee483..57ac6ed5 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -20,9 +20,9 @@ def test_seen(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) req = Requirement("testdist") version = Version("1.2") - assert not bt._has_been_seen(req, version) - bt._mark_as_seen(req, version) - assert bt._has_been_seen(req, version) + assert not bt.has_been_seen(req, version) + bt.mark_as_seen(req, version) + assert bt.has_been_seen(req, version) def test_seen_extras(tmp_context: WorkContext) -> None: @@ -30,46 +30,46 @@ def test_seen_extras(tmp_context: WorkContext) -> None: req2 = Requirement("testdist[extra]") version = Version("1.2") bt = bootstrapper.Bootstrapper(tmp_context) - assert not bt._has_been_seen(req1, version) - bt._mark_as_seen(req1, version) - assert bt._has_been_seen(req1, version) - assert not bt._has_been_seen(req2, version) - bt._mark_as_seen(req2, version) - assert bt._has_been_seen(req1, version) - assert bt._has_been_seen(req2, version) + assert not bt.has_been_seen(req1, version) + bt.mark_as_seen(req1, version) + assert bt.has_been_seen(req1, version) + assert not bt.has_been_seen(req2, version) + bt.mark_as_seen(req2, version) + assert bt.has_been_seen(req1, version) + assert bt.has_been_seen(req2, version) def test_seen_name_canonicalization(tmp_context: WorkContext) -> None: req = Requirement("flit_core") version = Version("1.2") bt = bootstrapper.Bootstrapper(tmp_context) - assert not bt._has_been_seen(req, version) - bt._mark_as_seen(req, version) - assert bt._has_been_seen(req, version) + assert not bt.has_been_seen(req, version) + bt.mark_as_seen(req, version) + assert bt.has_been_seen(req, version) def test_seen_requirements_sdist(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) req = Requirement("testdist") version = Version("1.2") - assert not bt._has_been_seen(req, version, sdist_only=False) - assert not bt._has_been_seen(req, version, sdist_only=True) + assert not bt.has_been_seen(req, version, sdist_only=False) + assert not bt.has_been_seen(req, version, sdist_only=True) # sdist only does not affect wheel status - bt._mark_as_seen(req, version, sdist_only=True) - assert bt._has_been_seen(req, version, sdist_only=True) - assert not bt._has_been_seen(req, version, sdist_only=False) + bt.mark_as_seen(req, version, sdist_only=True) + assert bt.has_been_seen(req, version, sdist_only=True) + assert not bt.has_been_seen(req, version, sdist_only=False) - bt._mark_as_seen(req, version, sdist_only=False) - assert bt._has_been_seen(req, version, sdist_only=True) - assert bt._has_been_seen(req, version, sdist_only=False) + bt.mark_as_seen(req, version, sdist_only=False) + assert bt.has_been_seen(req, version, sdist_only=True) + assert bt.has_been_seen(req, version, sdist_only=False) req2 = Requirement("testwheel") - assert not bt._has_been_seen(req2, version, sdist_only=False) - assert not bt._has_been_seen(req2, version, sdist_only=True) + assert not bt.has_been_seen(req2, version, sdist_only=False) + assert not bt.has_been_seen(req2, version, sdist_only=True) # full seen affects both sdist and wheel status - bt._mark_as_seen(req2, version, sdist_only=False) - assert bt._has_been_seen(req2, version, sdist_only=True) - assert bt._has_been_seen(req2, version, sdist_only=False) + bt.mark_as_seen(req2, version, sdist_only=False) + assert bt.has_been_seen(req2, version, sdist_only=True) + assert bt.has_been_seen(req2, version, sdist_only=False) def test_build_order(tmp_context: WorkContext) -> None: @@ -180,17 +180,17 @@ def test_build_order_name_canonicalization(tmp_context: WorkContext) -> None: def test_explain(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) bt.why = [(RequirementType.TOP_LEVEL, Requirement("foo"), Version("1.0.0"))] - assert bt._explain == f"{RequirementType.TOP_LEVEL} dependency foo (1.0.0)" + assert bt.explain == f"{RequirementType.TOP_LEVEL} dependency foo (1.0.0)" bt.why = [] - assert bt._explain == "" + assert bt.explain == "" bt.why = [ (RequirementType.TOP_LEVEL, Requirement("foo"), Version("1.0.0")), (RequirementType.BUILD_SYSTEM, Requirement("bar==4.0.0"), Version("4.0.0")), ] assert ( - bt._explain + bt.explain == f"{RequirementType.BUILD_SYSTEM} dependency bar==4.0.0 (4.0.0) for {RequirementType.TOP_LEVEL} dependency foo (1.0.0)" ) @@ -198,26 +198,26 @@ def test_explain(tmp_context: WorkContext) -> None: def test_is_build_requirement(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) bt.why = [] - assert not bt._processing_build_requirement(RequirementType.TOP_LEVEL) - assert bt._processing_build_requirement(RequirementType.BUILD_SYSTEM) - assert bt._processing_build_requirement(RequirementType.BUILD_BACKEND) - assert bt._processing_build_requirement(RequirementType.BUILD_SDIST) - assert not bt._processing_build_requirement(RequirementType.INSTALL) + assert not bt.processing_build_requirement(RequirementType.TOP_LEVEL) + assert bt.processing_build_requirement(RequirementType.BUILD_SYSTEM) + assert bt.processing_build_requirement(RequirementType.BUILD_BACKEND) + assert bt.processing_build_requirement(RequirementType.BUILD_SDIST) + assert not bt.processing_build_requirement(RequirementType.INSTALL) bt.why = [(RequirementType.TOP_LEVEL, Requirement("foo"), Version("1.0.0"))] - assert not bt._processing_build_requirement(RequirementType.INSTALL) - assert bt._processing_build_requirement(RequirementType.BUILD_SYSTEM) - assert bt._processing_build_requirement(RequirementType.BUILD_BACKEND) - assert bt._processing_build_requirement(RequirementType.BUILD_SDIST) + assert not bt.processing_build_requirement(RequirementType.INSTALL) + assert bt.processing_build_requirement(RequirementType.BUILD_SYSTEM) + assert bt.processing_build_requirement(RequirementType.BUILD_BACKEND) + assert bt.processing_build_requirement(RequirementType.BUILD_SDIST) bt.why = [ (RequirementType.TOP_LEVEL, Requirement("foo"), Version("1.0.0")), (RequirementType.BUILD_SYSTEM, Requirement("bar==4.0.0"), Version("4.0.0")), ] - assert bt._processing_build_requirement(RequirementType.INSTALL) - assert bt._processing_build_requirement(RequirementType.BUILD_SYSTEM) - assert bt._processing_build_requirement(RequirementType.BUILD_BACKEND) - assert bt._processing_build_requirement(RequirementType.BUILD_SDIST) + assert bt.processing_build_requirement(RequirementType.INSTALL) + assert bt.processing_build_requirement(RequirementType.BUILD_SYSTEM) + assert bt.processing_build_requirement(RequirementType.BUILD_BACKEND) + assert bt.processing_build_requirement(RequirementType.BUILD_SDIST) def test_find_cached_wheel_returns_tuple(tmp_context: WorkContext) -> None: @@ -339,7 +339,7 @@ def prepare_source_run( req = Requirement("testpkg>=1.0") with patch.object(bootstrapper.PrepareSourceItem, "run", prepare_source_run): - with patch.object(bt, "_has_been_seen", return_value=False): + with patch.object(bt, "has_been_seen", return_value=False): bt._bootstrap_one( req=req, req_type=RequirementType.INSTALL, diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index de4840a8..4ecdc7a7 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -571,7 +571,7 @@ def test_already_seen_returns_empty(self, tmp_context: WorkContext) -> None: # Mark as seen first assert wi.resolved_version is not None - bt._mark_as_seen(wi.req, wi.resolved_version) + bt.mark_as_seen(wi.req, wi.resolved_version) result = item.run(bt) @@ -626,9 +626,9 @@ def test_marks_as_seen(self, tmp_context: WorkContext) -> None: wi = item.work_item assert wi.resolved_version is not None - assert not bt._has_been_seen(wi.req, wi.resolved_version) + assert not bt.has_been_seen(wi.req, wi.resolved_version) item.run(bt) - assert bt._has_been_seen(wi.req, wi.resolved_version) + assert bt.has_been_seen(wi.req, wi.resolved_version) def test_sets_pbi_pre_built_before_prepare_source( self, tmp_context: WorkContext @@ -823,7 +823,7 @@ def test_multiple_versions_records_and_removes_from_graph( pre_built=False, ) # Mark as seen - bt._mark_as_seen(wi.req, wi.resolved_version) + bt.mark_as_seen(wi.req, wi.resolved_version) result = bt._handle_phase_error(item, err) @@ -835,8 +835,8 @@ def test_multiple_versions_records_and_removes_from_graph( key = f"{canonicalize_name('testpkg')}==1.0" assert key not in tmp_context.dependency_graph.nodes # Seen markers cleared - assert not bt._has_been_seen(wi.req, wi.resolved_version) - assert not bt._has_been_seen(wi.req, wi.resolved_version, sdist_only=True) + assert not bt.has_been_seen(wi.req, wi.resolved_version) + assert not bt.has_been_seen(wi.req, wi.resolved_version, sdist_only=True) def test_multiple_versions_logs_phase( self, tmp_context: WorkContext, caplog: pytest.LogCaptureFixture @@ -1071,7 +1071,7 @@ def prepare_source_run( ("url-1.0", Version("1.0")), ], ), - patch.object(bt, "_has_been_seen", return_value=False), + patch.object(bt, "has_been_seen", return_value=False), ): bt._bootstrap_one(Requirement("pkg"), RequirementType.INSTALL) @@ -1111,7 +1111,7 @@ def prepare_source_run( "resolve", return_value=[("url-1.0", Version("1.0"))], ), - patch.object(bt, "_has_been_seen", return_value=False), + patch.object(bt, "has_been_seen", return_value=False), ): bt._bootstrap_one(Requirement("good-pkg"), RequirementType.INSTALL) bt._bootstrap_one(Requirement("bad-dep"), RequirementType.INSTALL) @@ -1933,7 +1933,7 @@ def test_satisfied_dep_reuses_build_system_version( } parent = (Requirement("biotite==1.6.0"), Version("1.6.0")) - with patch.object(bt, "_add_to_graph") as mock_add: + with patch.object(bt, "add_to_graph") as mock_add: result = bt._filter_deps_satisfied_by_build_system( {Requirement("hatch-cython")}, resolved_build_sys, @@ -1960,7 +1960,7 @@ def test_satisfied_dep_with_compatible_specifier( } parent = (Requirement("testpkg==1.0"), Version("1.0")) - with patch.object(bt, "_add_to_graph") as mock_add: + with patch.object(bt, "add_to_graph") as mock_add: result = bt._filter_deps_satisfied_by_build_system( {Requirement("foo>=1.0")}, resolved_build_sys, @@ -1980,7 +1980,7 @@ def test_unsatisfied_dep_passes_through(self, tmp_context: WorkContext) -> None: parent = (Requirement("testpkg==1.0"), Version("1.0")) wheel_req = Requirement("wheel") - with patch.object(bt, "_add_to_graph") as mock_add: + with patch.object(bt, "add_to_graph") as mock_add: result = bt._filter_deps_satisfied_by_build_system( {wheel_req}, resolved_build_sys, @@ -2002,7 +2002,7 @@ def test_incompatible_specifier_passes_through( parent = (Requirement("testpkg==1.0"), Version("1.0")) foo_req = Requirement("foo>=2.0") - with patch.object(bt, "_add_to_graph") as mock_add: + with patch.object(bt, "add_to_graph") as mock_add: result = bt._filter_deps_satisfied_by_build_system( {foo_req}, resolved_build_sys, @@ -2023,7 +2023,7 @@ def test_incompatible_specifier_logs_warning( } parent = (Requirement("testpkg==1.0"), Version("1.0")) - with patch.object(bt, "_add_to_graph"): + with patch.object(bt, "add_to_graph"): bt._filter_deps_satisfied_by_build_system( {Requirement("foo>=2.0")}, resolved_build_sys, @@ -2048,7 +2048,7 @@ def test_mixed_satisfied_and_unsatisfied(self, tmp_context: WorkContext) -> None cython_req = Requirement("hatch-cython") wheel_req = Requirement("wheel") - with patch.object(bt, "_add_to_graph") as mock_add: + with patch.object(bt, "add_to_graph") as mock_add: result = bt._filter_deps_satisfied_by_build_system( {cython_req, wheel_req}, resolved_build_sys, @@ -2102,7 +2102,7 @@ def test_dep_with_extras_passes_through(self, tmp_context: WorkContext) -> None: parent = (Requirement("testpkg==1.0"), Version("1.0")) extras_req = Requirement("foo[bar]>=1.0") - with patch.object(bt, "_add_to_graph") as mock_add: + with patch.object(bt, "add_to_graph") as mock_add: result = bt._filter_deps_satisfied_by_build_system( {extras_req}, resolved_build_sys, From a134502768472fe1c31e009e8562aea60defa8f8 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 1 Jul 2026 21:59:28 -0400 Subject: [PATCH 06/19] refactor(bootstrapper): inline _create_build_env; make create_unresolved_work_items public Inline the thin wrapper _create_build_env() directly in PrepareSourceItem.run() since its req and resolved_version parameters were unused. This simplifies the code and removes an unnecessary layer of indirection. Rename _create_unresolved_work_items() to create_unresolved_work_items() to make it public, as it is called from three different PhaseItem subclasses (PrepareSourceItem, PrepareBuildItem, ProcessInstallDepsItem). Update all call sites in bootstrapper.py and test references in test_bootstrapper_iterative.py. Co-Authored-By: Claude Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper.py | 27 +++--------- tests/test_bootstrapper_iterative.py | 66 ++++++++++++++++------------ 2 files changed, 45 insertions(+), 48 deletions(-) diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index 1428a1fb..cf131594 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -729,9 +729,8 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: wi.sdist_root_dir = sdist_root_dir wi.unpack_dir = sdist_root_dir.parent - wi.build_env = bt._create_build_env( - req=wi.req, - resolved_version=wi.resolved_version, + wi.build_env = build_environment.BuildEnvironment( + ctx=bt.ctx, parent_dir=sdist_root_dir.parent, ) @@ -743,7 +742,7 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: sdist_root_dir=sdist_root_dir, ) - dep_items = bt._create_unresolved_work_items( + dep_items = bt.create_unresolved_work_items( wi.build_system_deps, RequirementType.BUILD_SYSTEM, wi.req, @@ -812,13 +811,13 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: parent, ) - backend_items = bt._create_unresolved_work_items( + backend_items = bt.create_unresolved_work_items( wi.build_backend_deps, RequirementType.BUILD_BACKEND, wi.req, wi.resolved_version, ) - sdist_items = bt._create_unresolved_work_items( + sdist_items = bt.create_unresolved_work_items( wi.build_sdist_deps, RequirementType.BUILD_SDIST, wi.req, @@ -957,7 +956,7 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: constraint=constraint, ) - dep_items = bt._create_unresolved_work_items( + dep_items = bt.create_unresolved_work_items( install_dependencies, RequirementType.INSTALL, wi.req, @@ -1651,18 +1650,6 @@ def _prepare_source( ) return result - def _create_build_env( - self, - req: Requirement, - resolved_version: Version, - parent_dir: pathlib.Path, - ) -> build_environment.BuildEnvironment: - """Create isolated build environment.""" - return build_environment.BuildEnvironment( - ctx=self.ctx, - parent_dir=parent_dir, - ) - def _do_build( self, req: Requirement, @@ -2053,7 +2040,7 @@ def _drain_background_pool(self) -> None: max_workers=self._num_bg_threads, thread_name_prefix="fromager-bg" ) - def _create_unresolved_work_items( + def create_unresolved_work_items( self, deps: typing.Iterable[Requirement], dep_req_type: RequirementType, diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index 4ecdc7a7..0edf82ea 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -4,7 +4,7 @@ - PhaseItem class hierarchy: class variables and base class behavior - WorkItem dataclass defaults and state accumulation - _track_why context manager behavior -- _create_unresolved_work_items helper +- create_unresolved_work_items helper - ResolveItem.run() version expansion - StartItem.run() graph addition and seen-check - PrepareSourceItem.run() all branches (prebuilt, source, cached, bad path) @@ -300,7 +300,7 @@ def test_creates_resolve_phase_items(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) deps = [Requirement("dep-a"), Requirement("dep-b")] - items = bt._create_unresolved_work_items( + items = bt.create_unresolved_work_items( deps, RequirementType.BUILD_SYSTEM, Requirement("parent"), Version("1.0") ) @@ -317,7 +317,7 @@ def test_captures_why_snapshot(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) bt.why = [(RequirementType.TOP_LEVEL, Requirement("root"), Version("2.0"))] - items = bt._create_unresolved_work_items( + items = bt.create_unresolved_work_items( [Requirement("dep")], RequirementType.INSTALL, Requirement("parent"), @@ -334,7 +334,7 @@ def test_sorts_by_name(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) deps = [Requirement("zebra"), Requirement("alpha"), Requirement("middle")] - items = bt._create_unresolved_work_items( + items = bt.create_unresolved_work_items( deps, RequirementType.INSTALL, Requirement("p"), Version("1.0") ) @@ -343,7 +343,7 @@ def test_sorts_by_name(self, tmp_context: WorkContext) -> None: def test_empty_deps(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) - items = bt._create_unresolved_work_items( + items = bt.create_unresolved_work_items( [], RequirementType.INSTALL, Requirement("p"), Version("1.0") ) assert items == [] @@ -1203,15 +1203,16 @@ def test_source_no_cache_uses_background_result( with ( patch.object(tmp_context.constraints, "get_constraint", return_value=None), - patch.object( - bt, "_create_build_env", return_value=mock_env - ) as mock_create_env, + patch( + "fromager.build_environment.BuildEnvironment", + return_value=mock_env, + ) as mock_build_env_cls, patch( "fromager.dependencies.get_build_system_dependencies", return_value={Requirement("setuptools")}, ), patch.object( - bt, "_create_unresolved_work_items", return_value=[mock_dep_item] + bt, "create_unresolved_work_items", return_value=[mock_dep_item] ) as mock_create_items, ): result = item.run(bt) @@ -1224,7 +1225,10 @@ def test_source_no_cache_uses_background_result( assert wi.unpack_dir == sdist_root.parent assert wi.cached_wheel_filename is None assert result[1] is mock_dep_item - mock_create_env.assert_called_once() + mock_build_env_cls.assert_called_once_with( + ctx=bt.ctx, + parent_dir=sdist_root.parent, + ) mock_create_items.assert_called_once_with( wi.build_system_deps, RequirementType.BUILD_SYSTEM, @@ -1252,12 +1256,15 @@ def test_source_cached_wheel_uses_background_result( with ( patch.object(tmp_context.constraints, "get_constraint", return_value=None), - patch.object(bt, "_create_build_env", return_value=mock_env), + patch( + "fromager.build_environment.BuildEnvironment", + return_value=mock_env, + ), patch( "fromager.dependencies.get_build_system_dependencies", return_value=set(), ), - patch.object(bt, "_create_unresolved_work_items", return_value=[]), + patch.object(bt, "create_unresolved_work_items", return_value=[]), ): result = item.run(bt) @@ -1300,12 +1307,15 @@ def test_constraint_logged_when_present( "get_constraint", return_value=Requirement("testpkg>=1.0"), ), - patch.object(bt, "_create_build_env", return_value=mock_env), + patch( + "fromager.build_environment.BuildEnvironment", + return_value=mock_env, + ), patch( "fromager.dependencies.get_build_system_dependencies", return_value=set(), ), - patch.object(bt, "_create_unresolved_work_items", return_value=[]), + patch.object(bt, "create_unresolved_work_items", return_value=[]), ): result = item.run(bt) @@ -1346,7 +1356,7 @@ def test_installs_system_deps_and_returns_backend_sdist_items( ), patch.object( bt, - "_create_unresolved_work_items", + "create_unresolved_work_items", side_effect=[[backend_item], [sdist_item]], ), ): @@ -1383,7 +1393,7 @@ def test_no_extra_deps_returns_item_only(self, tmp_context: WorkContext) -> None ), patch.object( bt, - "_create_unresolved_work_items", + "create_unresolved_work_items", side_effect=[[], []], ), ): @@ -1419,7 +1429,7 @@ def test_install_called_once_with_system_deps( ), patch.object( bt, - "_create_unresolved_work_items", + "create_unresolved_work_items", side_effect=[[], []], ), ): @@ -1453,7 +1463,7 @@ def test_creates_items_with_correct_requirement_types( ), patch.object( bt, - "_create_unresolved_work_items", + "create_unresolved_work_items", side_effect=[[], []], ) as mock_create_items, ): @@ -1709,7 +1719,7 @@ def test_normal_path_returns_item_and_dep_items( patch.object(tmp_context.constraints, "get_constraint", return_value=None), patch.object(bt, "_add_to_build_order") as mock_build_order, patch.object( - bt, "_create_unresolved_work_items", return_value=[dep_item] + bt, "create_unresolved_work_items", return_value=[dep_item] ) as mock_create_items, ): result = item.run(bt) @@ -1747,7 +1757,7 @@ def test_hook_error_test_mode_records_and_continues( ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), patch.object(bt, "_add_to_build_order") as mock_build_order, - patch.object(bt, "_create_unresolved_work_items", return_value=[]), + patch.object(bt, "create_unresolved_work_items", return_value=[]), ): result = item.run(bt) @@ -1794,7 +1804,7 @@ def test_dep_extraction_error_test_mode_uses_empty_deps( patch.object(tmp_context.constraints, "get_constraint", return_value=None), patch.object(bt, "_add_to_build_order") as mock_build_order, patch.object( - bt, "_create_unresolved_work_items", return_value=[] + bt, "create_unresolved_work_items", return_value=[] ) as mock_create_items, ): result = item.run(bt) @@ -1843,7 +1853,7 @@ def test_no_install_deps_returns_item_only(self, tmp_context: WorkContext) -> No ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), patch.object(bt, "_add_to_build_order"), - patch.object(bt, "_create_unresolved_work_items", return_value=[]), + patch.object(bt, "create_unresolved_work_items", return_value=[]), ): result = item.run(bt) @@ -1873,7 +1883,7 @@ def test_build_order_called_with_correct_args( return_value=constraint, ), patch.object(bt, "_add_to_build_order") as mock_build_order, - patch.object(bt, "_create_unresolved_work_items", return_value=[]), + patch.object(bt, "create_unresolved_work_items", return_value=[]), ): item.run(bt) @@ -2216,7 +2226,7 @@ def test_satisfied_backend_dep_skips_resolve( ), patch.object( bt, - "_create_unresolved_work_items", + "create_unresolved_work_items", return_value=[], ) as mock_create, ): @@ -2272,7 +2282,7 @@ def test_extras_dep_not_filtered(self, tmp_context: WorkContext) -> None: ), patch.object( bt, - "_create_unresolved_work_items", + "create_unresolved_work_items", side_effect=[[resolve_item], []], ) as mock_create, ): @@ -2319,7 +2329,7 @@ def test_unsatisfied_backend_dep_creates_resolve_item( ), patch.object( bt, - "_create_unresolved_work_items", + "create_unresolved_work_items", side_effect=[[resolve_item], []], ) as mock_create, ): @@ -2446,12 +2456,12 @@ def test_push_items_no_bg_future_when_pool_is_none( class TestCreateUnresolvedWorkItemsReturnType: - """_create_unresolved_work_items returns ResolveItem instances.""" + """create_unresolved_work_items returns ResolveItem instances.""" def test_returns_resolve_items(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) deps = [Requirement("dep-a")] - items = bt._create_unresolved_work_items( + items = bt.create_unresolved_work_items( deps, RequirementType.INSTALL, Requirement("parent"), Version("1.0") ) assert len(items) == 1 From 6271db894a682a74193028e9491f2113c102bedc Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 2 Jul 2026 07:26:32 -0400 Subject: [PATCH 07/19] refactor(bootstrapper): make PrepareBuildItem-accessed members public Remove leading underscores from `_get_resolved_build_system_versions` and `_filter_deps_satisfied_by_build_system` to make them public methods. These methods are accessed from `PrepareBuildItem.run()` across module boundaries and should be part of the public API. Co-Authored-By: Claude Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper.py | 14 +++++++------- tests/test_bootstrapper_iterative.py | 24 ++++++++++++------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index cf131594..ba883e1a 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -796,15 +796,15 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: # Filter out deps already satisfied by build-system dependencies # to avoid resolving to a different (typically newer) version. - resolved_build_sys = bt._get_resolved_build_system_versions(wi) + resolved_build_sys = bt.get_resolved_build_system_versions(wi) parent = (wi.req, wi.resolved_version) - wi.build_backend_deps = bt._filter_deps_satisfied_by_build_system( + wi.build_backend_deps = bt.filter_deps_satisfied_by_build_system( wi.build_backend_deps, resolved_build_sys, RequirementType.BUILD_BACKEND, parent, ) - wi.build_sdist_deps = bt._filter_deps_satisfied_by_build_system( + wi.build_sdist_deps = bt.filter_deps_satisfied_by_build_system( wi.build_sdist_deps, resolved_build_sys, RequirementType.BUILD_SDIST, @@ -1501,12 +1501,12 @@ def _prepare_build_dependencies( resolved_build_sys = self._resolve_build_system_versions_by_name( build_system_dependencies, ) - build_backend_dependencies = self._filter_deps_satisfied_by_build_system( + build_backend_dependencies = self.filter_deps_satisfied_by_build_system( build_backend_dependencies, resolved_build_sys, RequirementType.BUILD_BACKEND, ) - build_sdist_dependencies = self._filter_deps_satisfied_by_build_system( + build_sdist_dependencies = self.filter_deps_satisfied_by_build_system( build_sdist_dependencies, resolved_build_sys, RequirementType.BUILD_SDIST, @@ -2084,7 +2084,7 @@ def _resolve_build_system_versions_by_name( break return resolved - def _get_resolved_build_system_versions( + def get_resolved_build_system_versions( self, item: WorkItem, ) -> dict[NormalizedName, tuple[Version, str]]: @@ -2112,7 +2112,7 @@ def _get_resolved_build_system_versions( ) return resolved - def _filter_deps_satisfied_by_build_system( + def filter_deps_satisfied_by_build_system( self, deps: set[Requirement], resolved_build_sys: dict[NormalizedName, tuple[Version, str]], diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index 0edf82ea..89a71557 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -1944,7 +1944,7 @@ def test_satisfied_dep_reuses_build_system_version( parent = (Requirement("biotite==1.6.0"), Version("1.6.0")) with patch.object(bt, "add_to_graph") as mock_add: - result = bt._filter_deps_satisfied_by_build_system( + result = bt.filter_deps_satisfied_by_build_system( {Requirement("hatch-cython")}, resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -1971,7 +1971,7 @@ def test_satisfied_dep_with_compatible_specifier( parent = (Requirement("testpkg==1.0"), Version("1.0")) with patch.object(bt, "add_to_graph") as mock_add: - result = bt._filter_deps_satisfied_by_build_system( + result = bt.filter_deps_satisfied_by_build_system( {Requirement("foo>=1.0")}, resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -1991,7 +1991,7 @@ def test_unsatisfied_dep_passes_through(self, tmp_context: WorkContext) -> None: wheel_req = Requirement("wheel") with patch.object(bt, "add_to_graph") as mock_add: - result = bt._filter_deps_satisfied_by_build_system( + result = bt.filter_deps_satisfied_by_build_system( {wheel_req}, resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -2013,7 +2013,7 @@ def test_incompatible_specifier_passes_through( foo_req = Requirement("foo>=2.0") with patch.object(bt, "add_to_graph") as mock_add: - result = bt._filter_deps_satisfied_by_build_system( + result = bt.filter_deps_satisfied_by_build_system( {foo_req}, resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -2034,7 +2034,7 @@ def test_incompatible_specifier_logs_warning( parent = (Requirement("testpkg==1.0"), Version("1.0")) with patch.object(bt, "add_to_graph"): - bt._filter_deps_satisfied_by_build_system( + bt.filter_deps_satisfied_by_build_system( {Requirement("foo>=2.0")}, resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -2059,7 +2059,7 @@ def test_mixed_satisfied_and_unsatisfied(self, tmp_context: WorkContext) -> None wheel_req = Requirement("wheel") with patch.object(bt, "add_to_graph") as mock_add: - result = bt._filter_deps_satisfied_by_build_system( + result = bt.filter_deps_satisfied_by_build_system( {cython_req, wheel_req}, resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -2077,7 +2077,7 @@ def test_empty_deps_returns_empty(self, tmp_context: WorkContext) -> None: } parent = (Requirement("testpkg==1.0"), Version("1.0")) - result = bt._filter_deps_satisfied_by_build_system( + result = bt.filter_deps_satisfied_by_build_system( set(), resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -2094,7 +2094,7 @@ def test_empty_build_system_returns_all_deps( wheel_req = Requirement("wheel") parent = (Requirement("testpkg==1.0"), Version("1.0")) - result = bt._filter_deps_satisfied_by_build_system( + result = bt.filter_deps_satisfied_by_build_system( {wheel_req}, {}, RequirementType.BUILD_BACKEND, @@ -2113,7 +2113,7 @@ def test_dep_with_extras_passes_through(self, tmp_context: WorkContext) -> None: extras_req = Requirement("foo[bar]>=1.0") with patch.object(bt, "add_to_graph") as mock_add: - result = bt._filter_deps_satisfied_by_build_system( + result = bt.filter_deps_satisfied_by_build_system( {extras_req}, resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -2125,7 +2125,7 @@ def test_dep_with_extras_passes_through(self, tmp_context: WorkContext) -> None: class TestGetResolvedBuildSystemVersions: - """Tests for _get_resolved_build_system_versions.""" + """Tests for get_resolved_build_system_versions.""" def test_returns_build_system_edges(self, tmp_context: WorkContext) -> None: """Returns resolved versions from BUILD_SYSTEM edges.""" @@ -2157,7 +2157,7 @@ def test_returns_build_system_edges(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) wi = _make_work_item(req="biotite", version="1.6.0") - result = bt._get_resolved_build_system_versions(wi) + result = bt.get_resolved_build_system_versions(wi) assert canonicalize_name("hatch-cython") in result assert result[canonicalize_name("hatch-cython")] == ( @@ -2173,7 +2173,7 @@ def test_returns_empty_when_parent_not_in_graph( bt = bootstrapper.Bootstrapper(tmp_context) wi = _make_work_item(req="nonexistent", version="1.0") - result = bt._get_resolved_build_system_versions(wi) + result = bt.get_resolved_build_system_versions(wi) assert result == {} From fc0085175591a0cece951463fdbc05374980709f Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 2 Jul 2026 08:27:43 -0400 Subject: [PATCH 08/19] refactor(bootstrapper): make BuildItem-accessed members public Rename _drain_background_pool() to drain_background_pool() and _do_build() to do_build() to indicate these methods are accessed from BuildItem.run(). Internal methods _build_sdist() and _build_wheel() remain private as they are only called within Bootstrapper. Co-Authored-By: Claude Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper.py | 8 ++++---- tests/test_bootstrapper_iterative.py | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index ba883e1a..ba0eb6d2 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -849,14 +849,14 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: pbi = bt.ctx.package_build_info(wi.req) if pbi.exclusive_build: logger.info("%s requires exclusive build, draining background pool", wi.req) - bt._drain_background_pool() + bt.drain_background_pool() # Install backend+sdist deps if disjoint from system deps remaining_deps = wi.build_backend_deps | wi.build_sdist_deps if remaining_deps.isdisjoint(wi.build_system_deps): wi.build_env.install(remaining_deps) - wheel_filename, sdist_filename = bt._do_build( + wheel_filename, sdist_filename = bt.do_build( req=wi.req, resolved_version=wi.resolved_version, sdist_root_dir=wi.sdist_root_dir, @@ -1650,7 +1650,7 @@ def _prepare_source( ) return result - def _do_build( + def do_build( self, req: Requirement, resolved_version: Version, @@ -2027,7 +2027,7 @@ def _push_items(self, stack: list[PhaseItem], items: list[PhaseItem]) -> None: if bg_work is not None: item.bg_future = self._bg_pool.submit(bg_work) - def _drain_background_pool(self) -> None: + def drain_background_pool(self) -> None: """Drain all in-flight background tasks and recreate the pool. Used as an exclusive-build barrier: ensures all background I/O completes diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index 89a71557..fad4add3 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -1519,7 +1519,7 @@ def test_disjoint_deps_installs_remaining(self, tmp_context: WorkContext) -> Non mock_wheel = tmp_context.work_dir / "testpkg-1.0-py3-none-any.whl" with ( - patch.object(bt, "_do_build", return_value=(mock_wheel, None)), + patch.object(bt, "do_build", return_value=(mock_wheel, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): result = item.run(bt) @@ -1546,7 +1546,7 @@ def test_overlapping_deps_skips_install(self, tmp_context: WorkContext) -> None: ) with ( - patch.object(bt, "_do_build", return_value=(None, None)), + patch.object(bt, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): item.run(bt) @@ -1569,7 +1569,7 @@ def test_partial_overlap_deps_skips_install(self, tmp_context: WorkContext) -> N ) with ( - patch.object(bt, "_do_build", return_value=(None, None)), + patch.object(bt, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): item.run(bt) @@ -1577,7 +1577,7 @@ def test_partial_overlap_deps_skips_install(self, tmp_context: WorkContext) -> N mock_env.install.assert_not_called() def test_do_build_receives_item_fields(self, tmp_context: WorkContext) -> None: - """build_sdist_only and cached_wheel_filename are forwarded to _do_build.""" + """build_sdist_only and cached_wheel_filename are forwarded to do_build.""" bt = bootstrapper.Bootstrapper(tmp_context) mock_env = Mock() sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" @@ -1593,7 +1593,7 @@ def test_do_build_receives_item_fields(self, tmp_context: WorkContext) -> None: wi = item.work_item with ( - patch.object(bt, "_do_build", return_value=(None, None)) as mock_do_build, + patch.object(bt, "do_build", return_value=(None, None)) as mock_do_build, patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): item.run(bt) @@ -1616,7 +1616,7 @@ def test_build_result_references_item_build_env( wi = item.work_item with ( - patch.object(bt, "_do_build", return_value=(None, None)), + patch.object(bt, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): item.run(bt) @@ -1633,7 +1633,7 @@ def test_build_result_uses_source_type_from_sources( wi = item.work_item with ( - patch.object(bt, "_do_build", return_value=(None, None)), + patch.object(bt, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.PREBUILT), ): item.run(bt) @@ -1649,7 +1649,7 @@ def test_returns_single_process_install_deps_item( item = self._make_build_phase_item(tmp_context) with ( - patch.object(bt, "_do_build", return_value=(None, None)), + patch.object(bt, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): result = item.run(bt) @@ -1667,7 +1667,7 @@ def test_phase_advancement_preserves_work_item_identity( wi = item.work_item with ( - patch.object(bt, "_do_build", return_value=(None, None)), + patch.object(bt, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): result = item.run(bt) From 8aa247c9ba2021d69a9ba7915bd148d212f6ccfd Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 2 Jul 2026 10:07:37 -0400 Subject: [PATCH 09/19] refactor(bootstrapper): make ProcessInstallDepsItem-accessed members public Rename three private methods to public to support ProcessInstallDepsItem.run() accessing them without underscore prefix: - _record_test_mode_failure() -> record_test_mode_failure() - _get_install_dependencies() -> get_install_dependencies() - _add_to_build_order() -> add_to_build_order() All internal callers and test references have been updated accordingly. Co-Authored-By: Claude Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper.py | 24 ++++++++++++------------ tests/test_bootstrapper.py | 18 +++++++++--------- tests/test_bootstrapper_iterative.py | 24 ++++++++++++------------ 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index ba0eb6d2..1c56d3e2 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -909,7 +909,7 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: except Exception as hook_error: if not bt.test_mode: raise - bt._record_test_mode_failure( + bt.record_test_mode_failure( wi.req, str(wi.resolved_version), hook_error, @@ -919,7 +919,7 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: # Extract install dependencies (non-fatal in test mode) try: - install_dependencies = bt._get_install_dependencies( + install_dependencies = bt.get_install_dependencies( req=wi.req, resolved_version=wi.resolved_version, wheel_filename=wi.build_result.wheel_filename, @@ -931,7 +931,7 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: except Exception as dep_error: if not bt.test_mode: raise - bt._record_test_mode_failure( + bt.record_test_mode_failure( wi.req, str(wi.resolved_version), dep_error, @@ -947,7 +947,7 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: pbi = bt.ctx.package_build_info(wi.req) constraint = bt.ctx.constraints.get_constraint(wi.req.name) - bt._add_to_build_order( + bt.add_to_build_order( req=wi.req, version=wi.resolved_version, source_url=wi.source_url, @@ -1115,7 +1115,7 @@ def _resolve_and_add_top_level( req, "unresolved", err, "no versions resolved, skipping" ) if self.test_mode: - self._record_test_mode_failure(req, None, err, "resolution") + self.record_test_mode_failure(req, None, err, "resolution") return None raise RuntimeError(f"Could not resolve any versions for {req}") @@ -1125,11 +1125,11 @@ def _resolve_and_add_top_level( if self.multiple_versions: self._record_failed_version(req, "unresolved", err, "failed to resolve") if self.test_mode: - self._record_test_mode_failure(req, None, err, "resolution") + self.record_test_mode_failure(req, None, err, "resolution") return None if not self.test_mode: raise - self._record_test_mode_failure(req, None, err, "resolution") + self.record_test_mode_failure(req, None, err, "resolution") return None def resolve_versions( @@ -1360,7 +1360,7 @@ def _track_why( finally: self.why.pop() - def _record_test_mode_failure( + def record_test_mode_failure( self, req: Requirement, version: str | None, @@ -1570,7 +1570,7 @@ def _download_prebuilt( assert result.unpack_dir is not None return (result.wheel_filename, result.unpack_dir) - def _get_install_dependencies( + def get_install_dependencies( self, req: Requirement, resolved_version: Version, @@ -1964,7 +1964,7 @@ def has_been_seen( typ: typing.Literal["sdist", "wheel"] = "sdist" if sdist_only else "wheel" return self._resolved_key(req, version, typ) in self._seen_requirements - def _add_to_build_order( + def add_to_build_order( self, req: Requirement, version: Version, @@ -2182,7 +2182,7 @@ def _handle_phase_error( # Resolution failures: recoverable in test mode and multiple versions mode if isinstance(item, ResolveItem): if self.test_mode: - self._record_test_mode_failure(wi.req, None, err, "resolution") + self.record_test_mode_failure(wi.req, None, err, "resolution") if self.multiple_versions: self._record_failed_version( wi.req, @@ -2217,7 +2217,7 @@ def _handle_phase_error( if fallback is not None: wi.build_result = fallback return [ProcessInstallDepsItem(wi)] - self._record_test_mode_failure( + self.record_test_mode_failure( wi.req, str(wi.resolved_version), err, "bootstrap" ) return [] diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index 57ac6ed5..f718da00 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -74,13 +74,13 @@ def test_seen_requirements_sdist(tmp_context: WorkContext) -> None: def test_build_order(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) - bt._add_to_build_order( + bt.add_to_build_order( req=Requirement("buildme>1.0"), version=Version("6.0"), source_url="url", source_type=SourceType.SDIST, ) - bt._add_to_build_order( + bt.add_to_build_order( req=Requirement("testdist>1.0"), version=Version("1.2"), source_url="url", @@ -113,19 +113,19 @@ def test_build_order(tmp_context: WorkContext) -> None: def test_build_order_repeats(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) - bt._add_to_build_order( + bt.add_to_build_order( Requirement("buildme>1.0"), Version("6.0"), "url", SourceType.SDIST, ) - bt._add_to_build_order( + bt.add_to_build_order( Requirement("buildme>1.0"), Version("6.0"), "url", SourceType.SDIST, ) - bt._add_to_build_order( + bt.add_to_build_order( Requirement("buildme[extra]>1.0"), Version("6.0"), "url", @@ -149,13 +149,13 @@ def test_build_order_repeats(tmp_context: WorkContext) -> None: def test_build_order_name_canonicalization(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) - bt._add_to_build_order( + bt.add_to_build_order( Requirement("flit-core>1.0"), Version("3.9.0"), "url", SourceType.SDIST, ) - bt._add_to_build_order( + bt.add_to_build_order( Requirement("flit_core>1.0"), Version("3.9.0"), "url", @@ -241,14 +241,14 @@ def test_find_cached_wheel_returns_tuple(tmp_context: WorkContext) -> None: def test_get_install_dependencies_returns_list( mock_get_deps: Mock, tmp_context: WorkContext ) -> None: - """Verify _get_install_dependencies returns list.""" + """Verify get_install_dependencies returns list.""" bt = bootstrapper.Bootstrapper(tmp_context) # Create fake wheel file and mock dependencies wheel_file = pathlib.Path("/fake/package-1.0.0-py3-none-any.whl") unpack_dir = tmp_context.work_dir - result = bt._get_install_dependencies( + result = bt.get_install_dependencies( req=Requirement("test-package"), resolved_version=Version("1.0.0"), wheel_filename=wheel_file, diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index fad4add3..168a7091 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -1708,7 +1708,7 @@ def test_normal_path_returns_item_and_dep_items( patch("fromager.hooks.run_post_bootstrap_hooks"), patch.object( bt, - "_get_install_dependencies", + "get_install_dependencies", return_value=[Requirement("dep-a")], ), patch.object( @@ -1717,7 +1717,7 @@ def test_normal_path_returns_item_and_dep_items( return_value=Mock(pre_built=False), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), - patch.object(bt, "_add_to_build_order") as mock_build_order, + patch.object(bt, "add_to_build_order") as mock_build_order, patch.object( bt, "create_unresolved_work_items", return_value=[dep_item] ) as mock_create_items, @@ -1748,7 +1748,7 @@ def test_hook_error_test_mode_records_and_continues( side_effect=RuntimeError("hook failed"), ), patch.object( - bt, "_get_install_dependencies", return_value=[] + bt, "get_install_dependencies", return_value=[] ) as mock_get_deps, patch.object( tmp_context, @@ -1756,7 +1756,7 @@ def test_hook_error_test_mode_records_and_continues( return_value=Mock(pre_built=False), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), - patch.object(bt, "_add_to_build_order") as mock_build_order, + patch.object(bt, "add_to_build_order") as mock_build_order, patch.object(bt, "create_unresolved_work_items", return_value=[]), ): result = item.run(bt) @@ -1793,7 +1793,7 @@ def test_dep_extraction_error_test_mode_uses_empty_deps( patch("fromager.hooks.run_post_bootstrap_hooks"), patch.object( bt, - "_get_install_dependencies", + "get_install_dependencies", side_effect=RuntimeError("dep failed"), ), patch.object( @@ -1802,7 +1802,7 @@ def test_dep_extraction_error_test_mode_uses_empty_deps( return_value=Mock(pre_built=False), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), - patch.object(bt, "_add_to_build_order") as mock_build_order, + patch.object(bt, "add_to_build_order") as mock_build_order, patch.object( bt, "create_unresolved_work_items", return_value=[] ) as mock_create_items, @@ -1831,7 +1831,7 @@ def test_dep_extraction_error_normal_mode_raises( patch("fromager.hooks.run_post_bootstrap_hooks"), patch.object( bt, - "_get_install_dependencies", + "get_install_dependencies", side_effect=RuntimeError("dep failed"), ), ): @@ -1845,14 +1845,14 @@ def test_no_install_deps_returns_item_only(self, tmp_context: WorkContext) -> No with ( patch("fromager.hooks.run_post_bootstrap_hooks"), - patch.object(bt, "_get_install_dependencies", return_value=[]), + patch.object(bt, "get_install_dependencies", return_value=[]), patch.object( tmp_context, "package_build_info", return_value=Mock(pre_built=False), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), - patch.object(bt, "_add_to_build_order"), + patch.object(bt, "add_to_build_order"), patch.object(bt, "create_unresolved_work_items", return_value=[]), ): result = item.run(bt) @@ -1863,7 +1863,7 @@ def test_no_install_deps_returns_item_only(self, tmp_context: WorkContext) -> No def test_build_order_called_with_correct_args( self, tmp_context: WorkContext ) -> None: - """_add_to_build_order receives correct source_type, prebuilt, constraint.""" + """add_to_build_order receives correct source_type, prebuilt, constraint.""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_process_item(tmp_context) wi = item.work_item @@ -1871,7 +1871,7 @@ def test_build_order_called_with_correct_args( with ( patch("fromager.hooks.run_post_bootstrap_hooks"), - patch.object(bt, "_get_install_dependencies", return_value=[]), + patch.object(bt, "get_install_dependencies", return_value=[]), patch.object( tmp_context, "package_build_info", @@ -1882,7 +1882,7 @@ def test_build_order_called_with_correct_args( "get_constraint", return_value=constraint, ), - patch.object(bt, "_add_to_build_order") as mock_build_order, + patch.object(bt, "add_to_build_order") as mock_build_order, patch.object(bt, "create_unresolved_work_items", return_value=[]), ): item.run(bt) From 4a1269c6feb068f6ab2e7c8c716add2758644a83 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 2 Jul 2026 10:22:31 -0400 Subject: [PATCH 10/19] refactor(bootstrapper): move exclusive-build pool drain into the loop Each PhaseItem now declares whether it requires exclusive execution via a `requires_exclusive_run` property (default False). `_run_bootstrap_loop` drains the background pool before dispatching any item that returns True, rather than having `BuildItem.run()` reach into the Bootstrapper to do it. `BuildItem` overrides `requires_exclusive_run` to return `self.work_item.exclusive_build`, which is cached on `WorkItem` during `StartItem.run()` alongside `pbi_pre_built`. `drain_background_pool` is now private again since it's only called from within the Bootstrapper loop. Co-Authored-By: Claude Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index 1c56d3e2..a9ca07a6 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -161,6 +161,7 @@ class WorkItem: cached_wheel_filename: pathlib.Path | None = None build_result: SourceBuildResult | None = None pbi_pre_built: bool = False + exclusive_build: bool = False build_system_deps: set[Requirement] = dataclasses.field(default_factory=set) build_backend_deps: set[Requirement] = dataclasses.field(default_factory=set) build_sdist_deps: set[Requirement] = dataclasses.field(default_factory=set) @@ -436,6 +437,15 @@ def __init__(self, work_item: WorkItem) -> None: @abc.abstractmethod def run(self, bt: Bootstrapper) -> list[PhaseItem]: ... + @property + def requires_exclusive_run(self) -> bool: + """Return True if this item must run without concurrent background I/O. + + When True, the bootstrap loop drains the background thread pool before + calling ``run()``. Override in subclasses that require exclusive access. + """ + return False + def background_work( self, bt: Bootstrapper ) -> typing.Callable[[], typing.Any] | None: @@ -635,7 +645,9 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: # Must set pbi_pre_built before constructing PrepareSourceItem so that # PrepareSourceItem.background_work() immediately sees the correct value. - wi.pbi_pre_built = bt.ctx.package_build_info(wi.req).pre_built + pbi = bt.ctx.package_build_info(wi.req) + wi.pbi_pre_built = pbi.pre_built + wi.exclusive_build = pbi.exclusive_build return [PrepareSourceItem(wi)] @@ -834,6 +846,10 @@ class BuildItem(PhaseItem): phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.BUILD tracks_why: typing.ClassVar[bool] = True + @property + def requires_exclusive_run(self) -> bool: + return self.work_item.exclusive_build + def run(self, bt: Bootstrapper) -> list[PhaseItem]: """BUILD phase: install remaining deps, build wheel/sdist. @@ -845,12 +861,6 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: assert wi.build_env is not None assert wi.sdist_root_dir is not None - # Drain all in-flight background I/O before an exclusive build starts. - pbi = bt.ctx.package_build_info(wi.req) - if pbi.exclusive_build: - logger.info("%s requires exclusive build, draining background pool", wi.req) - bt.drain_background_pool() - # Install backend+sdist deps if disjoint from system deps remaining_deps = wi.build_backend_deps | wi.build_sdist_deps if remaining_deps.isdisjoint(wi.build_system_deps): @@ -1245,6 +1255,13 @@ def _run_bootstrap_loop(self, stack: list[PhaseItem]) -> None: item = stack.pop() self.why = list(item.work_item.why_snapshot) + if item.requires_exclusive_run: + logger.info( + "%s requires exclusive build, draining background pool", + item.work_item.req, + ) + self._drain_background_pool() + with ( req_ctxvar_context(item.work_item.req, item.work_item.resolved_version), self._track_why(item), @@ -2027,7 +2044,7 @@ def _push_items(self, stack: list[PhaseItem], items: list[PhaseItem]) -> None: if bg_work is not None: item.bg_future = self._bg_pool.submit(bg_work) - def drain_background_pool(self) -> None: + def _drain_background_pool(self) -> None: """Drain all in-flight background tasks and recreate the pool. Used as an exclusive-build barrier: ensures all background I/O completes From fbb652d0f5d1c1136a5d52684269bc1e69bf470a Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 2 Jul 2026 10:27:23 -0400 Subject: [PATCH 11/19] refactor(bootstrapper): convert get_install_dependencies to module-level function The method only used `self.ctx` from the Bootstrapper; all other arguments were passed explicitly. It now follows the same pattern as `_bg_resolve`, `_bg_prepare_source`, and `_find_cached_wheel`: a module-level function that takes `ctx` directly. Co-Authored-By: Claude Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper.py | 104 ++++++++++++++------------- tests/test_bootstrapper.py | 7 +- tests/test_bootstrapper_iterative.py | 23 +++--- 3 files changed, 66 insertions(+), 68 deletions(-) diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index a9ca07a6..7b4551d1 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -419,6 +419,57 @@ def _bg_prepare_prebuilt( return PreparedSourceData(wheel_filename=wheel_filename, unpack_dir=unpack_dir) +def _get_install_dependencies( + ctx: context.WorkContext, + req: Requirement, + resolved_version: Version, + wheel_filename: pathlib.Path | None, + sdist_filename: pathlib.Path | None, + sdist_root_dir: pathlib.Path | None, + build_env: build_environment.BuildEnvironment | None, + unpack_dir: pathlib.Path | None, +) -> list[Requirement]: + """Extract install dependencies from a built wheel or sdist. + + Returns: + List of install requirements. + + Raises: + RuntimeError: If both wheel_filename and sdist_filename are None. + """ + if wheel_filename is not None: + assert unpack_dir is not None + logger.debug( + "get install dependencies of wheel %s", + wheel_filename.name, + ) + return list( + dependencies.get_install_dependencies_of_wheel( + req=req, + wheel_filename=wheel_filename, + requirements_file_dir=unpack_dir, + ) + ) + elif sdist_filename is not None: + assert sdist_root_dir is not None + assert build_env is not None + logger.debug( + "get install dependencies of sdist from directory %s", + sdist_root_dir, + ) + return list( + dependencies.get_install_dependencies_of_sdist( + ctx=ctx, + req=req, + version=resolved_version, + sdist_root_dir=sdist_root_dir, + build_env=build_env, + ) + ) + else: + raise RuntimeError("wheel_filename and sdist_filename are None") + + class PhaseItem(abc.ABC): """Abstract base for items pushed onto the bootstrap stack. @@ -929,7 +980,8 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: # Extract install dependencies (non-fatal in test mode) try: - install_dependencies = bt.get_install_dependencies( + install_dependencies = _get_install_dependencies( + ctx=bt.ctx, req=wi.req, resolved_version=wi.resolved_version, wheel_filename=wi.build_result.wheel_filename, @@ -1587,56 +1639,6 @@ def _download_prebuilt( assert result.unpack_dir is not None return (result.wheel_filename, result.unpack_dir) - def get_install_dependencies( - self, - req: Requirement, - resolved_version: Version, - wheel_filename: pathlib.Path | None, - sdist_filename: pathlib.Path | None, - sdist_root_dir: pathlib.Path | None, - build_env: build_environment.BuildEnvironment | None, - unpack_dir: pathlib.Path | None, - ) -> list[Requirement]: - """Extract install dependencies from wheel or sdist. - - Returns: - List of install requirements. - - Raises: - RuntimeError: If both wheel_filename and sdist_filename are None. - """ - if wheel_filename is not None: - assert unpack_dir is not None - logger.debug( - "get install dependencies of wheel %s", - wheel_filename.name, - ) - return list( - dependencies.get_install_dependencies_of_wheel( - req=req, - wheel_filename=wheel_filename, - requirements_file_dir=unpack_dir, - ) - ) - elif sdist_filename is not None: - assert sdist_root_dir is not None - assert build_env is not None - logger.debug( - "get install dependencies of sdist from directory %s", - sdist_root_dir, - ) - return list( - dependencies.get_install_dependencies_of_sdist( - ctx=self.ctx, - req=req, - version=resolved_version, - sdist_root_dir=sdist_root_dir, - build_env=build_env, - ) - ) - else: - raise RuntimeError("wheel_filename and sdist_filename are None") - def _download_source( self, req: Requirement, diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index f718da00..3574b04d 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -241,14 +241,13 @@ def test_find_cached_wheel_returns_tuple(tmp_context: WorkContext) -> None: def test_get_install_dependencies_returns_list( mock_get_deps: Mock, tmp_context: WorkContext ) -> None: - """Verify get_install_dependencies returns list.""" - bt = bootstrapper.Bootstrapper(tmp_context) - + """Verify _get_install_dependencies returns list.""" # Create fake wheel file and mock dependencies wheel_file = pathlib.Path("/fake/package-1.0.0-py3-none-any.whl") unpack_dir = tmp_context.work_dir - result = bt.get_install_dependencies( + result = bootstrapper._get_install_dependencies( + ctx=tmp_context, req=Requirement("test-package"), resolved_version=Version("1.0.0"), wheel_filename=wheel_file, diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index 168a7091..06015ebc 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -1706,9 +1706,8 @@ def test_normal_path_returns_item_and_dep_items( with ( patch("fromager.hooks.run_post_bootstrap_hooks"), - patch.object( - bt, - "get_install_dependencies", + patch( + "fromager.bootstrapper._get_install_dependencies", return_value=[Requirement("dep-a")], ), patch.object( @@ -1747,8 +1746,8 @@ def test_hook_error_test_mode_records_and_continues( "fromager.hooks.run_post_bootstrap_hooks", side_effect=RuntimeError("hook failed"), ), - patch.object( - bt, "get_install_dependencies", return_value=[] + patch( + "fromager.bootstrapper._get_install_dependencies", return_value=[] ) as mock_get_deps, patch.object( tmp_context, @@ -1791,9 +1790,8 @@ def test_dep_extraction_error_test_mode_uses_empty_deps( with ( patch("fromager.hooks.run_post_bootstrap_hooks"), - patch.object( - bt, - "get_install_dependencies", + patch( + "fromager.bootstrapper._get_install_dependencies", side_effect=RuntimeError("dep failed"), ), patch.object( @@ -1829,9 +1827,8 @@ def test_dep_extraction_error_normal_mode_raises( with ( patch("fromager.hooks.run_post_bootstrap_hooks"), - patch.object( - bt, - "get_install_dependencies", + patch( + "fromager.bootstrapper._get_install_dependencies", side_effect=RuntimeError("dep failed"), ), ): @@ -1845,7 +1842,7 @@ def test_no_install_deps_returns_item_only(self, tmp_context: WorkContext) -> No with ( patch("fromager.hooks.run_post_bootstrap_hooks"), - patch.object(bt, "get_install_dependencies", return_value=[]), + patch("fromager.bootstrapper._get_install_dependencies", return_value=[]), patch.object( tmp_context, "package_build_info", @@ -1871,7 +1868,7 @@ def test_build_order_called_with_correct_args( with ( patch("fromager.hooks.run_post_bootstrap_hooks"), - patch.object(bt, "get_install_dependencies", return_value=[]), + patch("fromager.bootstrapper._get_install_dependencies", return_value=[]), patch.object( tmp_context, "package_build_info", From 34d49092750c4cdb9603b475c0333d2dcdbdc8c0 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 2 Jul 2026 16:28:32 -0400 Subject: [PATCH 12/19] refactor(bootstrapper): move processing_build_requirement to WorkItem The method only needed `self.req_type` and `self.why` (which equals `wi.why_snapshot` at call time in the bootstrap loop). It is now `WorkItem.is_build_requirement_context()`, reading `self.req_type` and `self.why_snapshot` directly with no parameters. Co-Authored-By: Claude Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper.py | 55 +++++++++++++++---------------- tests/test_bootstrapper.py | 64 ++++++++++++++++++++++++++---------- 2 files changed, 72 insertions(+), 47 deletions(-) diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index 7b4551d1..d9f1a691 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -166,6 +166,31 @@ class WorkItem: build_backend_deps: set[Requirement] = dataclasses.field(default_factory=set) build_sdist_deps: set[Requirement] = dataclasses.field(default_factory=set) + def is_build_requirement_context(self) -> bool: + """Return True if this item is being processed as part of a build requirement. + + A package is a build dependency if its own requirement type is + build_system, build_backend, or build_sdist, OR if it is an install + requirement of something that is itself a build dependency (checked + by walking the ``why_snapshot`` chain). + """ + if self.req_type.is_build_requirement: + logger.debug(f"is itself a build requirement: {self.req_type}") + return True + if not self.req_type.is_install_requirement: + logger.debug( + "is not an install requirement, not checking dependency chain for a build requirement" + ) + return False + for req_type, req, resolved_version in reversed(self.why_snapshot): + if req_type.is_build_requirement: + logger.debug( + f"is a build requirement because {req_type} dependency {req} ({resolved_version}) depends on it" + ) + return True + logger.debug("is not a build requirement") + return False + def _create_unpack_dir( work_dir: pathlib.Path, @@ -677,9 +702,7 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: wi.parent, ) - wi.build_sdist_only = bt.sdist_only and not bt.processing_build_requirement( - wi.req_type - ) + wi.build_sdist_only = bt.sdist_only and not wi.is_build_requirement_context() if bt.has_been_seen(wi.req, wi.resolved_version, wi.build_sdist_only): logger.debug( @@ -1331,32 +1354,6 @@ def _run_bootstrap_loop(self, stack: list[PhaseItem]) -> None: self._push_items(stack, new_items) - def processing_build_requirement(self, current_req_type: RequirementType) -> bool: - """Are we currently processing a build requirement? - - We determine that a package is a build dependency if its requirement - type is build_system, build_backend, or build_sdist OR if it is an - installation requirement of something that is a build dependency. We - use a verbose loop to determine the status so we can log the reason - something is treated as a build dependency. - """ - if current_req_type.is_build_requirement: - logger.debug(f"is itself a build requirement: {current_req_type}") - return True - if not current_req_type.is_install_requirement: - logger.debug( - "is not an install requirement, not checking dependency chain for a build requirement" - ) - return False - for req_type, req, resolved_version in reversed(self.why): - if req_type.is_build_requirement: - logger.debug( - f"is a build requirement because {req_type} dependency {req} ({resolved_version}) depends on it" - ) - return True - logger.debug("is not a build requirement") - return False - def _bootstrap_one(self, req: Requirement, req_type: RequirementType) -> None: """Bootstrap a single requirement using an iterative DFS loop. diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index 3574b04d..ddcf5120 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -195,29 +195,57 @@ def test_explain(tmp_context: WorkContext) -> None: ) -def test_is_build_requirement(tmp_context: WorkContext) -> None: - bt = bootstrapper.Bootstrapper(tmp_context) - bt.why = [] - assert not bt.processing_build_requirement(RequirementType.TOP_LEVEL) - assert bt.processing_build_requirement(RequirementType.BUILD_SYSTEM) - assert bt.processing_build_requirement(RequirementType.BUILD_BACKEND) - assert bt.processing_build_requirement(RequirementType.BUILD_SDIST) - assert not bt.processing_build_requirement(RequirementType.INSTALL) +def _make_work_item( + req_type: RequirementType, + why_snapshot: list[tuple[RequirementType, Requirement, Version]] | None = None, +) -> bootstrapper.WorkItem: + return bootstrapper.WorkItem( + req=Requirement("testpkg"), + req_type=req_type, + why_snapshot=why_snapshot or [], + ) - bt.why = [(RequirementType.TOP_LEVEL, Requirement("foo"), Version("1.0.0"))] - assert not bt.processing_build_requirement(RequirementType.INSTALL) - assert bt.processing_build_requirement(RequirementType.BUILD_SYSTEM) - assert bt.processing_build_requirement(RequirementType.BUILD_BACKEND) - assert bt.processing_build_requirement(RequirementType.BUILD_SDIST) - bt.why = [ +def test_is_build_requirement(tmp_context: WorkContext) -> None: + # No ancestry: req_type alone determines result + assert not _make_work_item(RequirementType.TOP_LEVEL).is_build_requirement_context() + assert _make_work_item(RequirementType.BUILD_SYSTEM).is_build_requirement_context() + assert _make_work_item(RequirementType.BUILD_BACKEND).is_build_requirement_context() + assert _make_work_item(RequirementType.BUILD_SDIST).is_build_requirement_context() + assert not _make_work_item(RequirementType.INSTALL).is_build_requirement_context() + + # TOP_LEVEL ancestry: install is still not a build requirement + top_level_why = [(RequirementType.TOP_LEVEL, Requirement("foo"), Version("1.0.0"))] + assert not _make_work_item( + RequirementType.INSTALL, top_level_why + ).is_build_requirement_context() + assert _make_work_item( + RequirementType.BUILD_SYSTEM, top_level_why + ).is_build_requirement_context() + assert _make_work_item( + RequirementType.BUILD_BACKEND, top_level_why + ).is_build_requirement_context() + assert _make_work_item( + RequirementType.BUILD_SDIST, top_level_why + ).is_build_requirement_context() + + # BUILD_SYSTEM in ancestry: install becomes a build requirement + build_why = [ (RequirementType.TOP_LEVEL, Requirement("foo"), Version("1.0.0")), (RequirementType.BUILD_SYSTEM, Requirement("bar==4.0.0"), Version("4.0.0")), ] - assert bt.processing_build_requirement(RequirementType.INSTALL) - assert bt.processing_build_requirement(RequirementType.BUILD_SYSTEM) - assert bt.processing_build_requirement(RequirementType.BUILD_BACKEND) - assert bt.processing_build_requirement(RequirementType.BUILD_SDIST) + assert _make_work_item( + RequirementType.INSTALL, build_why + ).is_build_requirement_context() + assert _make_work_item( + RequirementType.BUILD_SYSTEM, build_why + ).is_build_requirement_context() + assert _make_work_item( + RequirementType.BUILD_BACKEND, build_why + ).is_build_requirement_context() + assert _make_work_item( + RequirementType.BUILD_SDIST, build_why + ).is_build_requirement_context() def test_find_cached_wheel_returns_tuple(tmp_context: WorkContext) -> None: From 1bd7e108d7ea06bccdf4f115bff384b2bb13eeac Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 2 Jul 2026 17:20:09 -0400 Subject: [PATCH 13/19] refactor(bootstrapper): move do_build and helpers to BuildItem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `do_build()`, `_build_sdist()`, and `_build_wheel()` on `Bootstrapper` are only called from `BuildItem.run()` and operate exclusively on `WorkItem` state. Moving them to `BuildItem` eliminates the need to pass `req`, `resolved_version`, and other `WorkItem` fields as parameters: the methods read them from `self.work_item` directly. `BuildItem.run()` now calls `self.do_build(bt.ctx, bt.explain)` instead of `bt.do_build(req=wi.req, ...)`. The only external state needed from `Bootstrapper` is: - `ctx` (WorkContext) — passed explicitly - `explain` (formatted why-stack string used in one log line) — passed explicitly Existing tests that patched `bt.do_build` or `bt._build_wheel` are updated to patch the corresponding `BuildItem` instance methods instead. New unit tests cover the three `BuildItem` methods directly. Co-Authored-By: Claude Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper.py | 177 ++++++++++++++------------- tests/test_bootstrapper.py | 174 +++++++++++++++++++++++++- tests/test_bootstrapper_iterative.py | 28 ++--- 3 files changed, 275 insertions(+), 104 deletions(-) diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index d9f1a691..edddfa8d 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -940,14 +940,7 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: if remaining_deps.isdisjoint(wi.build_system_deps): wi.build_env.install(remaining_deps) - wheel_filename, sdist_filename = bt.do_build( - req=wi.req, - resolved_version=wi.resolved_version, - sdist_root_dir=wi.sdist_root_dir, - build_env=wi.build_env, - build_sdist_only=wi.build_sdist_only, - cached_wheel_filename=wi.cached_wheel_filename, - ) + wheel_filename, sdist_filename = self.do_build(bt.ctx, bt.explain) source_type = sources.get_source_type(bt.ctx, wi.req) @@ -962,6 +955,97 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: return [ProcessInstallDepsItem(wi)] + def _build_sdist(self, ctx: context.WorkContext) -> pathlib.Path: + """Build or locate an sdist for this item's package. + + Checks ``ctx.sdists_builds`` for an existing sdist before invoking + ``sources.build_sdist()``. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.sdist_root_dir is not None + assert wi.build_env is not None + find_sdist_result = finders.find_sdist( + ctx, ctx.sdists_builds, wi.req, str(wi.resolved_version) + ) + if not find_sdist_result: + sdist_filename: pathlib.Path = sources.build_sdist( + ctx=ctx, + req=wi.req, + version=wi.resolved_version, + sdist_root_dir=wi.sdist_root_dir, + build_env=wi.build_env, + ) + else: + sdist_filename = find_sdist_result + logger.info( + f"have sdist version {wi.resolved_version}: {find_sdist_result}" + ) + return sdist_filename + + def _build_wheel( + self, + ctx: context.WorkContext, + explain: str = "", + ) -> tuple[pathlib.Path, pathlib.Path]: + """Build a wheel for this item's package. + + Calls :meth:`_build_sdist` first, then invokes ``wheels.build_wheel()`` + and updates the local wheel mirror. Returns ``(wheel_filename, + sdist_filename)`` where *wheel_filename* is the path in + ``ctx.wheels_downloads`` after the mirror update. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.sdist_root_dir is not None + assert wi.build_env is not None + sdist_filename = self._build_sdist(ctx) + + logger.info(f"starting build of {explain} for {ctx.variant}") + built_filename = wheels.build_wheel( + ctx=ctx, + req=wi.req, + sdist_root_dir=wi.sdist_root_dir, + version=wi.resolved_version, + build_env=wi.build_env, + ) + server.update_wheel_mirror(ctx) + # When we update the mirror, the built file moves to the downloads directory. + wheel_filename = ctx.wheels_downloads / built_filename.name + logger.info(f"built wheel for version {wi.resolved_version}: {wheel_filename}") + return wheel_filename, sdist_filename + + def do_build( + self, + ctx: context.WorkContext, + explain: str = "", + ) -> tuple[pathlib.Path | None, pathlib.Path | None]: + """Build a wheel or sdist for this item's package. + + Returns ``(wheel_filename, sdist_filename)``. Either value may be + ``None`` depending on the build path: + + - Cached wheel: ``(cached_wheel_filename, None)`` + - sdist-only: ``(None, sdist_filename)`` + - Full wheel: ``(wheel_filename, sdist_filename)`` + """ + wi = self.work_item + if wi.cached_wheel_filename: + logger.debug( + f"getting install requirements from cached wheel {wi.cached_wheel_filename.name}" + ) + return wi.cached_wheel_filename, None + elif wi.build_sdist_only: + logger.debug( + f"getting install requirements from sdist {wi.req.name}=={wi.resolved_version}" + ) + return None, self._build_sdist(ctx) + else: + logger.debug( + f"building wheel {wi.req.name}=={wi.resolved_version} to get install requirements" + ) + return self._build_wheel(ctx, explain) + class ProcessInstallDepsItem(PhaseItem): """PROCESS_INSTALL_DEPS phase: hooks, extract deps, build order.""" @@ -1468,55 +1552,6 @@ def explain(self) -> str: for req_type, req, resolved_version in reversed(self.why) ) - def _build_sdist( - self, - req: Requirement, - resolved_version: Version, - sdist_root_dir: pathlib.Path, - build_env: build_environment.BuildEnvironment, - ) -> pathlib.Path: - find_sdist_result = finders.find_sdist( - self.ctx, self.ctx.sdists_builds, req, str(resolved_version) - ) - if not find_sdist_result: - sdist_filename: pathlib.Path = sources.build_sdist( - ctx=self.ctx, - req=req, - version=resolved_version, - sdist_root_dir=sdist_root_dir, - build_env=build_env, - ) - else: - sdist_filename = find_sdist_result - logger.info(f"have sdist version {resolved_version}: {find_sdist_result}") - return sdist_filename - - def _build_wheel( - self, - req: Requirement, - resolved_version: Version, - sdist_root_dir: pathlib.Path, - build_env: build_environment.BuildEnvironment, - ) -> tuple[pathlib.Path, pathlib.Path]: - sdist_filename = self._build_sdist( - req, resolved_version, sdist_root_dir, build_env - ) - - logger.info(f"starting build of {self.explain} for {self.ctx.variant}") - built_filename = wheels.build_wheel( - ctx=self.ctx, - req=req, - sdist_root_dir=sdist_root_dir, - version=resolved_version, - build_env=build_env, - ) - server.update_wheel_mirror(self.ctx) - # When we update the mirror, the built file moves to the - # downloads directory. - wheel_filename = self.ctx.wheels_downloads / built_filename.name - logger.info(f"built wheel for version {resolved_version}: {wheel_filename}") - return wheel_filename, sdist_filename - def _prepare_build_dependencies( self, req: Requirement, @@ -1666,34 +1701,6 @@ def _prepare_source( ) return result - def do_build( - self, - req: Requirement, - resolved_version: Version, - sdist_root_dir: pathlib.Path, - build_env: build_environment.BuildEnvironment, - build_sdist_only: bool, - cached_wheel_filename: pathlib.Path | None, - ) -> tuple[pathlib.Path | None, pathlib.Path | None]: - """Build wheel or sdist from prepared source.""" - if cached_wheel_filename: - logger.debug( - f"getting install requirements from cached wheel {cached_wheel_filename.name}" - ) - return cached_wheel_filename, None - elif build_sdist_only: - logger.debug( - f"getting install requirements from sdist {req.name}=={resolved_version}" - ) - return None, self._build_sdist( - req, resolved_version, sdist_root_dir, build_env - ) - else: - logger.debug( - f"building wheel {req.name}=={resolved_version} to get install requirements" - ) - return self._build_wheel(req, resolved_version, sdist_root_dir, build_env) - def _handle_test_mode_failure( self, req: Requirement, diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index ddcf5120..ea62506c 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -319,7 +319,7 @@ def test_phase_build_produces_source_build_result(tmp_context: WorkContext) -> N with ( patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), - patch.object(bt, "_build_wheel", return_value=(mock_wheel, None)), + patch.object(item, "_build_wheel", return_value=(mock_wheel, None)), ): with bt._track_why(item): result_items = item.run(bt) @@ -937,3 +937,175 @@ def test_bg_prepare_prebuilt_log_prefix_includes_version( assert msg.startswith("mypkg-1.2.3: "), ( f"Expected 'mypkg-1.2.3: ' prefix, got: {msg!r}" ) + + +def test_build_item_build_sdist_finds_existing(tmp_context: WorkContext) -> None: + """BuildItem._build_sdist returns cached sdist when finders.find_sdist hits.""" + sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" + sdist_root.mkdir(parents=True, exist_ok=True) + wi = bootstrapper.WorkItem( + req=Requirement("testpkg"), + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + resolved_version=Version("1.0"), + sdist_root_dir=sdist_root, + build_env=Mock(), + ) + item = bootstrapper.BuildItem(wi) + cached = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" + cached.touch() + + with patch("fromager.finders.find_sdist", return_value=cached): + result = item._build_sdist(tmp_context) + + assert result == cached + + +def test_build_item_build_sdist_calls_build_when_not_cached( + tmp_context: WorkContext, +) -> None: + """BuildItem._build_sdist calls sources.build_sdist when no cached sdist found.""" + sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" + sdist_root.mkdir(parents=True, exist_ok=True) + wi = bootstrapper.WorkItem( + req=Requirement("testpkg"), + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + resolved_version=Version("1.0"), + sdist_root_dir=sdist_root, + build_env=Mock(), + ) + item = bootstrapper.BuildItem(wi) + built = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" + + with ( + patch("fromager.finders.find_sdist", return_value=None), + patch("fromager.sources.build_sdist", return_value=built) as mock_build, + ): + result = item._build_sdist(tmp_context) + + mock_build.assert_called_once_with( + ctx=tmp_context, + req=wi.req, + version=wi.resolved_version, + sdist_root_dir=sdist_root, + build_env=wi.build_env, + ) + assert result == built + + +def test_build_item_build_wheel(tmp_context: WorkContext) -> None: + """BuildItem._build_wheel builds sdist then wheel and updates mirror.""" + sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" + sdist_root.mkdir(parents=True, exist_ok=True) + wi = bootstrapper.WorkItem( + req=Requirement("testpkg"), + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + resolved_version=Version("1.0"), + sdist_root_dir=sdist_root, + build_env=Mock(), + ) + item = bootstrapper.BuildItem(wi) + built_wheel = tmp_context.wheels_build / "testpkg-1.0-py3-none-any.whl" + built_sdist = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" + + with ( + patch.object( + item, "_build_sdist", return_value=built_sdist + ) as mock_build_sdist, + patch("fromager.wheels.build_wheel", return_value=built_wheel), + patch("fromager.server.update_wheel_mirror"), + ): + wheel_filename, sdist_filename = item._build_wheel(tmp_context) + + mock_build_sdist.assert_called_once_with(tmp_context) + assert wheel_filename == tmp_context.wheels_downloads / built_wheel.name + assert sdist_filename == built_sdist + + +def test_build_item_do_build_returns_cached_wheel( + tmp_context: WorkContext, +) -> None: + """BuildItem.do_build returns cached wheel immediately without building.""" + cached = tmp_context.wheels_downloads / "testpkg-1.0-py3-none-any.whl" + cached.touch() + wi = bootstrapper.WorkItem( + req=Requirement("testpkg"), + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + resolved_version=Version("1.0"), + build_sdist_only=False, + cached_wheel_filename=cached, + sdist_root_dir=tmp_context.work_dir, + build_env=Mock(), + ) + item = bootstrapper.BuildItem(wi) + + with patch.object(item, "_build_wheel") as mock_wheel: + wheel, sdist = item.do_build(tmp_context) + + mock_wheel.assert_not_called() + assert wheel == cached + assert sdist is None + + +def test_build_item_do_build_sdist_only(tmp_context: WorkContext) -> None: + """BuildItem.do_build calls _build_sdist and returns (None, sdist) when build_sdist_only=True.""" + built_sdist = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" + sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" + sdist_root.mkdir(parents=True, exist_ok=True) + wi = bootstrapper.WorkItem( + req=Requirement("testpkg"), + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + resolved_version=Version("1.0"), + build_sdist_only=True, + cached_wheel_filename=None, + sdist_root_dir=sdist_root, + build_env=Mock(), + ) + item = bootstrapper.BuildItem(wi) + + with ( + patch.object(item, "_build_sdist", return_value=built_sdist) as mock_sdist, + patch.object(item, "_build_wheel") as mock_wheel, + ): + wheel, sdist = item.do_build(tmp_context) + + mock_wheel.assert_not_called() + mock_sdist.assert_called_once_with(tmp_context) + assert wheel is None + assert sdist == built_sdist + + +def test_build_item_do_build_builds_wheel(tmp_context: WorkContext) -> None: + """BuildItem.do_build calls _build_wheel when no cache and not sdist_only.""" + built_wheel = tmp_context.wheels_downloads / "testpkg-1.0-py3-none-any.whl" + built_sdist = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" + sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" + sdist_root.mkdir(parents=True, exist_ok=True) + wi = bootstrapper.WorkItem( + req=Requirement("testpkg"), + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + resolved_version=Version("1.0"), + build_sdist_only=False, + cached_wheel_filename=None, + sdist_root_dir=sdist_root, + build_env=Mock(), + ) + item = bootstrapper.BuildItem(wi) + + with patch.object( + item, "_build_wheel", return_value=(built_wheel, built_sdist) + ) as mock_wheel: + wheel, sdist = item.do_build( + tmp_context, explain="top_level dependency testpkg (1.0)" + ) + + mock_wheel.assert_called_once_with( + tmp_context, "top_level dependency testpkg (1.0)" + ) + assert wheel == built_wheel + assert sdist == built_sdist diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index 06015ebc..c17434d5 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -1519,7 +1519,7 @@ def test_disjoint_deps_installs_remaining(self, tmp_context: WorkContext) -> Non mock_wheel = tmp_context.work_dir / "testpkg-1.0-py3-none-any.whl" with ( - patch.object(bt, "do_build", return_value=(mock_wheel, None)), + patch.object(item, "do_build", return_value=(mock_wheel, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): result = item.run(bt) @@ -1546,7 +1546,7 @@ def test_overlapping_deps_skips_install(self, tmp_context: WorkContext) -> None: ) with ( - patch.object(bt, "do_build", return_value=(None, None)), + patch.object(item, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): item.run(bt) @@ -1569,7 +1569,7 @@ def test_partial_overlap_deps_skips_install(self, tmp_context: WorkContext) -> N ) with ( - patch.object(bt, "do_build", return_value=(None, None)), + patch.object(item, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): item.run(bt) @@ -1577,7 +1577,7 @@ def test_partial_overlap_deps_skips_install(self, tmp_context: WorkContext) -> N mock_env.install.assert_not_called() def test_do_build_receives_item_fields(self, tmp_context: WorkContext) -> None: - """build_sdist_only and cached_wheel_filename are forwarded to do_build.""" + """build_sdist_only and cached_wheel_filename are read directly by do_build, not forwarded as parameters.""" bt = bootstrapper.Bootstrapper(tmp_context) mock_env = Mock() sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" @@ -1590,22 +1590,14 @@ def test_do_build_receives_item_fields(self, tmp_context: WorkContext) -> None: build_sdist_only=True, cached_wheel_filename=cached_wheel, ) - wi = item.work_item with ( - patch.object(bt, "do_build", return_value=(None, None)) as mock_do_build, + patch.object(item, "do_build", return_value=(None, None)) as mock_do_build, patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): item.run(bt) - mock_do_build.assert_called_once_with( - req=wi.req, - resolved_version=wi.resolved_version, - sdist_root_dir=sdist_root, - build_env=mock_env, - build_sdist_only=True, - cached_wheel_filename=cached_wheel, - ) + mock_do_build.assert_called_once_with(bt.ctx, bt.explain) def test_build_result_references_item_build_env( self, tmp_context: WorkContext @@ -1616,7 +1608,7 @@ def test_build_result_references_item_build_env( wi = item.work_item with ( - patch.object(bt, "do_build", return_value=(None, None)), + patch.object(item, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): item.run(bt) @@ -1633,7 +1625,7 @@ def test_build_result_uses_source_type_from_sources( wi = item.work_item with ( - patch.object(bt, "do_build", return_value=(None, None)), + patch.object(item, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.PREBUILT), ): item.run(bt) @@ -1649,7 +1641,7 @@ def test_returns_single_process_install_deps_item( item = self._make_build_phase_item(tmp_context) with ( - patch.object(bt, "do_build", return_value=(None, None)), + patch.object(item, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): result = item.run(bt) @@ -1667,7 +1659,7 @@ def test_phase_advancement_preserves_work_item_identity( wi = item.work_item with ( - patch.object(bt, "do_build", return_value=(None, None)), + patch.object(item, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): result = item.run(bt) From 568c002d56b4a664be124febd46901940dcfc798 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 2 Jul 2026 18:32:47 -0400 Subject: [PATCH 14/19] refactor(bootstrapper): convert bootstrapper.py to a subpackage Split the 2,351-line `bootstrapper.py` into a subpackage `src/fromager/bootstrapper/` with one module per logical layer: - `_types.py`: type aliases, dataclasses, enums, and constants - `_cache.py`: shared wheel-cache and prebuilt-download helpers - `_work_item.py`: `WorkItem` dataclass - `_phase_item.py`: `PhaseItem` ABC - `_resolve_item.py`: `ResolveItem` + `_bg_resolve` - `_start_item.py`: `StartItem` - `_prepare_source_item.py`: `PrepareSourceItem` + `_bg_prepare_source` - `_prepare_build_item.py`: `PrepareBuildItem` - `_build_item.py`: `BuildItem` - `_process_install_deps_item.py`: `ProcessInstallDepsItem` + `_get_install_dependencies` - `_complete_item.py`: `CompleteItem` - `_bootstrapper.py`: `Bootstrapper` class - `__init__.py`: exports `Bootstrapper` and `_DEFAULT_BG_THREADS` only Phase modules that call `_cache` helpers use module-level imports (`from . import _cache`) so mock patch targets remain unambiguous (e.g. `fromager.bootstrapper._cache._find_cached_wheel`). Tests updated to import directly from submodules and use the new patch paths. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper/__init__.py | 4 + .../_bootstrapper.py} | 1151 +---------------- src/fromager/bootstrapper/_build_item.py | 149 +++ src/fromager/bootstrapper/_cache.py | 227 ++++ src/fromager/bootstrapper/_complete_item.py | 30 + src/fromager/bootstrapper/_phase_item.py | 85 ++ .../bootstrapper/_prepare_build_item.py | 91 ++ .../bootstrapper/_prepare_source_item.py | 165 +++ .../_process_install_deps_item.py | 160 +++ src/fromager/bootstrapper/_resolve_item.py | 144 +++ src/fromager/bootstrapper/_start_item.py | 68 + src/fromager/bootstrapper/_types.py | 96 ++ src/fromager/bootstrapper/_work_item.py | 77 ++ tests/test_bootstrapper.py | 140 +- tests/test_bootstrapper_iterative.py | 62 +- 15 files changed, 1434 insertions(+), 1215 deletions(-) create mode 100644 src/fromager/bootstrapper/__init__.py rename src/fromager/{bootstrapper.py => bootstrapper/_bootstrapper.py} (53%) create mode 100644 src/fromager/bootstrapper/_build_item.py create mode 100644 src/fromager/bootstrapper/_cache.py create mode 100644 src/fromager/bootstrapper/_complete_item.py create mode 100644 src/fromager/bootstrapper/_phase_item.py create mode 100644 src/fromager/bootstrapper/_prepare_build_item.py create mode 100644 src/fromager/bootstrapper/_prepare_source_item.py create mode 100644 src/fromager/bootstrapper/_process_install_deps_item.py create mode 100644 src/fromager/bootstrapper/_resolve_item.py create mode 100644 src/fromager/bootstrapper/_start_item.py create mode 100644 src/fromager/bootstrapper/_types.py create mode 100644 src/fromager/bootstrapper/_work_item.py diff --git a/src/fromager/bootstrapper/__init__.py b/src/fromager/bootstrapper/__init__.py new file mode 100644 index 00000000..fda5fb83 --- /dev/null +++ b/src/fromager/bootstrapper/__init__.py @@ -0,0 +1,4 @@ +from ._bootstrapper import Bootstrapper +from ._types import _DEFAULT_BG_THREADS + +__all__ = ["_DEFAULT_BG_THREADS", "Bootstrapper"] diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py similarity index 53% rename from src/fromager/bootstrapper.py rename to src/fromager/bootstrapper/_bootstrapper.py index edddfa8d..b653397c 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -1,1160 +1,52 @@ from __future__ import annotations -import abc import concurrent.futures import contextlib -import dataclasses import datetime import json import logging import operator -import os import pathlib import shutil import tempfile import typing -import zipfile -from enum import StrEnum -from urllib.parse import urlparse -import requests.exceptions from packaging.requirements import Requirement from packaging.utils import NormalizedName, canonicalize_name from packaging.version import Version -from resolvelib.resolvers import ResolverException -from . import ( +from .. import ( bootstrap_requirement_resolver, build_environment, dependencies, - finders, gitutils, - hooks, progress, - resolver, - server, sources, - threading_utils, - wheels, ) -from .dependency_graph import DependencyGraph -from .log import req_ctxvar_context, requirement_ctxvar -from .requirements_file import RequirementType, SourceType +from ..dependency_graph import DependencyGraph +from ..log import req_ctxvar_context, requirement_ctxvar +from ..requirements_file import RequirementType, SourceType +from . import _cache +from ._build_item import BuildItem +from ._phase_item import PhaseItem +from ._prepare_build_item import PrepareBuildItem +from ._prepare_source_item import PrepareSourceItem +from ._process_install_deps_item import ProcessInstallDepsItem +from ._resolve_item import ResolveItem +from ._types import ( + _DEFAULT_BG_THREADS, + FailureRecord, + FailureType, + SeenKey, + SourceBuildResult, +) +from ._work_item import WorkItem if typing.TYPE_CHECKING: - from . import context + from .. import context logger = logging.getLogger(__name__) -# package name, extras, version, sdist/wheel -SeenKey = tuple[NormalizedName, tuple[str, ...], str, typing.Literal["sdist", "wheel"]] - -_DEFAULT_BG_THREADS: int = max(1, threading_utils.get_cpu_count() // 2) - - -@dataclasses.dataclass -class SourceBuildResult: - """Result of building or downloading a package. - - Captures the output artifacts from either a source build or - prebuilt wheel download, used across bootstrap phases. - """ - - wheel_filename: pathlib.Path | None - sdist_filename: pathlib.Path | None - unpack_dir: pathlib.Path - sdist_root_dir: pathlib.Path | None - build_env: build_environment.BuildEnvironment | None - source_type: SourceType - - -@dataclasses.dataclass -class PreparedSourceData: - """Result of background I/O pre-fetching returned to the main thread. - - Fields are set in one of three combinations depending on the result type: - - - Source (no cache hit): only ``sdist_root_dir`` is set. - - Source (cache hit): both ``sdist_root_dir`` and ``cached_wheel_filename`` are set. - - Prebuilt wheel: both ``wheel_filename`` and ``unpack_dir`` are set. - """ - - # Source path: set after download+unpack OR cache hit - sdist_root_dir: pathlib.Path | None = None - # Source path: set when the result came from the wheel cache - cached_wheel_filename: pathlib.Path | None = None - # Prebuilt path: downloaded wheel file - wheel_filename: pathlib.Path | None = None - # Prebuilt path: unpack directory (created by mkdir) - unpack_dir: pathlib.Path | None = None - - -# Valid failure types for test mode error recording -FailureType = typing.Literal["resolution", "bootstrap", "hook", "dependency_extraction"] - - -class FailureRecord(typing.TypedDict): - """Record of a package that failed during bootstrap in test mode. - - Attributes: - package: The package name that failed. - version: The resolved version (None if resolution failed). - exception_type: The exception class name. - exception_message: The exception message string. - failure_type: Category of failure for analysis. - """ - - package: str - version: str | None - exception_type: str - exception_message: str - failure_type: FailureType - - -class BootstrapPhase(StrEnum): - """Processing phases for iterative bootstrap. - - All packages: RESOLVE -> START -> ... - Source packages: ... -> PREPARE_SOURCE -> PREPARE_BUILD -> BUILD - -> PROCESS_INSTALL_DEPS -> COMPLETE. - Prebuilt packages: ... -> PREPARE_SOURCE -> PROCESS_INSTALL_DEPS -> COMPLETE. - """ - - RESOLVE = "resolve" - START = "start" - PREPARE_SOURCE = "prepare-source" - PREPARE_BUILD = "prepare-build" - BUILD = "build" - PROCESS_INSTALL_DEPS = "process-install-deps" - COMPLETE = "complete" - - -@dataclasses.dataclass -class WorkItem: - """A unit of work in the iterative bootstrap loop. - - Carries identity fields set at creation time and accumulated state - populated across phases as processing advances. The current phase is - encoded by the ``PhaseItem`` subclass wrapping this object. - - Items enter at the RESOLVE phase with only req and req_type set. - The RESOLVE phase populates source_url and resolved_version, then - creates new items at the START phase for each resolved version. - """ - - # Identity (set at creation) - req: Requirement - req_type: RequirementType - why_snapshot: list[tuple[RequirementType, Requirement, Version]] - parent: tuple[Requirement, Version] | None = None - - # Populated by RESOLVE phase (None until then) - source_url: str | None = None - resolved_version: Version | None = None - - build_sdist_only: bool = False - - # Accumulated state (populated during phases) - build_env: build_environment.BuildEnvironment | None = None - sdist_root_dir: pathlib.Path | None = None - unpack_dir: pathlib.Path | None = None - cached_wheel_filename: pathlib.Path | None = None - build_result: SourceBuildResult | None = None - pbi_pre_built: bool = False - exclusive_build: bool = False - build_system_deps: set[Requirement] = dataclasses.field(default_factory=set) - build_backend_deps: set[Requirement] = dataclasses.field(default_factory=set) - build_sdist_deps: set[Requirement] = dataclasses.field(default_factory=set) - - def is_build_requirement_context(self) -> bool: - """Return True if this item is being processed as part of a build requirement. - - A package is a build dependency if its own requirement type is - build_system, build_backend, or build_sdist, OR if it is an install - requirement of something that is itself a build dependency (checked - by walking the ``why_snapshot`` chain). - """ - if self.req_type.is_build_requirement: - logger.debug(f"is itself a build requirement: {self.req_type}") - return True - if not self.req_type.is_install_requirement: - logger.debug( - "is not an install requirement, not checking dependency chain for a build requirement" - ) - return False - for req_type, req, resolved_version in reversed(self.why_snapshot): - if req_type.is_build_requirement: - logger.debug( - f"is a build requirement because {req_type} dependency {req} ({resolved_version}) depends on it" - ) - return True - logger.debug("is not a build requirement") - return False - - -def _create_unpack_dir( - work_dir: pathlib.Path, - req: Requirement, - resolved_version: Version, -) -> pathlib.Path: - unpack_dir = work_dir / f"{req.name}-{resolved_version}" - unpack_dir.mkdir(parents=True, exist_ok=True) - return unpack_dir - - -def _extract_build_reqs_from_wheel( - work_dir: pathlib.Path, - req: Requirement, - resolved_version: Version, - wheel_filename: pathlib.Path, -) -> pathlib.Path | None: - """Extract fromager build requirement files from a wheel archive. - - Looks for files prefixed with `FROMAGER_BUILD_REQ_PREFIX` inside the - wheel's `.dist-info` directory and extracts them to a local unpack - directory. Returns the unpack directory on success, or None if the - files are not present or extraction fails. - """ - dist_name, dist_version, _, _ = wheels.extract_info_from_wheel_file( - req, wheel_filename - ) - unpack_dir = _create_unpack_dir(work_dir, req, resolved_version) - dist_filename = f"{dist_name}-{dist_version}" - dist_info_path = pathlib.Path(f"{dist_filename}.dist-info") - req_filenames: list[str] = [ - dependencies.BUILD_BACKEND_REQ_FILE_NAME, - dependencies.BUILD_SDIST_REQ_FILE_NAME, - dependencies.BUILD_SYSTEM_REQ_FILE_NAME, - ] - try: - with zipfile.ZipFile(wheel_filename) as archive: - for filename in req_filenames: - zipinfo = archive.getinfo( - str( - dist_info_path - / f"{wheels.FROMAGER_BUILD_REQ_PREFIX}-{filename}" - ) - ) - if os.path.isabs(zipinfo.filename) or ".." in zipinfo.filename: - raise ValueError(f"Unsafe path in wheel: {zipinfo.filename}") - zipinfo.filename = filename - output_file = archive.extract(zipinfo, unpack_dir) - logger.info(f"extracted {output_file}") - logger.info(f"extracted build requirements from wheel into {unpack_dir}") - return unpack_dir - except Exception as e: - logger.info(f"could not extract build requirements from wheel: {e}") - for filename in req_filenames: - unpack_dir.joinpath(filename).unlink(missing_ok=True) - return None - - -def _look_for_existing_wheel( - ctx: context.WorkContext, - req: Requirement, - resolved_version: Version, - search_in: pathlib.Path, -) -> tuple[pathlib.Path | None, pathlib.Path | None]: - pbi = ctx.package_build_info(req) - expected_build_tag = pbi.build_tag(resolved_version) - logger.info( - f"looking for existing wheel for version {resolved_version} with build tag {expected_build_tag} in {search_in}" - ) - wheel_filename = finders.find_wheel( - downloads_dir=search_in, - req=req, - dist_version=str(resolved_version), - build_tag=expected_build_tag, - ) - if not wheel_filename: - return None, None - _, _, build_tag, _ = wheels.extract_info_from_wheel_file(req, wheel_filename) - if expected_build_tag and expected_build_tag != build_tag: - logger.info( - f"found wheel for {resolved_version} in {wheel_filename} but build tag does not match. Got {build_tag} but expected {expected_build_tag}" - ) - return None, None - logger.info(f"found existing wheel {wheel_filename}") - build_reqs_dir = _extract_build_reqs_from_wheel( - ctx.work_dir, req, resolved_version, wheel_filename - ) - return wheel_filename, build_reqs_dir - - -def _download_wheel_from_cache( - ctx: context.WorkContext, - cache_wheel_server_url: str | None, - req: Requirement, - resolved_version: Version, -) -> tuple[pathlib.Path | None, pathlib.Path | None]: - if not cache_wheel_server_url: - return None, None - logger.info(f"checking if wheel was already uploaded to {cache_wheel_server_url}") - try: - pinned_req = Requirement(f"{req.name}=={resolved_version}") - provider = finders.PyPICacheProvider( - cache_server_url=cache_wheel_server_url, - constraints=ctx.constraints, - ) - results = resolver.find_all_matching_from_provider(provider, pinned_req) - wheel_url, _ = results[0] - wheelfile_name = pathlib.Path(urlparse(wheel_url).path) - pbi = ctx.package_build_info(req) - expected_build_tag = pbi.build_tag(resolved_version) - logger.info(f"has expected build tag {expected_build_tag}") - changelogs = pbi.get_changelog(resolved_version) - logger.debug(f"has change logs {changelogs}") - - _, _, build_tag, _ = wheels.extract_info_from_wheel_file(req, wheelfile_name) - if expected_build_tag and expected_build_tag != build_tag: - logger.info( - f"found wheel for {resolved_version} in cache but build tag does not match. Got {build_tag} but expected {expected_build_tag}" - ) - return None, None - - cached_wheel = wheels.download_wheel( - req=req, wheel_url=wheel_url, output_directory=ctx.wheels_downloads - ) - if cache_wheel_server_url != ctx.wheel_server_url: - server.update_wheel_mirror(ctx) - logger.info("found built wheel on cache server") - unpack_dir = _extract_build_reqs_from_wheel( - ctx.work_dir, req, resolved_version, cached_wheel - ) - return cached_wheel, unpack_dir - except ResolverException: - logger.info( - f"did not find wheel for {resolved_version} in {cache_wheel_server_url}" - ) - return None, None - except requests.exceptions.RequestException as err: - logger.warning( - f"network error checking wheel cache for {resolved_version} " - f"at {cache_wheel_server_url}: {err}" - ) - return None, None - except Exception as err: - logger.warning( - f"unexpected error checking wheel cache for {resolved_version} " - f"at {cache_wheel_server_url}: {err}" - ) - return None, None - - -def _find_cached_wheel( - ctx: context.WorkContext, - cache_wheel_server_url: str | None, - req: Requirement, - resolved_version: Version, -) -> tuple[pathlib.Path | None, pathlib.Path | None]: - """Look for cached wheel in 3 locations (thread-safe, no Bootstrapper state). - - Checks for cached wheels in order: - 1. wheels_build directory (previously built) - 2. wheels_downloads directory (previously downloaded) - 3. Cache server (remote cache) - - Returns: - Tuple of (cached_wheel_filename, unpacked_cached_wheel). - Both None if no cache hit. - """ - cached_wheel, unpacked = _look_for_existing_wheel( - ctx, req, resolved_version, ctx.wheels_build - ) - if cached_wheel: - return cached_wheel, unpacked - - cached_wheel, unpacked = _look_for_existing_wheel( - ctx, req, resolved_version, ctx.wheels_downloads - ) - if cached_wheel: - return cached_wheel, unpacked - - cached_wheel, unpacked = _download_wheel_from_cache( - ctx, cache_wheel_server_url, req, resolved_version - ) - if cached_wheel: - return cached_wheel, unpacked - - return None, None - - -def _bg_resolve( - bg_resolver: bootstrap_requirement_resolver.BootstrapRequirementResolver, - req: Requirement, - req_type: RequirementType, - parent_req: Requirement | None, - return_all_versions: bool, -) -> list[tuple[str, Version]]: - """Background-safe resolution: no Bootstrapper state accessed.""" - logger.info(f"{BootstrapPhase.RESOLVE} for {req_type} requirement") - return bg_resolver.resolve( - req=req, - req_type=req_type, - parent_req=parent_req, - return_all_versions=return_all_versions, - ) - - -def _bg_prepare_source( - ctx: context.WorkContext, - cache_wheel_server_url: str | None, - req: Requirement, - resolved_version: Version, - source_url: str, -) -> PreparedSourceData: - """Background-safe source download+unpack: no Bootstrapper state accessed.""" - # Thread-safe: _seen_requirements in the main thread prevents the same - # package from being submitted to the thread pool more than once. - # Paths from download_source() and prepare_source() include {name}-{version}, - # making them unique across concurrent threads processing different packages. - logger.info("preparing source") - cached_wheel, unpacked = _find_cached_wheel( - ctx, cache_wheel_server_url, req, resolved_version - ) - if unpacked is not None: - return PreparedSourceData( - sdist_root_dir=unpacked / unpacked.stem, - cached_wheel_filename=cached_wheel, - ) - source_filename = sources.download_source( - ctx=ctx, req=req, version=resolved_version, download_url=source_url - ) - sdist_root_dir = sources.prepare_source( - ctx=ctx, req=req, source_filename=source_filename, version=resolved_version - ) - return PreparedSourceData(sdist_root_dir=sdist_root_dir) - - -def _bg_prepare_prebuilt( - ctx: context.WorkContext, - req: Requirement, - req_type: RequirementType, - resolved_version: Version, - wheel_url: str, -) -> PreparedSourceData: - """Background-safe prebuilt download: no Bootstrapper state accessed.""" - # Thread-safe: paths include {req.name}-{resolved_version} (unique per package), - # mkdir uses exist_ok=True (atomic), and update_wheel_mirror() is already locked. - logger.info(f"using pre-built wheel for {req_type} requirement") - wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt) - unpack_dir = ctx.work_dir / f"{req.name}-{resolved_version}" - unpack_dir.mkdir(parents=True, exist_ok=True) - server.update_wheel_mirror(ctx) - return PreparedSourceData(wheel_filename=wheel_filename, unpack_dir=unpack_dir) - - -def _get_install_dependencies( - ctx: context.WorkContext, - req: Requirement, - resolved_version: Version, - wheel_filename: pathlib.Path | None, - sdist_filename: pathlib.Path | None, - sdist_root_dir: pathlib.Path | None, - build_env: build_environment.BuildEnvironment | None, - unpack_dir: pathlib.Path | None, -) -> list[Requirement]: - """Extract install dependencies from a built wheel or sdist. - - Returns: - List of install requirements. - - Raises: - RuntimeError: If both wheel_filename and sdist_filename are None. - """ - if wheel_filename is not None: - assert unpack_dir is not None - logger.debug( - "get install dependencies of wheel %s", - wheel_filename.name, - ) - return list( - dependencies.get_install_dependencies_of_wheel( - req=req, - wheel_filename=wheel_filename, - requirements_file_dir=unpack_dir, - ) - ) - elif sdist_filename is not None: - assert sdist_root_dir is not None - assert build_env is not None - logger.debug( - "get install dependencies of sdist from directory %s", - sdist_root_dir, - ) - return list( - dependencies.get_install_dependencies_of_sdist( - ctx=ctx, - req=req, - version=resolved_version, - sdist_root_dir=sdist_root_dir, - build_env=build_env, - ) - ) - else: - raise RuntimeError("wheel_filename and sdist_filename are None") - - -class PhaseItem(abc.ABC): - """Abstract base for items pushed onto the bootstrap stack. - - Each subclass encodes one phase of the bootstrap pipeline. - Wraps a ``WorkItem`` (accumulated per-package state) and implements - the processing logic for that phase in ``run()``. - """ - - phase: typing.ClassVar[BootstrapPhase] - tracks_why: typing.ClassVar[bool] = True - - def __init__(self, work_item: WorkItem) -> None: - self.work_item = work_item - self.bg_future: concurrent.futures.Future[typing.Any] | None = None - - @abc.abstractmethod - def run(self, bt: Bootstrapper) -> list[PhaseItem]: ... - - @property - def requires_exclusive_run(self) -> bool: - """Return True if this item must run without concurrent background I/O. - - When True, the bootstrap loop drains the background thread pool before - calling ``run()``. Override in subclasses that require exclusive access. - """ - return False - - def background_work( - self, bt: Bootstrapper - ) -> typing.Callable[[], typing.Any] | None: - """Return a zero-argument callable for background I/O, or None. - - Override in subclasses that need background prefetching. - ``bt`` is provided so subclasses can capture Bootstrapper state - (e.g. resolver, ctx) into the returned closure without storing - a circular reference on the item itself. - """ - return None - - def __str__(self) -> str: - """Human-readable representation: ``"()"``.""" - wi = self.work_item - return f"{type(self).__name__}({wi.req})" - - def as_json(self) -> dict[str, typing.Any]: - """Return a JSON-serialisable dict for stack-state recording.""" - wi = self.work_item - return { - "req": str(wi.req), - "req_type": str(wi.req_type), - "phase": str(self.phase), - "resolved_version": str(wi.resolved_version) - if wi.resolved_version is not None - else None, - "source_url": wi.source_url, - "build_sdist_only": wi.build_sdist_only, - "why": [ - {"req_type": str(rt), "req": str(r), "version": str(v)} - for rt, r, v in wi.why_snapshot - ], - "parent": ( - {"req": str(wi.parent[0]), "version": str(wi.parent[1])} - if wi.parent - else None - ), - "build_system_deps": sorted(str(r) for r in wi.build_system_deps), - "build_backend_deps": sorted(str(r) for r in wi.build_backend_deps), - "build_sdist_deps": sorted(str(r) for r in wi.build_sdist_deps), - } - - -class ResolveItem(PhaseItem): - """RESOLVE phase: resolve versions and expand into StartItems.""" - - phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.RESOLVE - tracks_why: typing.ClassVar[bool] = False - - def background_work( - self, bt: Bootstrapper - ) -> typing.Callable[[], typing.Any] | None: - """Return closure that calls ``_bg_resolve`` in a thread.""" - bg_resolver = bt.resolver - req = self.work_item.req - req_type = self.work_item.req_type - parent_req = ( - self.work_item.why_snapshot[-1][1] if self.work_item.why_snapshot else None - ) - return_all = bt.multiple_versions - - def do_resolve() -> list[tuple[str, Version]]: - with req_ctxvar_context(req): - return _bg_resolve(bg_resolver, req, req_type, parent_req, return_all) - - return do_resolve - - def run(self, bt: Bootstrapper) -> list[PhaseItem]: - """RESOLVE phase: resolve versions and expand into StartItems. - - Centralizes version resolution so all dependencies are expanded - uniformly. In multiple_versions mode, filters out versions that - already failed in this run and versions whose wheels are already - cached to avoid redundant builds and transitive dependency - processing. - - Returns: - One StartItem per resolved version that needs building. - Empty list if all versions are already cached. - """ - assert self.bg_future is not None - # bg_future.result() blocks until the background resolution completes, - # then returns the result or re-raises any exception from the background. - resolved_versions = self.bg_future.result() - if not resolved_versions: - raise RuntimeError( - f"Could not resolve any versions for {self.work_item.req}" - ) - - if bt.multiple_versions: - pkg_name = canonicalize_name(self.work_item.req.name) - resolved_versions = [ - (url, ver) - for url, ver in resolved_versions - if not bt.has_failed_version(pkg_name, ver) - ] - if not resolved_versions: - raise RuntimeError( - f"Could not resolve any versions for {self.work_item.req}" - f" (all candidates failed previously)" - ) - - logger.info( - f"resolved {len(resolved_versions)} version(s) for {self.work_item.req}" - ) - filtered: list[tuple[str, Version]] = [] - for source_url, version in resolved_versions: - cached_wheel, _ = _find_cached_wheel( - bt.ctx, bt.cache_wheel_server_url, self.work_item.req, version - ) - if cached_wheel: - logger.info( - f"{self.work_item.req.name}=={version}: wheel already cached " - f"at {cached_wheel.name}, skipping" - ) - else: - filtered.append((source_url, version)) - if not filtered: - # Always process the highest version (first in - # resolved_versions) so new transitive dependencies - # are discovered even when every wheel is cached. - logger.info( - f"all versions of {self.work_item.req.name} already cached, " - f"keeping highest version {resolved_versions[0][1]} " - f"for dependency discovery" - ) - filtered.append(resolved_versions[0]) - resolved_versions = filtered - - # Build list so highest version ends up on top of the stack - # (last element after extend) and is processed first. - items: list[PhaseItem] = [] - for source_url, version in reversed(resolved_versions): - items.append( - StartItem( - WorkItem( - req=self.work_item.req, - req_type=self.work_item.req_type, - why_snapshot=list(self.work_item.why_snapshot), - parent=self.work_item.parent, - source_url=source_url, - resolved_version=version, - ) - ) - ) - return items - - -class StartItem(PhaseItem): - """START phase: add to graph, check if already seen.""" - - phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.START - tracks_why: typing.ClassVar[bool] = False - - def run(self, bt: Bootstrapper) -> list[PhaseItem]: - """START phase: add to graph, check if already seen. - - _track_why is a no-op for this phase (tracks_why is False), - matching the original behavior where graph addition and - seen-check happen before pushing onto the why stack. - - Returns: - Empty list if already seen (nothing to do). - [PrepareSourceItem] if this is new work. - """ - wi = self.work_item - assert wi.resolved_version is not None - assert wi.source_url is not None - - # Add to graph (skip TOP_LEVEL, already added in _resolve_and_add_top_level) - if wi.req_type != RequirementType.TOP_LEVEL: - bt.add_to_graph( - wi.req, - wi.req_type, - wi.resolved_version, - wi.source_url, - wi.parent, - ) - - wi.build_sdist_only = bt.sdist_only and not wi.is_build_requirement_context() - - if bt.has_been_seen(wi.req, wi.resolved_version, wi.build_sdist_only): - logger.debug( - f"redundant {wi.req_type} dependency {wi.req} " - f"({wi.resolved_version}, sdist_only={wi.build_sdist_only}) " - f"for {bt.explain}" - ) - return [] - bt.mark_as_seen(wi.req, wi.resolved_version, wi.build_sdist_only) - - logger.info( - f"new {wi.req_type} dependency {wi.req} resolves to {wi.resolved_version}" - ) - - # Must set pbi_pre_built before constructing PrepareSourceItem so that - # PrepareSourceItem.background_work() immediately sees the correct value. - pbi = bt.ctx.package_build_info(wi.req) - wi.pbi_pre_built = pbi.pre_built - wi.exclusive_build = pbi.exclusive_build - return [PrepareSourceItem(wi)] - - -class PrepareSourceItem(PhaseItem): - """PREPARE_SOURCE phase: download source or prebuilt, get build system deps.""" - - phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PREPARE_SOURCE - tracks_why: typing.ClassVar[bool] = True - - def background_work( - self, bt: Bootstrapper - ) -> typing.Callable[[], typing.Any] | None: - """Return closure for background source download or prebuilt fetch.""" - wi = self.work_item - assert wi.resolved_version is not None - assert wi.source_url is not None - ctx = bt.ctx - cache_wheel_server_url = bt.cache_wheel_server_url - req = wi.req - req_type = wi.req_type - resolved_version = wi.resolved_version - source_url = wi.source_url - - if wi.pbi_pre_built: - - def do_prepare_prebuilt() -> PreparedSourceData: - with req_ctxvar_context(req, resolved_version): - return _bg_prepare_prebuilt( - ctx, req, req_type, resolved_version, source_url - ) - - return do_prepare_prebuilt - - def do_prepare_source() -> PreparedSourceData: - with req_ctxvar_context(req, resolved_version): - return _bg_prepare_source( - ctx, cache_wheel_server_url, req, resolved_version, source_url - ) - - return do_prepare_source - - def run(self, bt: Bootstrapper) -> list[PhaseItem]: - """PREPARE_SOURCE phase: download source or prebuilt, get build system deps. - - Uses background I/O result from ``self.bg_future`` when available, - falling back to inline I/O otherwise. - - Returns: - Prebuilt: [ProcessInstallDepsItem] (skip build phases). - Source: [PrepareBuildItem, *build_system_dep_items]. - """ - wi = self.work_item - assert wi.resolved_version is not None - assert wi.source_url is not None - - # bg_future is always set for PREPARE_SOURCE items (see _push_items). - # bg_future.result() blocks until done and re-raises any background exception. - assert self.bg_future is not None - prepared: PreparedSourceData = self.bg_future.result() - - constraint = bt.ctx.constraints.get_constraint(wi.req.name) - if constraint: - logger.info( - f"incoming requirement {wi.req} matches constraint " - f"{constraint}. Will apply both." - ) - - if wi.pbi_pre_built: - # Background task already downloaded the prebuilt wheel - assert prepared.wheel_filename is not None - assert prepared.unpack_dir is not None - wi.build_result = SourceBuildResult( - wheel_filename=prepared.wheel_filename, - sdist_filename=None, - unpack_dir=prepared.unpack_dir, - sdist_root_dir=None, - build_env=None, - source_type=SourceType.PREBUILT, - ) - return [ProcessInstallDepsItem(wi)] - - # Source build path: background task already downloaded and prepared the source - assert prepared.sdist_root_dir is not None - sdist_root_dir = prepared.sdist_root_dir - wi.cached_wheel_filename = prepared.cached_wheel_filename - - assert sdist_root_dir is not None - - if sdist_root_dir.parent.parent != bt.ctx.work_dir: - raise ValueError(f"'{sdist_root_dir}/../..' should be {bt.ctx.work_dir}") - wi.sdist_root_dir = sdist_root_dir - wi.unpack_dir = sdist_root_dir.parent - - wi.build_env = build_environment.BuildEnvironment( - ctx=bt.ctx, - parent_dir=sdist_root_dir.parent, - ) - - # Get build system dependencies - wi.build_system_deps = dependencies.get_build_system_dependencies( - ctx=bt.ctx, - req=wi.req, - version=wi.resolved_version, - sdist_root_dir=sdist_root_dir, - ) - - dep_items = bt.create_unresolved_work_items( - wi.build_system_deps, - RequirementType.BUILD_SYSTEM, - wi.req, - wi.resolved_version, - ) - - return [PrepareBuildItem(wi)] + dep_items - - -class PrepareBuildItem(PhaseItem): - """PREPARE_BUILD phase: install system deps, get backend/sdist deps.""" - - phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PREPARE_BUILD - tracks_why: typing.ClassVar[bool] = True - - def run(self, bt: Bootstrapper) -> list[PhaseItem]: - """PREPARE_BUILD phase: install system deps, get backend/sdist deps. - - Build-backend and build-sdist dependencies that are already satisfied - by a resolved build-system dependency reuse that version instead of - resolving independently (see :issue:`1194`). - - Returns: - [BuildItem, *backend_dep_items, *sdist_dep_items]. - """ - wi = self.work_item - assert wi.resolved_version is not None - assert wi.build_env is not None - assert wi.sdist_root_dir is not None - - # Install build system deps (their wheels exist from DFS processing) - wi.build_env.install(wi.build_system_deps) - - # Get build backend dependencies - wi.build_backend_deps = dependencies.get_build_backend_dependencies( - ctx=bt.ctx, - req=wi.req, - version=wi.resolved_version, - sdist_root_dir=wi.sdist_root_dir, - build_env=wi.build_env, - ) - - # Get build sdist dependencies - wi.build_sdist_deps = dependencies.get_build_sdist_dependencies( - ctx=bt.ctx, - req=wi.req, - version=wi.resolved_version, - sdist_root_dir=wi.sdist_root_dir, - build_env=wi.build_env, - ) - - # Filter out deps already satisfied by build-system dependencies - # to avoid resolving to a different (typically newer) version. - resolved_build_sys = bt.get_resolved_build_system_versions(wi) - parent = (wi.req, wi.resolved_version) - wi.build_backend_deps = bt.filter_deps_satisfied_by_build_system( - wi.build_backend_deps, - resolved_build_sys, - RequirementType.BUILD_BACKEND, - parent, - ) - wi.build_sdist_deps = bt.filter_deps_satisfied_by_build_system( - wi.build_sdist_deps, - resolved_build_sys, - RequirementType.BUILD_SDIST, - parent, - ) - - backend_items = bt.create_unresolved_work_items( - wi.build_backend_deps, - RequirementType.BUILD_BACKEND, - wi.req, - wi.resolved_version, - ) - sdist_items = bt.create_unresolved_work_items( - wi.build_sdist_deps, - RequirementType.BUILD_SDIST, - wi.req, - wi.resolved_version, - ) - dep_items = backend_items + sdist_items - - return [BuildItem(wi)] + dep_items - - -class BuildItem(PhaseItem): - """BUILD phase: install remaining deps, build wheel/sdist.""" - - phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.BUILD - tracks_why: typing.ClassVar[bool] = True - - @property - def requires_exclusive_run(self) -> bool: - return self.work_item.exclusive_build - - def run(self, bt: Bootstrapper) -> list[PhaseItem]: - """BUILD phase: install remaining deps, build wheel/sdist. - - Returns: - [ProcessInstallDepsItem]. - """ - wi = self.work_item - assert wi.resolved_version is not None - assert wi.build_env is not None - assert wi.sdist_root_dir is not None - - # Install backend+sdist deps if disjoint from system deps - remaining_deps = wi.build_backend_deps | wi.build_sdist_deps - if remaining_deps.isdisjoint(wi.build_system_deps): - wi.build_env.install(remaining_deps) - - wheel_filename, sdist_filename = self.do_build(bt.ctx, bt.explain) - - source_type = sources.get_source_type(bt.ctx, wi.req) - - wi.build_result = SourceBuildResult( - wheel_filename=wheel_filename, - sdist_filename=sdist_filename, - unpack_dir=wi.sdist_root_dir.parent, - sdist_root_dir=wi.sdist_root_dir, - build_env=wi.build_env, - source_type=source_type, - ) - - return [ProcessInstallDepsItem(wi)] - - def _build_sdist(self, ctx: context.WorkContext) -> pathlib.Path: - """Build or locate an sdist for this item's package. - - Checks ``ctx.sdists_builds`` for an existing sdist before invoking - ``sources.build_sdist()``. - """ - wi = self.work_item - assert wi.resolved_version is not None - assert wi.sdist_root_dir is not None - assert wi.build_env is not None - find_sdist_result = finders.find_sdist( - ctx, ctx.sdists_builds, wi.req, str(wi.resolved_version) - ) - if not find_sdist_result: - sdist_filename: pathlib.Path = sources.build_sdist( - ctx=ctx, - req=wi.req, - version=wi.resolved_version, - sdist_root_dir=wi.sdist_root_dir, - build_env=wi.build_env, - ) - else: - sdist_filename = find_sdist_result - logger.info( - f"have sdist version {wi.resolved_version}: {find_sdist_result}" - ) - return sdist_filename - - def _build_wheel( - self, - ctx: context.WorkContext, - explain: str = "", - ) -> tuple[pathlib.Path, pathlib.Path]: - """Build a wheel for this item's package. - - Calls :meth:`_build_sdist` first, then invokes ``wheels.build_wheel()`` - and updates the local wheel mirror. Returns ``(wheel_filename, - sdist_filename)`` where *wheel_filename* is the path in - ``ctx.wheels_downloads`` after the mirror update. - """ - wi = self.work_item - assert wi.resolved_version is not None - assert wi.sdist_root_dir is not None - assert wi.build_env is not None - sdist_filename = self._build_sdist(ctx) - - logger.info(f"starting build of {explain} for {ctx.variant}") - built_filename = wheels.build_wheel( - ctx=ctx, - req=wi.req, - sdist_root_dir=wi.sdist_root_dir, - version=wi.resolved_version, - build_env=wi.build_env, - ) - server.update_wheel_mirror(ctx) - # When we update the mirror, the built file moves to the downloads directory. - wheel_filename = ctx.wheels_downloads / built_filename.name - logger.info(f"built wheel for version {wi.resolved_version}: {wheel_filename}") - return wheel_filename, sdist_filename - - def do_build( - self, - ctx: context.WorkContext, - explain: str = "", - ) -> tuple[pathlib.Path | None, pathlib.Path | None]: - """Build a wheel or sdist for this item's package. - - Returns ``(wheel_filename, sdist_filename)``. Either value may be - ``None`` depending on the build path: - - - Cached wheel: ``(cached_wheel_filename, None)`` - - sdist-only: ``(None, sdist_filename)`` - - Full wheel: ``(wheel_filename, sdist_filename)`` - """ - wi = self.work_item - if wi.cached_wheel_filename: - logger.debug( - f"getting install requirements from cached wheel {wi.cached_wheel_filename.name}" - ) - return wi.cached_wheel_filename, None - elif wi.build_sdist_only: - logger.debug( - f"getting install requirements from sdist {wi.req.name}=={wi.resolved_version}" - ) - return None, self._build_sdist(ctx) - else: - logger.debug( - f"building wheel {wi.req.name}=={wi.resolved_version} to get install requirements" - ) - return self._build_wheel(ctx, explain) - - -class ProcessInstallDepsItem(PhaseItem): - """PROCESS_INSTALL_DEPS phase: hooks, extract deps, build order.""" - - phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PROCESS_INSTALL_DEPS - tracks_why: typing.ClassVar[bool] = True - - def run(self, bt: Bootstrapper) -> list[PhaseItem]: - """PROCESS_INSTALL_DEPS phase: hooks, extract deps, build order. - - Returns: - [CompleteItem, *install_dep_items]. - """ - wi = self.work_item - assert wi.resolved_version is not None - assert wi.source_url is not None - assert wi.build_result is not None - - # Run post-bootstrap hooks (non-fatal in test mode) - try: - hooks.run_post_bootstrap_hooks( - ctx=bt.ctx, - req=wi.req, - dist_name=canonicalize_name(wi.req.name), - dist_version=str(wi.resolved_version), - sdist_filename=wi.build_result.sdist_filename, - wheel_filename=wi.build_result.wheel_filename, - ) - except Exception as hook_error: - if not bt.test_mode: - raise - bt.record_test_mode_failure( - wi.req, - str(wi.resolved_version), - hook_error, - "hook", - "warning", - ) - - # Extract install dependencies (non-fatal in test mode) - try: - install_dependencies = _get_install_dependencies( - ctx=bt.ctx, - req=wi.req, - resolved_version=wi.resolved_version, - wheel_filename=wi.build_result.wheel_filename, - sdist_filename=wi.build_result.sdist_filename, - sdist_root_dir=wi.build_result.sdist_root_dir, - build_env=wi.build_result.build_env, - unpack_dir=wi.build_result.unpack_dir, - ) - except Exception as dep_error: - if not bt.test_mode: - raise - bt.record_test_mode_failure( - wi.req, - str(wi.resolved_version), - dep_error, - "dependency_extraction", - "warning", - ) - install_dependencies = [] - - logger.debug( - "install dependencies: %s", - ", ".join(sorted(str(r) for r in install_dependencies)), - ) - - pbi = bt.ctx.package_build_info(wi.req) - constraint = bt.ctx.constraints.get_constraint(wi.req.name) - bt.add_to_build_order( - req=wi.req, - version=wi.resolved_version, - source_url=wi.source_url, - source_type=wi.build_result.source_type, - prebuilt=pbi.pre_built, - constraint=constraint, - ) - - dep_items = bt.create_unresolved_work_items( - install_dependencies, - RequirementType.INSTALL, - wi.req, - wi.resolved_version, - ) - - return [CompleteItem(wi)] + dep_items - - -class CompleteItem(PhaseItem): - """COMPLETE phase: clean up build directories.""" - - phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.COMPLETE - tracks_why: typing.ClassVar[bool] = True - - def run(self, bt: Bootstrapper) -> list[PhaseItem]: - """COMPLETE phase: clean up build directories. - - Returns: - Empty list (processing finished for this item). - """ - wi = self.work_item - if wi.build_result is not None: - bt.ctx.clean_build_dirs( - wi.build_result.sdist_root_dir, - wi.build_result.build_env, - ) - return [] - class Bootstrapper: def __init__( @@ -1664,7 +556,7 @@ def _download_prebuilt( resolved_version: Version, wheel_url: str, ) -> tuple[pathlib.Path, pathlib.Path]: - result = _bg_prepare_prebuilt( + result = _cache._bg_prepare_prebuilt( self.ctx, req, req_type, resolved_version, wheel_url ) assert result.wheel_filename is not None @@ -1756,6 +648,7 @@ def _handle_test_mode_failure( req.name, fallback_version, ) + # Package succeeded via fallback - no failure to record return SourceBuildResult( diff --git a/src/fromager/bootstrapper/_build_item.py b/src/fromager/bootstrapper/_build_item.py new file mode 100644 index 00000000..d3f93b03 --- /dev/null +++ b/src/fromager/bootstrapper/_build_item.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import logging +import pathlib +import typing + +from .. import finders, server, sources, wheels +from ._phase_item import PhaseItem +from ._process_install_deps_item import ProcessInstallDepsItem +from ._types import BootstrapPhase, SourceBuildResult + +if typing.TYPE_CHECKING: + from .. import context + from ._bootstrapper import Bootstrapper + +logger = logging.getLogger(__name__) + + +class BuildItem(PhaseItem): + """BUILD phase: install remaining deps, build wheel/sdist.""" + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.BUILD + tracks_why: typing.ClassVar[bool] = True + + @property + def requires_exclusive_run(self) -> bool: + return self.work_item.exclusive_build + + def run(self, bt: Bootstrapper) -> list[PhaseItem]: + """BUILD phase: install remaining deps, build wheel/sdist. + + Returns: + [ProcessInstallDepsItem]. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.build_env is not None + assert wi.sdist_root_dir is not None + + # Install backend+sdist deps if disjoint from system deps + remaining_deps = wi.build_backend_deps | wi.build_sdist_deps + if remaining_deps.isdisjoint(wi.build_system_deps): + wi.build_env.install(remaining_deps) + + wheel_filename, sdist_filename = self.do_build(bt.ctx, bt.explain) + + source_type = sources.get_source_type(bt.ctx, wi.req) + + wi.build_result = SourceBuildResult( + wheel_filename=wheel_filename, + sdist_filename=sdist_filename, + unpack_dir=wi.sdist_root_dir.parent, + sdist_root_dir=wi.sdist_root_dir, + build_env=wi.build_env, + source_type=source_type, + ) + + return [ProcessInstallDepsItem(wi)] + + def _build_sdist(self, ctx: context.WorkContext) -> pathlib.Path: + """Build or locate an sdist for this item's package. + + Checks ``ctx.sdists_builds`` for an existing sdist before invoking + ``sources.build_sdist()``. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.sdist_root_dir is not None + assert wi.build_env is not None + find_sdist_result = finders.find_sdist( + ctx, ctx.sdists_builds, wi.req, str(wi.resolved_version) + ) + if not find_sdist_result: + sdist_filename: pathlib.Path = sources.build_sdist( + ctx=ctx, + req=wi.req, + version=wi.resolved_version, + sdist_root_dir=wi.sdist_root_dir, + build_env=wi.build_env, + ) + else: + sdist_filename = find_sdist_result + logger.info( + f"have sdist version {wi.resolved_version}: {find_sdist_result}" + ) + return sdist_filename + + def _build_wheel( + self, + ctx: context.WorkContext, + explain: str = "", + ) -> tuple[pathlib.Path, pathlib.Path]: + """Build a wheel for this item's package. + + Calls :meth:`_build_sdist` first, then invokes ``wheels.build_wheel()`` + and updates the local wheel mirror. Returns ``(wheel_filename, + sdist_filename)`` where *wheel_filename* is the path in + ``ctx.wheels_downloads`` after the mirror update. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.sdist_root_dir is not None + assert wi.build_env is not None + sdist_filename = self._build_sdist(ctx) + + logger.info(f"starting build of {explain} for {ctx.variant}") + built_filename = wheels.build_wheel( + ctx=ctx, + req=wi.req, + sdist_root_dir=wi.sdist_root_dir, + version=wi.resolved_version, + build_env=wi.build_env, + ) + server.update_wheel_mirror(ctx) + # When we update the mirror, the built file moves to the downloads directory. + wheel_filename = ctx.wheels_downloads / built_filename.name + logger.info(f"built wheel for version {wi.resolved_version}: {wheel_filename}") + return wheel_filename, sdist_filename + + def do_build( + self, + ctx: context.WorkContext, + explain: str = "", + ) -> tuple[pathlib.Path | None, pathlib.Path | None]: + """Build a wheel or sdist for this item's package. + + Returns ``(wheel_filename, sdist_filename)``. Either value may be + ``None`` depending on the build path: + + - Cached wheel: ``(cached_wheel_filename, None)`` + - sdist-only: ``(None, sdist_filename)`` + - Full wheel: ``(wheel_filename, sdist_filename)`` + """ + wi = self.work_item + if wi.cached_wheel_filename: + logger.debug( + f"getting install requirements from cached wheel {wi.cached_wheel_filename.name}" + ) + return wi.cached_wheel_filename, None + elif wi.build_sdist_only: + logger.debug( + f"getting install requirements from sdist {wi.req.name}=={wi.resolved_version}" + ) + return None, self._build_sdist(ctx) + else: + logger.debug( + f"building wheel {wi.req.name}=={wi.resolved_version} to get install requirements" + ) + return self._build_wheel(ctx, explain) diff --git a/src/fromager/bootstrapper/_cache.py b/src/fromager/bootstrapper/_cache.py new file mode 100644 index 00000000..3b87ff1b --- /dev/null +++ b/src/fromager/bootstrapper/_cache.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import logging +import os +import pathlib +import typing +import zipfile +from urllib.parse import urlparse + +import requests.exceptions +from packaging.requirements import Requirement +from packaging.version import Version +from resolvelib.resolvers import ResolverException + +from .. import dependencies, finders, resolver, server, wheels +from ..requirements_file import RequirementType +from ._types import PreparedSourceData + +if typing.TYPE_CHECKING: + from .. import context + +logger = logging.getLogger(__name__) + + +def _create_unpack_dir( + work_dir: pathlib.Path, + req: Requirement, + resolved_version: Version, +) -> pathlib.Path: + unpack_dir = work_dir / f"{req.name}-{resolved_version}" + unpack_dir.mkdir(parents=True, exist_ok=True) + return unpack_dir + + +def _extract_build_reqs_from_wheel( + work_dir: pathlib.Path, + req: Requirement, + resolved_version: Version, + wheel_filename: pathlib.Path, +) -> pathlib.Path | None: + """Extract fromager build requirement files from a wheel archive. + + Looks for files prefixed with `FROMAGER_BUILD_REQ_PREFIX` inside the + wheel's `.dist-info` directory and extracts them to a local unpack + directory. Returns the unpack directory on success, or None if the + files are not present or extraction fails. + """ + dist_name, dist_version, _, _ = wheels.extract_info_from_wheel_file( + req, wheel_filename + ) + unpack_dir = _create_unpack_dir(work_dir, req, resolved_version) + dist_filename = f"{dist_name}-{dist_version}" + dist_info_path = pathlib.Path(f"{dist_filename}.dist-info") + req_filenames: list[str] = [ + dependencies.BUILD_BACKEND_REQ_FILE_NAME, + dependencies.BUILD_SDIST_REQ_FILE_NAME, + dependencies.BUILD_SYSTEM_REQ_FILE_NAME, + ] + try: + with zipfile.ZipFile(wheel_filename) as archive: + for filename in req_filenames: + zipinfo = archive.getinfo( + str( + dist_info_path + / f"{wheels.FROMAGER_BUILD_REQ_PREFIX}-{filename}" + ) + ) + if os.path.isabs(zipinfo.filename) or ".." in zipinfo.filename: + raise ValueError(f"Unsafe path in wheel: {zipinfo.filename}") + zipinfo.filename = filename + output_file = archive.extract(zipinfo, unpack_dir) + logger.info(f"extracted {output_file}") + logger.info(f"extracted build requirements from wheel into {unpack_dir}") + return unpack_dir + except Exception as e: + logger.info(f"could not extract build requirements from wheel: {e}") + for filename in req_filenames: + unpack_dir.joinpath(filename).unlink(missing_ok=True) + return None + + +def _look_for_existing_wheel( + ctx: context.WorkContext, + req: Requirement, + resolved_version: Version, + search_in: pathlib.Path, +) -> tuple[pathlib.Path | None, pathlib.Path | None]: + pbi = ctx.package_build_info(req) + expected_build_tag = pbi.build_tag(resolved_version) + logger.info( + f"looking for existing wheel for version {resolved_version} with build tag {expected_build_tag} in {search_in}" + ) + wheel_filename = finders.find_wheel( + downloads_dir=search_in, + req=req, + dist_version=str(resolved_version), + build_tag=expected_build_tag, + ) + if not wheel_filename: + return None, None + _, _, build_tag, _ = wheels.extract_info_from_wheel_file(req, wheel_filename) + if expected_build_tag and expected_build_tag != build_tag: + logger.info( + f"found wheel for {resolved_version} in {wheel_filename} but build tag does not match. Got {build_tag} but expected {expected_build_tag}" + ) + return None, None + logger.info(f"found existing wheel {wheel_filename}") + build_reqs_dir = _extract_build_reqs_from_wheel( + ctx.work_dir, req, resolved_version, wheel_filename + ) + return wheel_filename, build_reqs_dir + + +def _download_wheel_from_cache( + ctx: context.WorkContext, + cache_wheel_server_url: str | None, + req: Requirement, + resolved_version: Version, +) -> tuple[pathlib.Path | None, pathlib.Path | None]: + if not cache_wheel_server_url: + return None, None + logger.info(f"checking if wheel was already uploaded to {cache_wheel_server_url}") + try: + pinned_req = Requirement(f"{req.name}=={resolved_version}") + provider = finders.PyPICacheProvider( + cache_server_url=cache_wheel_server_url, + constraints=ctx.constraints, + ) + results = resolver.find_all_matching_from_provider(provider, pinned_req) + wheel_url, _ = results[0] + wheelfile_name = pathlib.Path(urlparse(wheel_url).path) + pbi = ctx.package_build_info(req) + expected_build_tag = pbi.build_tag(resolved_version) + logger.info(f"has expected build tag {expected_build_tag}") + changelogs = pbi.get_changelog(resolved_version) + logger.debug(f"has change logs {changelogs}") + + _, _, build_tag, _ = wheels.extract_info_from_wheel_file(req, wheelfile_name) + if expected_build_tag and expected_build_tag != build_tag: + logger.info( + f"found wheel for {resolved_version} in cache but build tag does not match. Got {build_tag} but expected {expected_build_tag}" + ) + return None, None + + cached_wheel = wheels.download_wheel( + req=req, wheel_url=wheel_url, output_directory=ctx.wheels_downloads + ) + if cache_wheel_server_url != ctx.wheel_server_url: + server.update_wheel_mirror(ctx) + logger.info("found built wheel on cache server") + unpack_dir = _extract_build_reqs_from_wheel( + ctx.work_dir, req, resolved_version, cached_wheel + ) + return cached_wheel, unpack_dir + except ResolverException: + logger.info( + f"did not find wheel for {resolved_version} in {cache_wheel_server_url}" + ) + return None, None + except requests.exceptions.RequestException as err: + logger.warning( + f"network error checking wheel cache for {resolved_version} " + f"at {cache_wheel_server_url}: {err}" + ) + return None, None + except Exception as err: + logger.warning( + f"unexpected error checking wheel cache for {resolved_version} " + f"at {cache_wheel_server_url}: {err}" + ) + return None, None + + +def _find_cached_wheel( + ctx: context.WorkContext, + cache_wheel_server_url: str | None, + req: Requirement, + resolved_version: Version, +) -> tuple[pathlib.Path | None, pathlib.Path | None]: + """Look for cached wheel in 3 locations (thread-safe, no Bootstrapper state). + + Checks for cached wheels in order: + 1. wheels_build directory (previously built) + 2. wheels_downloads directory (previously downloaded) + 3. Cache server (remote cache) + + Returns: + Tuple of (cached_wheel_filename, unpacked_cached_wheel). + Both None if no cache hit. + """ + cached_wheel, unpacked = _look_for_existing_wheel( + ctx, req, resolved_version, ctx.wheels_build + ) + if cached_wheel: + return cached_wheel, unpacked + + cached_wheel, unpacked = _look_for_existing_wheel( + ctx, req, resolved_version, ctx.wheels_downloads + ) + if cached_wheel: + return cached_wheel, unpacked + + cached_wheel, unpacked = _download_wheel_from_cache( + ctx, cache_wheel_server_url, req, resolved_version + ) + if cached_wheel: + return cached_wheel, unpacked + + return None, None + + +def _bg_prepare_prebuilt( + ctx: context.WorkContext, + req: Requirement, + req_type: RequirementType, + resolved_version: Version, + wheel_url: str, +) -> PreparedSourceData: + """Background-safe prebuilt download: no Bootstrapper state accessed.""" + # Thread-safe: paths include {req.name}-{resolved_version} (unique per package), + # mkdir uses exist_ok=True (atomic), and update_wheel_mirror() is already locked. + logger.info(f"using pre-built wheel for {req_type} requirement") + wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt) + unpack_dir = ctx.work_dir / f"{req.name}-{resolved_version}" + unpack_dir.mkdir(parents=True, exist_ok=True) + server.update_wheel_mirror(ctx) + return PreparedSourceData(wheel_filename=wheel_filename, unpack_dir=unpack_dir) diff --git a/src/fromager/bootstrapper/_complete_item.py b/src/fromager/bootstrapper/_complete_item.py new file mode 100644 index 00000000..574c8615 --- /dev/null +++ b/src/fromager/bootstrapper/_complete_item.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import typing + +from ._phase_item import PhaseItem +from ._types import BootstrapPhase + +if typing.TYPE_CHECKING: + from ._bootstrapper import Bootstrapper + + +class CompleteItem(PhaseItem): + """COMPLETE phase: clean up build directories.""" + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.COMPLETE + tracks_why: typing.ClassVar[bool] = True + + def run(self, bt: Bootstrapper) -> list[PhaseItem]: + """COMPLETE phase: clean up build directories. + + Returns: + Empty list (processing finished for this item). + """ + wi = self.work_item + if wi.build_result is not None: + bt.ctx.clean_build_dirs( + wi.build_result.sdist_root_dir, + wi.build_result.build_env, + ) + return [] diff --git a/src/fromager/bootstrapper/_phase_item.py b/src/fromager/bootstrapper/_phase_item.py new file mode 100644 index 00000000..95379a6e --- /dev/null +++ b/src/fromager/bootstrapper/_phase_item.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import abc +import concurrent.futures +import logging +import typing + +from ._types import BootstrapPhase +from ._work_item import WorkItem + +if typing.TYPE_CHECKING: + from ._bootstrapper import Bootstrapper + +logger = logging.getLogger(__name__) + + +class PhaseItem(abc.ABC): + """Abstract base for items pushed onto the bootstrap stack. + + Each subclass encodes one phase of the bootstrap pipeline. + Wraps a ``WorkItem`` (accumulated per-package state) and implements + the processing logic for that phase in ``run()``. + """ + + phase: typing.ClassVar[BootstrapPhase] + tracks_why: typing.ClassVar[bool] = True + + def __init__(self, work_item: WorkItem) -> None: + self.work_item = work_item + self.bg_future: concurrent.futures.Future[typing.Any] | None = None + + @abc.abstractmethod + def run(self, bt: Bootstrapper) -> list[PhaseItem]: ... + + @property + def requires_exclusive_run(self) -> bool: + """Return True if this item must run without concurrent background I/O. + + When True, the bootstrap loop drains the background thread pool before + calling ``run()``. Override in subclasses that require exclusive access. + """ + return False + + def background_work( + self, bt: Bootstrapper + ) -> typing.Callable[[], typing.Any] | None: + """Return a zero-argument callable for background I/O, or None. + + Override in subclasses that need background prefetching. + ``bt`` is provided so subclasses can capture Bootstrapper state + (e.g. resolver, ctx) into the returned closure without storing + a circular reference on the item itself. + """ + return None + + def __str__(self) -> str: + """Human-readable representation: ``"()"``.""" + wi = self.work_item + return f"{type(self).__name__}({wi.req})" + + def as_json(self) -> dict[str, typing.Any]: + """Return a JSON-serialisable dict for stack-state recording.""" + wi = self.work_item + return { + "req": str(wi.req), + "req_type": str(wi.req_type), + "phase": str(self.phase), + "resolved_version": str(wi.resolved_version) + if wi.resolved_version is not None + else None, + "source_url": wi.source_url, + "build_sdist_only": wi.build_sdist_only, + "why": [ + {"req_type": str(rt), "req": str(r), "version": str(v)} + for rt, r, v in wi.why_snapshot + ], + "parent": ( + {"req": str(wi.parent[0]), "version": str(wi.parent[1])} + if wi.parent + else None + ), + "build_system_deps": sorted(str(r) for r in wi.build_system_deps), + "build_backend_deps": sorted(str(r) for r in wi.build_backend_deps), + "build_sdist_deps": sorted(str(r) for r in wi.build_sdist_deps), + } diff --git a/src/fromager/bootstrapper/_prepare_build_item.py b/src/fromager/bootstrapper/_prepare_build_item.py new file mode 100644 index 00000000..87c3f89b --- /dev/null +++ b/src/fromager/bootstrapper/_prepare_build_item.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import logging +import typing + +from .. import dependencies +from ..requirements_file import RequirementType +from ._build_item import BuildItem +from ._phase_item import PhaseItem +from ._types import BootstrapPhase + +if typing.TYPE_CHECKING: + from ._bootstrapper import Bootstrapper + +logger = logging.getLogger(__name__) + + +class PrepareBuildItem(PhaseItem): + """PREPARE_BUILD phase: install system deps, get backend/sdist deps.""" + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PREPARE_BUILD + tracks_why: typing.ClassVar[bool] = True + + def run(self, bt: Bootstrapper) -> list[PhaseItem]: + """PREPARE_BUILD phase: install system deps, get backend/sdist deps. + + Build-backend and build-sdist dependencies that are already satisfied + by a resolved build-system dependency reuse that version instead of + resolving independently (see :issue:`1194`). + + Returns: + [BuildItem, *backend_dep_items, *sdist_dep_items]. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.build_env is not None + assert wi.sdist_root_dir is not None + + # Install build system deps (their wheels exist from DFS processing) + wi.build_env.install(wi.build_system_deps) + + # Get build backend dependencies + wi.build_backend_deps = dependencies.get_build_backend_dependencies( + ctx=bt.ctx, + req=wi.req, + version=wi.resolved_version, + sdist_root_dir=wi.sdist_root_dir, + build_env=wi.build_env, + ) + + # Get build sdist dependencies + wi.build_sdist_deps = dependencies.get_build_sdist_dependencies( + ctx=bt.ctx, + req=wi.req, + version=wi.resolved_version, + sdist_root_dir=wi.sdist_root_dir, + build_env=wi.build_env, + ) + + # Filter out deps already satisfied by build-system dependencies + # to avoid resolving to a different (typically newer) version. + resolved_build_sys = bt.get_resolved_build_system_versions(wi) + parent = (wi.req, wi.resolved_version) + wi.build_backend_deps = bt.filter_deps_satisfied_by_build_system( + wi.build_backend_deps, + resolved_build_sys, + RequirementType.BUILD_BACKEND, + parent, + ) + wi.build_sdist_deps = bt.filter_deps_satisfied_by_build_system( + wi.build_sdist_deps, + resolved_build_sys, + RequirementType.BUILD_SDIST, + parent, + ) + + backend_items: list[PhaseItem] = bt.create_unresolved_work_items( + wi.build_backend_deps, + RequirementType.BUILD_BACKEND, + wi.req, + wi.resolved_version, + ) + sdist_items: list[PhaseItem] = bt.create_unresolved_work_items( + wi.build_sdist_deps, + RequirementType.BUILD_SDIST, + wi.req, + wi.resolved_version, + ) + dep_items = backend_items + sdist_items + + return [BuildItem(wi)] + dep_items diff --git a/src/fromager/bootstrapper/_prepare_source_item.py b/src/fromager/bootstrapper/_prepare_source_item.py new file mode 100644 index 00000000..bc1cd8c6 --- /dev/null +++ b/src/fromager/bootstrapper/_prepare_source_item.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import logging +import typing + +from packaging.requirements import Requirement +from packaging.version import Version + +from .. import build_environment, dependencies, sources +from ..log import req_ctxvar_context +from ..requirements_file import RequirementType, SourceType +from . import _cache +from ._phase_item import PhaseItem +from ._prepare_build_item import PrepareBuildItem +from ._process_install_deps_item import ProcessInstallDepsItem +from ._types import BootstrapPhase, PreparedSourceData, SourceBuildResult + +if typing.TYPE_CHECKING: + from .. import context + from ._bootstrapper import Bootstrapper + +logger = logging.getLogger(__name__) + + +def _bg_prepare_source( + ctx: context.WorkContext, + cache_wheel_server_url: str | None, + req: Requirement, + resolved_version: Version, + source_url: str, +) -> PreparedSourceData: + """Background-safe source download+unpack: no Bootstrapper state accessed.""" + # Thread-safe: _seen_requirements in the main thread prevents the same + # package from being submitted to the thread pool more than once. + # Paths from download_source() and prepare_source() include {name}-{version}, + # making them unique across concurrent threads processing different packages. + logger.info("preparing source") + cached_wheel, unpacked = _cache._find_cached_wheel( + ctx, cache_wheel_server_url, req, resolved_version + ) + if unpacked is not None: + return PreparedSourceData( + sdist_root_dir=unpacked / unpacked.stem, + cached_wheel_filename=cached_wheel, + ) + source_filename = sources.download_source( + ctx=ctx, req=req, version=resolved_version, download_url=source_url + ) + sdist_root_dir = sources.prepare_source( + ctx=ctx, req=req, source_filename=source_filename, version=resolved_version + ) + return PreparedSourceData(sdist_root_dir=sdist_root_dir) + + +class PrepareSourceItem(PhaseItem): + """PREPARE_SOURCE phase: download source or prebuilt, get build system deps.""" + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PREPARE_SOURCE + tracks_why: typing.ClassVar[bool] = True + + def background_work( + self, bt: Bootstrapper + ) -> typing.Callable[[], typing.Any] | None: + """Return closure for background source download or prebuilt fetch.""" + wi = self.work_item + assert wi.resolved_version is not None + assert wi.source_url is not None + ctx = bt.ctx + cache_wheel_server_url = bt.cache_wheel_server_url + req = wi.req + req_type = wi.req_type + resolved_version = wi.resolved_version + source_url = wi.source_url + + if wi.pbi_pre_built: + + def do_prepare_prebuilt() -> PreparedSourceData: + with req_ctxvar_context(req, resolved_version): + return _cache._bg_prepare_prebuilt( + ctx, req, req_type, resolved_version, source_url + ) + + return do_prepare_prebuilt + + def do_prepare_source() -> PreparedSourceData: + with req_ctxvar_context(req, resolved_version): + return _bg_prepare_source( + ctx, cache_wheel_server_url, req, resolved_version, source_url + ) + + return do_prepare_source + + def run(self, bt: Bootstrapper) -> list[PhaseItem]: + """PREPARE_SOURCE phase: download source or prebuilt, get build system deps. + + Uses background I/O result from ``self.bg_future`` when available, + falling back to inline I/O otherwise. + + Returns: + Prebuilt: [ProcessInstallDepsItem] (skip build phases). + Source: [PrepareBuildItem, *build_system_dep_items]. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.source_url is not None + + # bg_future is always set for PREPARE_SOURCE items (see _push_items). + # bg_future.result() blocks until done and re-raises any background exception. + assert self.bg_future is not None + prepared: PreparedSourceData = self.bg_future.result() + + constraint = bt.ctx.constraints.get_constraint(wi.req.name) + if constraint: + logger.info( + f"incoming requirement {wi.req} matches constraint " + f"{constraint}. Will apply both." + ) + + if wi.pbi_pre_built: + # Background task already downloaded the prebuilt wheel + assert prepared.wheel_filename is not None + assert prepared.unpack_dir is not None + wi.build_result = SourceBuildResult( + wheel_filename=prepared.wheel_filename, + sdist_filename=None, + unpack_dir=prepared.unpack_dir, + sdist_root_dir=None, + build_env=None, + source_type=SourceType.PREBUILT, + ) + return [ProcessInstallDepsItem(wi)] + + # Source build path: background task already downloaded and prepared the source + assert prepared.sdist_root_dir is not None + sdist_root_dir = prepared.sdist_root_dir + wi.cached_wheel_filename = prepared.cached_wheel_filename + + assert sdist_root_dir is not None + + if sdist_root_dir.parent.parent != bt.ctx.work_dir: + raise ValueError(f"'{sdist_root_dir}/../..' should be {bt.ctx.work_dir}") + wi.sdist_root_dir = sdist_root_dir + wi.unpack_dir = sdist_root_dir.parent + + wi.build_env = build_environment.BuildEnvironment( + ctx=bt.ctx, + parent_dir=sdist_root_dir.parent, + ) + + # Get build system dependencies + wi.build_system_deps = dependencies.get_build_system_dependencies( + ctx=bt.ctx, + req=wi.req, + version=wi.resolved_version, + sdist_root_dir=sdist_root_dir, + ) + + dep_items: list[PhaseItem] = bt.create_unresolved_work_items( + wi.build_system_deps, + RequirementType.BUILD_SYSTEM, + wi.req, + wi.resolved_version, + ) + + return [PrepareBuildItem(wi)] + dep_items diff --git a/src/fromager/bootstrapper/_process_install_deps_item.py b/src/fromager/bootstrapper/_process_install_deps_item.py new file mode 100644 index 00000000..5f79b549 --- /dev/null +++ b/src/fromager/bootstrapper/_process_install_deps_item.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import logging +import pathlib +import typing + +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name +from packaging.version import Version + +from .. import build_environment, dependencies, hooks +from ..requirements_file import RequirementType +from ._complete_item import CompleteItem +from ._phase_item import PhaseItem +from ._types import BootstrapPhase + +if typing.TYPE_CHECKING: + from .. import context + from ._bootstrapper import Bootstrapper + +logger = logging.getLogger(__name__) + + +def _get_install_dependencies( + ctx: context.WorkContext, + req: Requirement, + resolved_version: Version, + wheel_filename: pathlib.Path | None, + sdist_filename: pathlib.Path | None, + sdist_root_dir: pathlib.Path | None, + build_env: build_environment.BuildEnvironment | None, + unpack_dir: pathlib.Path | None, +) -> list[Requirement]: + """Extract install dependencies from a built wheel or sdist. + + Returns: + List of install requirements. + + Raises: + RuntimeError: If both wheel_filename and sdist_filename are None. + """ + if wheel_filename is not None: + assert unpack_dir is not None + logger.debug( + "get install dependencies of wheel %s", + wheel_filename.name, + ) + return list( + dependencies.get_install_dependencies_of_wheel( + req=req, + wheel_filename=wheel_filename, + requirements_file_dir=unpack_dir, + ) + ) + elif sdist_filename is not None: + assert sdist_root_dir is not None + assert build_env is not None + logger.debug( + "get install dependencies of sdist from directory %s", + sdist_root_dir, + ) + return list( + dependencies.get_install_dependencies_of_sdist( + ctx=ctx, + req=req, + version=resolved_version, + sdist_root_dir=sdist_root_dir, + build_env=build_env, + ) + ) + else: + raise RuntimeError("wheel_filename and sdist_filename are None") + + +class ProcessInstallDepsItem(PhaseItem): + """PROCESS_INSTALL_DEPS phase: hooks, extract deps, build order.""" + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PROCESS_INSTALL_DEPS + tracks_why: typing.ClassVar[bool] = True + + def run(self, bt: Bootstrapper) -> list[PhaseItem]: + """PROCESS_INSTALL_DEPS phase: hooks, extract deps, build order. + + Returns: + [CompleteItem, *install_dep_items]. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.source_url is not None + assert wi.build_result is not None + + # Run post-bootstrap hooks (non-fatal in test mode) + try: + hooks.run_post_bootstrap_hooks( + ctx=bt.ctx, + req=wi.req, + dist_name=canonicalize_name(wi.req.name), + dist_version=str(wi.resolved_version), + sdist_filename=wi.build_result.sdist_filename, + wheel_filename=wi.build_result.wheel_filename, + ) + except Exception as hook_error: + if not bt.test_mode: + raise + bt.record_test_mode_failure( + wi.req, + str(wi.resolved_version), + hook_error, + "hook", + "warning", + ) + + # Extract install dependencies (non-fatal in test mode) + try: + install_dependencies = _get_install_dependencies( + ctx=bt.ctx, + req=wi.req, + resolved_version=wi.resolved_version, + wheel_filename=wi.build_result.wheel_filename, + sdist_filename=wi.build_result.sdist_filename, + sdist_root_dir=wi.build_result.sdist_root_dir, + build_env=wi.build_result.build_env, + unpack_dir=wi.build_result.unpack_dir, + ) + except Exception as dep_error: + if not bt.test_mode: + raise + bt.record_test_mode_failure( + wi.req, + str(wi.resolved_version), + dep_error, + "dependency_extraction", + "warning", + ) + install_dependencies = [] + + logger.debug( + "install dependencies: %s", + ", ".join(sorted(str(r) for r in install_dependencies)), + ) + + pbi = bt.ctx.package_build_info(wi.req) + constraint = bt.ctx.constraints.get_constraint(wi.req.name) + bt.add_to_build_order( + req=wi.req, + version=wi.resolved_version, + source_url=wi.source_url, + source_type=wi.build_result.source_type, + prebuilt=pbi.pre_built, + constraint=constraint, + ) + + dep_items: list[PhaseItem] = bt.create_unresolved_work_items( + install_dependencies, + RequirementType.INSTALL, + wi.req, + wi.resolved_version, + ) + + return [CompleteItem(wi)] + dep_items diff --git a/src/fromager/bootstrapper/_resolve_item.py b/src/fromager/bootstrapper/_resolve_item.py new file mode 100644 index 00000000..c793ab25 --- /dev/null +++ b/src/fromager/bootstrapper/_resolve_item.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import logging +import typing + +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name +from packaging.version import Version + +from .. import bootstrap_requirement_resolver +from ..log import req_ctxvar_context +from ..requirements_file import RequirementType +from . import _cache +from ._phase_item import PhaseItem +from ._start_item import StartItem +from ._types import BootstrapPhase +from ._work_item import WorkItem + +if typing.TYPE_CHECKING: + from ._bootstrapper import Bootstrapper + +logger = logging.getLogger(__name__) + + +def _bg_resolve( + bg_resolver: bootstrap_requirement_resolver.BootstrapRequirementResolver, + req: Requirement, + req_type: RequirementType, + parent_req: Requirement | None, + return_all_versions: bool, +) -> list[tuple[str, Version]]: + """Background-safe resolution: no Bootstrapper state accessed.""" + logger.info(f"{BootstrapPhase.RESOLVE} for {req_type} requirement") + return bg_resolver.resolve( + req=req, + req_type=req_type, + parent_req=parent_req, + return_all_versions=return_all_versions, + ) + + +class ResolveItem(PhaseItem): + """RESOLVE phase: resolve versions and expand into StartItems.""" + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.RESOLVE + tracks_why: typing.ClassVar[bool] = False + + def background_work( + self, bt: Bootstrapper + ) -> typing.Callable[[], typing.Any] | None: + """Return closure that calls ``_bg_resolve`` in a thread.""" + bg_resolver = bt.resolver + req = self.work_item.req + req_type = self.work_item.req_type + parent_req = ( + self.work_item.why_snapshot[-1][1] if self.work_item.why_snapshot else None + ) + return_all = bt.multiple_versions + + def do_resolve() -> list[tuple[str, Version]]: + with req_ctxvar_context(req): + return _bg_resolve(bg_resolver, req, req_type, parent_req, return_all) + + return do_resolve + + def run(self, bt: Bootstrapper) -> list[PhaseItem]: + """RESOLVE phase: resolve versions and expand into StartItems. + + Centralizes version resolution so all dependencies are expanded + uniformly. In multiple_versions mode, filters out versions that + already failed in this run and versions whose wheels are already + cached to avoid redundant builds and transitive dependency + processing. + + Returns: + One StartItem per resolved version that needs building. + Empty list if all versions are already cached. + """ + assert self.bg_future is not None + # bg_future.result() blocks until the background resolution completes, + # then returns the result or re-raises any exception from the background. + resolved_versions = self.bg_future.result() + if not resolved_versions: + raise RuntimeError( + f"Could not resolve any versions for {self.work_item.req}" + ) + + if bt.multiple_versions: + pkg_name = canonicalize_name(self.work_item.req.name) + resolved_versions = [ + (url, ver) + for url, ver in resolved_versions + if not bt.has_failed_version(pkg_name, ver) + ] + if not resolved_versions: + raise RuntimeError( + f"Could not resolve any versions for {self.work_item.req}" + f" (all candidates failed previously)" + ) + + logger.info( + f"resolved {len(resolved_versions)} version(s) for {self.work_item.req}" + ) + filtered: list[tuple[str, Version]] = [] + for source_url, version in resolved_versions: + cached_wheel, _ = _cache._find_cached_wheel( + bt.ctx, bt.cache_wheel_server_url, self.work_item.req, version + ) + if cached_wheel: + logger.info( + f"{self.work_item.req.name}=={version}: wheel already cached " + f"at {cached_wheel.name}, skipping" + ) + else: + filtered.append((source_url, version)) + if not filtered: + # Always process the highest version (first in + # resolved_versions) so new transitive dependencies + # are discovered even when every wheel is cached. + logger.info( + f"all versions of {self.work_item.req.name} already cached, " + f"keeping highest version {resolved_versions[0][1]} " + f"for dependency discovery" + ) + filtered.append(resolved_versions[0]) + resolved_versions = filtered + + # Build list so highest version ends up on top of the stack + # (last element after extend) and is processed first. + items: list[PhaseItem] = [] + for source_url, version in reversed(resolved_versions): + items.append( + StartItem( + WorkItem( + req=self.work_item.req, + req_type=self.work_item.req_type, + why_snapshot=list(self.work_item.why_snapshot), + parent=self.work_item.parent, + source_url=source_url, + resolved_version=version, + ) + ) + ) + return items diff --git a/src/fromager/bootstrapper/_start_item.py b/src/fromager/bootstrapper/_start_item.py new file mode 100644 index 00000000..32561da9 --- /dev/null +++ b/src/fromager/bootstrapper/_start_item.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import logging +import typing + +from ..requirements_file import RequirementType +from ._phase_item import PhaseItem +from ._prepare_source_item import PrepareSourceItem +from ._types import BootstrapPhase + +if typing.TYPE_CHECKING: + from ._bootstrapper import Bootstrapper + +logger = logging.getLogger(__name__) + + +class StartItem(PhaseItem): + """START phase: add to graph, check if already seen.""" + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.START + tracks_why: typing.ClassVar[bool] = False + + def run(self, bt: Bootstrapper) -> list[PhaseItem]: + """START phase: add to graph, check if already seen. + + _track_why is a no-op for this phase (tracks_why is False), + matching the original behavior where graph addition and + seen-check happen before pushing onto the why stack. + + Returns: + Empty list if already seen (nothing to do). + [PrepareSourceItem] if this is new work. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.source_url is not None + + # Add to graph (skip TOP_LEVEL, already added in _resolve_and_add_top_level) + if wi.req_type != RequirementType.TOP_LEVEL: + bt.add_to_graph( + wi.req, + wi.req_type, + wi.resolved_version, + wi.source_url, + wi.parent, + ) + + wi.build_sdist_only = bt.sdist_only and not wi.is_build_requirement_context() + + if bt.has_been_seen(wi.req, wi.resolved_version, wi.build_sdist_only): + logger.debug( + f"redundant {wi.req_type} dependency {wi.req} " + f"({wi.resolved_version}, sdist_only={wi.build_sdist_only}) " + f"for {bt.explain}" + ) + return [] + bt.mark_as_seen(wi.req, wi.resolved_version, wi.build_sdist_only) + + logger.info( + f"new {wi.req_type} dependency {wi.req} resolves to {wi.resolved_version}" + ) + + # Must set pbi_pre_built before constructing PrepareSourceItem so that + # PrepareSourceItem.background_work() immediately sees the correct value. + pbi = bt.ctx.package_build_info(wi.req) + wi.pbi_pre_built = pbi.pre_built + wi.exclusive_build = pbi.exclusive_build + return [PrepareSourceItem(wi)] diff --git a/src/fromager/bootstrapper/_types.py b/src/fromager/bootstrapper/_types.py new file mode 100644 index 00000000..cd395006 --- /dev/null +++ b/src/fromager/bootstrapper/_types.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import dataclasses +import logging +import pathlib +import typing +from enum import StrEnum + +from packaging.utils import NormalizedName + +from .. import build_environment, threading_utils +from ..requirements_file import SourceType + +logger = logging.getLogger(__name__) + +# package name, extras, version, sdist/wheel +SeenKey = tuple[NormalizedName, tuple[str, ...], str, typing.Literal["sdist", "wheel"]] + +_DEFAULT_BG_THREADS: int = max(1, threading_utils.get_cpu_count() // 2) + + +@dataclasses.dataclass +class SourceBuildResult: + """Result of building or downloading a package. + + Captures the output artifacts from either a source build or + prebuilt wheel download, used across bootstrap phases. + """ + + wheel_filename: pathlib.Path | None + sdist_filename: pathlib.Path | None + unpack_dir: pathlib.Path + sdist_root_dir: pathlib.Path | None + build_env: build_environment.BuildEnvironment | None + source_type: SourceType + + +@dataclasses.dataclass +class PreparedSourceData: + """Result of background I/O pre-fetching returned to the main thread. + + Fields are set in one of three combinations depending on the result type: + + - Source (no cache hit): only ``sdist_root_dir`` is set. + - Source (cache hit): both ``sdist_root_dir`` and ``cached_wheel_filename`` are set. + - Prebuilt wheel: both ``wheel_filename`` and ``unpack_dir`` are set. + """ + + # Source path: set after download+unpack OR cache hit + sdist_root_dir: pathlib.Path | None = None + # Source path: set when the result came from the wheel cache + cached_wheel_filename: pathlib.Path | None = None + # Prebuilt path: downloaded wheel file + wheel_filename: pathlib.Path | None = None + # Prebuilt path: unpack directory (created by mkdir) + unpack_dir: pathlib.Path | None = None + + +# Valid failure types for test mode error recording +FailureType = typing.Literal["resolution", "bootstrap", "hook", "dependency_extraction"] + + +class FailureRecord(typing.TypedDict): + """Record of a package that failed during bootstrap in test mode. + + Attributes: + package: The package name that failed. + version: The resolved version (None if resolution failed). + exception_type: The exception class name. + exception_message: The exception message string. + failure_type: Category of failure for analysis. + """ + + package: str + version: str | None + exception_type: str + exception_message: str + failure_type: FailureType + + +class BootstrapPhase(StrEnum): + """Processing phases for iterative bootstrap. + + All packages: RESOLVE -> START -> ... + Source packages: ... -> PREPARE_SOURCE -> PREPARE_BUILD -> BUILD + -> PROCESS_INSTALL_DEPS -> COMPLETE. + Prebuilt packages: ... -> PREPARE_SOURCE -> PROCESS_INSTALL_DEPS -> COMPLETE. + """ + + RESOLVE = "resolve" + START = "start" + PREPARE_SOURCE = "prepare-source" + PREPARE_BUILD = "prepare-build" + BUILD = "build" + PROCESS_INSTALL_DEPS = "process-install-deps" + COMPLETE = "complete" diff --git a/src/fromager/bootstrapper/_work_item.py b/src/fromager/bootstrapper/_work_item.py new file mode 100644 index 00000000..beb04973 --- /dev/null +++ b/src/fromager/bootstrapper/_work_item.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import dataclasses +import logging +import pathlib + +from packaging.requirements import Requirement +from packaging.version import Version + +from .. import build_environment +from ..requirements_file import RequirementType +from ._types import SourceBuildResult + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class WorkItem: + """A unit of work in the iterative bootstrap loop. + + Carries identity fields set at creation time and accumulated state + populated across phases as processing advances. The current phase is + encoded by the ``PhaseItem`` subclass wrapping this object. + + Items enter at the RESOLVE phase with only req and req_type set. + The RESOLVE phase populates source_url and resolved_version, then + creates new items at the START phase for each resolved version. + """ + + # Identity (set at creation) + req: Requirement + req_type: RequirementType + why_snapshot: list[tuple[RequirementType, Requirement, Version]] + parent: tuple[Requirement, Version] | None = None + + # Populated by RESOLVE phase (None until then) + source_url: str | None = None + resolved_version: Version | None = None + + build_sdist_only: bool = False + + # Accumulated state (populated during phases) + build_env: build_environment.BuildEnvironment | None = None + sdist_root_dir: pathlib.Path | None = None + unpack_dir: pathlib.Path | None = None + cached_wheel_filename: pathlib.Path | None = None + build_result: SourceBuildResult | None = None + pbi_pre_built: bool = False + exclusive_build: bool = False + build_system_deps: set[Requirement] = dataclasses.field(default_factory=set) + build_backend_deps: set[Requirement] = dataclasses.field(default_factory=set) + build_sdist_deps: set[Requirement] = dataclasses.field(default_factory=set) + + def is_build_requirement_context(self) -> bool: + """Return True if this item is being processed as part of a build requirement. + + A package is a build dependency if its own requirement type is + build_system, build_backend, or build_sdist, OR if it is an install + requirement of something that is itself a build dependency (checked + by walking the ``why_snapshot`` chain). + """ + if self.req_type.is_build_requirement: + logger.debug(f"is itself a build requirement: {self.req_type}") + return True + if not self.req_type.is_install_requirement: + logger.debug( + "is not an install requirement, not checking dependency chain for a build requirement" + ) + return False + for req_type, req, resolved_version in reversed(self.why_snapshot): + if req_type.is_build_requirement: + logger.debug( + f"is a build requirement because {req_type} dependency {req} ({resolved_version}) depends on it" + ) + return True + logger.debug("is not a build requirement") + return False diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index ea62506c..b116eada 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -12,6 +12,28 @@ from resolvelib.resolvers import ResolverException from fromager import bootstrapper, log +from fromager.bootstrapper._build_item import BuildItem +from fromager.bootstrapper._cache import ( + _bg_prepare_prebuilt, + _download_wheel_from_cache, + _find_cached_wheel, +) +from fromager.bootstrapper._phase_item import PhaseItem +from fromager.bootstrapper._prepare_source_item import ( + PrepareSourceItem, + _bg_prepare_source, +) +from fromager.bootstrapper._process_install_deps_item import ( + ProcessInstallDepsItem, + _get_install_dependencies, +) +from fromager.bootstrapper._resolve_item import ResolveItem +from fromager.bootstrapper._start_item import StartItem +from fromager.bootstrapper._types import ( + BootstrapPhase, + SourceBuildResult, +) +from fromager.bootstrapper._work_item import WorkItem from fromager.context import WorkContext from fromager.requirements_file import RequirementType, SourceType @@ -198,8 +220,8 @@ def test_explain(tmp_context: WorkContext) -> None: def _make_work_item( req_type: RequirementType, why_snapshot: list[tuple[RequirementType, Requirement, Version]] | None = None, -) -> bootstrapper.WorkItem: - return bootstrapper.WorkItem( +) -> WorkItem: + return WorkItem( req=Requirement("testpkg"), req_type=req_type, why_snapshot=why_snapshot or [], @@ -253,7 +275,7 @@ def test_find_cached_wheel_returns_tuple(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) # Call method (will return None, None since no wheels exist) - result = bootstrapper._find_cached_wheel( + result = _find_cached_wheel( bt.ctx, bt.cache_wheel_server_url, req=Requirement("test-package"), @@ -274,7 +296,7 @@ def test_get_install_dependencies_returns_list( wheel_file = pathlib.Path("/fake/package-1.0.0-py3-none-any.whl") unpack_dir = tmp_context.work_dir - result = bootstrapper._get_install_dependencies( + result = _get_install_dependencies( ctx=tmp_context, req=Requirement("test-package"), resolved_version=Version("1.0.0"), @@ -299,7 +321,7 @@ def test_phase_build_produces_source_build_result(tmp_context: WorkContext) -> N mock_sdist_root.parent.mkdir(parents=True, exist_ok=True) mock_wheel = tmp_context.work_dir / "package-1.0.0-py3-none-any.whl" - wi = bootstrapper.WorkItem( + wi = WorkItem( req=Requirement("test-package"), req_type=RequirementType.TOP_LEVEL, source_url="https://pypi.org/simple/test-package", @@ -312,7 +334,7 @@ def test_phase_build_produces_source_build_result(tmp_context: WorkContext) -> N build_backend_deps=set(), build_sdist_deps=set(), ) - item = bootstrapper.BuildItem(wi) + item = BuildItem(wi) # Set up why stack so _track_why works bt.why = [] @@ -325,10 +347,10 @@ def test_phase_build_produces_source_build_result(tmp_context: WorkContext) -> N result_items = item.run(bt) assert len(result_items) == 1 - assert isinstance(result_items[0], bootstrapper.ProcessInstallDepsItem) + assert isinstance(result_items[0], ProcessInstallDepsItem) result = result_items[0].work_item.build_result - assert isinstance(result, bootstrapper.SourceBuildResult) + assert isinstance(result, SourceBuildResult) assert result.wheel_filename == mock_wheel assert result.sdist_filename is None assert result.unpack_dir == mock_sdist_root.parent @@ -355,9 +377,9 @@ def test_multiple_versions_continues_on_error(tmp_context: WorkContext) -> None: build_phase_count = {"count": 0} def prepare_source_run( - self: bootstrapper.PrepareSourceItem, + self: PrepareSourceItem, bt_arg: bootstrapper.Bootstrapper, - ) -> list[bootstrapper.PhaseItem]: + ) -> list[PhaseItem]: build_phase_count["count"] += 1 if str(self.work_item.resolved_version) == "1.5": raise ValueError("Simulated failure for version 1.5") @@ -365,7 +387,7 @@ def prepare_source_run( req = Requirement("testpkg>=1.0") - with patch.object(bootstrapper.PrepareSourceItem, "run", prepare_source_run): + with patch.object(PrepareSourceItem, "run", prepare_source_run): with patch.object(bt, "has_been_seen", return_value=False): bt._bootstrap_one( req=req, @@ -413,7 +435,7 @@ def test_download_wheel_from_cache_bypasses_hooks( mock_find_all.side_effect = RuntimeError("no match") with patch("fromager.overrides.find_and_invoke") as mock_override: - result = bootstrapper._download_wheel_from_cache( + result = _download_wheel_from_cache( bt.ctx, bt.cache_wheel_server_url, req=Requirement("test-pkg"), @@ -455,7 +477,7 @@ def test_cache_lookup_resolver_exception_logs_info( side_effect=ResolverException("no matching version"), ), ): - result = bootstrapper._download_wheel_from_cache( + result = _download_wheel_from_cache( bt.ctx, bt.cache_wheel_server_url, req=Requirement("test-package"), @@ -487,7 +509,7 @@ def test_cache_lookup_request_exception_logs_warning( "fromager.resolver.find_all_matching_from_provider", side_effect=exc_class(exc_msg), ): - result = bootstrapper._download_wheel_from_cache( + result = _download_wheel_from_cache( bt.ctx, bt.cache_wheel_server_url, req=Requirement("test-package"), @@ -509,7 +531,7 @@ def test_cache_lookup_unexpected_exception_logs_warning( "fromager.resolver.find_all_matching_from_provider", side_effect=ValueError("unexpected parsing error"), ): - result = bootstrapper._download_wheel_from_cache( + result = _download_wheel_from_cache( bt.ctx, bt.cache_wheel_server_url, req=Requirement("test-package"), @@ -552,15 +574,15 @@ def test_cache_lookup_download_wheel_error_logs_warning( ], ), patch( - "fromager.bootstrapper.wheels.extract_info_from_wheel_file", + "fromager.wheels.extract_info_from_wheel_file", return_value=("test_package", "1.0.0", None, None), ), patch( - "fromager.bootstrapper.wheels.download_wheel", + "fromager.wheels.download_wheel", side_effect=exc_class(exc_msg), ), ): - result = bootstrapper._download_wheel_from_cache( + result = _download_wheel_from_cache( bt.ctx, bt.cache_wheel_server_url, req=Requirement("test-package"), @@ -576,7 +598,7 @@ def test_cache_lookup_no_cache_url_returns_none(tmp_context: WorkContext) -> Non bt = bootstrapper.Bootstrapper(tmp_context) bt.cache_wheel_server_url = "" - result = bootstrapper._download_wheel_from_cache( + result = _download_wheel_from_cache( bt.ctx, bt.cache_wheel_server_url, req=Requirement("test-package"), @@ -591,9 +613,9 @@ def _make_resolve_item( req_type: RequirementType = RequirementType.TOP_LEVEL, why_snapshot: list[tuple[RequirementType, Requirement, Version]] | None = None, parent: tuple[Requirement, Version] | None = None, -) -> bootstrapper.ResolveItem: - return bootstrapper.ResolveItem( - bootstrapper.WorkItem( +) -> ResolveItem: + return ResolveItem( + WorkItem( req=Requirement(req), req_type=req_type, why_snapshot=why_snapshot or [], @@ -603,7 +625,7 @@ def _make_resolve_item( def _record_and_load( - bt: bootstrapper.Bootstrapper, stack: list[bootstrapper.PhaseItem] + bt: bootstrapper.Bootstrapper, stack: list[PhaseItem] ) -> list[typing.Any]: bt._record_stack_state(stack) return typing.cast(list[typing.Any], json.loads(bt._stack_filename.read_text())) @@ -617,7 +639,7 @@ def test_record_stack_state_minimal_item(tmp_context: WorkContext) -> None: result = contents[0] assert result["req"] == "testpkg" assert result["req_type"] == str(RequirementType.TOP_LEVEL) - assert result["phase"] == str(bootstrapper.BootstrapPhase.RESOLVE) + assert result["phase"] == str(BootstrapPhase.RESOLVE) assert result["resolved_version"] is None assert result["source_url"] is None assert result["build_sdist_only"] is False @@ -635,8 +657,8 @@ def test_record_stack_state_full_item(tmp_context: WorkContext) -> None: parent_version = Version("2.0") why_snapshot = [(RequirementType.INSTALL, parent_req, parent_version)] - item = bootstrapper.BuildItem( - bootstrapper.WorkItem( + item = BuildItem( + WorkItem( req=Requirement("child-pkg>=1.0"), req_type=RequirementType.INSTALL, why_snapshot=why_snapshot, @@ -672,8 +694,8 @@ def test_record_stack_state_full_item(tmp_context: WorkContext) -> None: def test_record_stack_state_dep_sets_are_sorted(tmp_context: WorkContext) -> None: """Mixed-order dep sets come out alphabetically sorted.""" bt = bootstrapper.Bootstrapper(tmp_context) - item = bootstrapper.BuildItem( - bootstrapper.WorkItem( + item = BuildItem( + WorkItem( req=Requirement("mypkg"), req_type=RequirementType.TOP_LEVEL, why_snapshot=[], @@ -692,7 +714,7 @@ def test_record_stack_state_dep_sets_are_sorted(tmp_context: WorkContext) -> Non def test_record_stack_state_writes_file(tmp_context: WorkContext) -> None: """File is created; list length matches stack size.""" bt = bootstrapper.Bootstrapper(tmp_context) - stack: list[bootstrapper.PhaseItem] = [ + stack: list[PhaseItem] = [ _make_resolve_item("pkga"), _make_resolve_item("pkgb"), ] @@ -708,7 +730,7 @@ def test_record_stack_state_writes_file(tmp_context: WorkContext) -> None: def test_record_stack_state_ordering(tmp_context: WorkContext) -> None: """Index 0 = stack[-1] (next to pop); last index = stack[0].""" bt = bootstrapper.Bootstrapper(tmp_context) - stack: list[bootstrapper.PhaseItem] = [ + stack: list[PhaseItem] = [ _make_resolve_item("pkga"), _make_resolve_item("pkgb"), _make_resolve_item("pkgc"), @@ -743,7 +765,7 @@ def test_bootstrap_calls_record_stack_state(tmp_context: WorkContext) -> None: original = bt._record_stack_state - def counting_record(stack: list[bootstrapper.PhaseItem]) -> None: + def counting_record(stack: list[PhaseItem]) -> None: call_count["n"] += 1 original(stack) @@ -756,7 +778,7 @@ def counting_record(stack: list[bootstrapper.PhaseItem]) -> None: "resolve", return_value=[("https://pypi.test/testpkg-1.0.tar.gz", Version("1.0"))], ), - patch.object(bootstrapper.StartItem, "run", return_value=[]), + patch.object(StartItem, "run", return_value=[]), ): bt._bootstrap_one(req=req, req_type=RequirementType.TOP_LEVEL) @@ -766,7 +788,7 @@ def counting_record(stack: list[bootstrapper.PhaseItem]) -> None: def test_bootstrap_with_empty_list(tmp_context: WorkContext) -> None: """bootstrap([]) completes without error and runs no phases.""" bt = bootstrapper.Bootstrapper(tmp_context) - with patch.object(bootstrapper.ResolveItem, "run") as mock_run: + with patch.object(ResolveItem, "run") as mock_run: bt.bootstrap([]) mock_run.assert_not_called() @@ -775,11 +797,11 @@ def test_bootstrap_with_single_requirement(tmp_context: WorkContext) -> None: """bootstrap([req]) resolves and processes the requirement.""" bt = bootstrapper.Bootstrapper(tmp_context) req = Requirement("testpkg==1.0") - captured: list[bootstrapper.ResolveItem] = [] + captured: list[ResolveItem] = [] def capture_run( - self: bootstrapper.ResolveItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[bootstrapper.PhaseItem]: + self: ResolveItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: captured.append(self) return [] @@ -789,13 +811,13 @@ def capture_run( "_resolve_and_add_top_level", return_value=("http://example.test/testpkg-1.0.tar.gz", Version("1.0")), ), - patch.object(bootstrapper.ResolveItem, "run", capture_run), + patch.object(ResolveItem, "run", capture_run), patch.object(bt, "_record_stack_state"), ): bt.bootstrap([req]) assert len(captured) == 1 - assert isinstance(captured[0], bootstrapper.ResolveItem) + assert isinstance(captured[0], ResolveItem) assert captured[0].work_item.req == req assert captured[0].work_item.req_type == RequirementType.TOP_LEVEL @@ -807,7 +829,7 @@ def test_bootstrap_skips_failed_resolution(tmp_context: WorkContext) -> None: with ( patch.object(bt, "_resolve_and_add_top_level", return_value=None), - patch.object(bootstrapper.ResolveItem, "run") as mock_run, + patch.object(ResolveItem, "run") as mock_run, patch.object(bt, "_record_stack_state"), ): bt.bootstrap([req]) @@ -824,8 +846,8 @@ def test_bootstrap_two_requirements_both_processed(tmp_context: WorkContext) -> dispatch_calls: list = [] def capture_run( - self: bootstrapper.ResolveItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[bootstrapper.PhaseItem]: + self: ResolveItem, bt_arg: bootstrapper.Bootstrapper + ) -> list[PhaseItem]: dispatch_calls.append(self.work_item.req.name) return [] @@ -835,7 +857,7 @@ def capture_run( "_resolve_and_add_top_level", return_value=("http://example.test/pkg-1.0.tar.gz", Version("1.0")), ), - patch.object(bootstrapper.ResolveItem, "run", capture_run), + patch.object(ResolveItem, "run", capture_run), patch.object(bt, "_record_stack_state"), ): bt.bootstrap([req1, req2]) @@ -858,7 +880,7 @@ def test_bg_prepare_source_log_prefix_includes_version( with ( caplog.at_level(logging.INFO, logger="fromager.bootstrapper"), patch( - "fromager.bootstrapper._find_cached_wheel", + "fromager.bootstrapper._cache._find_cached_wheel", return_value=(None, None), ), patch( @@ -871,7 +893,7 @@ def test_bg_prepare_source_log_prefix_includes_version( ), log.req_ctxvar_context(req, version), ): - bootstrapper._bg_prepare_source( + _bg_prepare_source( ctx=tmp_context, cache_wheel_server_url=None, req=req, @@ -883,7 +905,7 @@ def test_bg_prepare_source_log_prefix_includes_version( messages = [ r.getMessage() for r in caplog.records - if r.name == "fromager.bootstrapper" + if r.name.startswith("fromager.bootstrapper") ] finally: logging.setLogRecordFactory(old_factory) @@ -915,7 +937,7 @@ def test_bg_prepare_prebuilt_log_prefix_includes_version( patch("fromager.server.update_wheel_mirror"), log.req_ctxvar_context(req, version), ): - bootstrapper._bg_prepare_prebuilt( + _bg_prepare_prebuilt( ctx=tmp_context, req=req, req_type=RequirementType.INSTALL, @@ -927,7 +949,7 @@ def test_bg_prepare_prebuilt_log_prefix_includes_version( messages = [ r.getMessage() for r in caplog.records - if r.name == "fromager.bootstrapper" + if r.name.startswith("fromager.bootstrapper") ] finally: logging.setLogRecordFactory(old_factory) @@ -943,7 +965,7 @@ def test_build_item_build_sdist_finds_existing(tmp_context: WorkContext) -> None """BuildItem._build_sdist returns cached sdist when finders.find_sdist hits.""" sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" sdist_root.mkdir(parents=True, exist_ok=True) - wi = bootstrapper.WorkItem( + wi = WorkItem( req=Requirement("testpkg"), req_type=RequirementType.TOP_LEVEL, why_snapshot=[], @@ -951,7 +973,7 @@ def test_build_item_build_sdist_finds_existing(tmp_context: WorkContext) -> None sdist_root_dir=sdist_root, build_env=Mock(), ) - item = bootstrapper.BuildItem(wi) + item = BuildItem(wi) cached = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" cached.touch() @@ -967,7 +989,7 @@ def test_build_item_build_sdist_calls_build_when_not_cached( """BuildItem._build_sdist calls sources.build_sdist when no cached sdist found.""" sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" sdist_root.mkdir(parents=True, exist_ok=True) - wi = bootstrapper.WorkItem( + wi = WorkItem( req=Requirement("testpkg"), req_type=RequirementType.TOP_LEVEL, why_snapshot=[], @@ -975,7 +997,7 @@ def test_build_item_build_sdist_calls_build_when_not_cached( sdist_root_dir=sdist_root, build_env=Mock(), ) - item = bootstrapper.BuildItem(wi) + item = BuildItem(wi) built = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" with ( @@ -998,7 +1020,7 @@ def test_build_item_build_wheel(tmp_context: WorkContext) -> None: """BuildItem._build_wheel builds sdist then wheel and updates mirror.""" sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" sdist_root.mkdir(parents=True, exist_ok=True) - wi = bootstrapper.WorkItem( + wi = WorkItem( req=Requirement("testpkg"), req_type=RequirementType.TOP_LEVEL, why_snapshot=[], @@ -1006,7 +1028,7 @@ def test_build_item_build_wheel(tmp_context: WorkContext) -> None: sdist_root_dir=sdist_root, build_env=Mock(), ) - item = bootstrapper.BuildItem(wi) + item = BuildItem(wi) built_wheel = tmp_context.wheels_build / "testpkg-1.0-py3-none-any.whl" built_sdist = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" @@ -1030,7 +1052,7 @@ def test_build_item_do_build_returns_cached_wheel( """BuildItem.do_build returns cached wheel immediately without building.""" cached = tmp_context.wheels_downloads / "testpkg-1.0-py3-none-any.whl" cached.touch() - wi = bootstrapper.WorkItem( + wi = WorkItem( req=Requirement("testpkg"), req_type=RequirementType.TOP_LEVEL, why_snapshot=[], @@ -1040,7 +1062,7 @@ def test_build_item_do_build_returns_cached_wheel( sdist_root_dir=tmp_context.work_dir, build_env=Mock(), ) - item = bootstrapper.BuildItem(wi) + item = BuildItem(wi) with patch.object(item, "_build_wheel") as mock_wheel: wheel, sdist = item.do_build(tmp_context) @@ -1055,7 +1077,7 @@ def test_build_item_do_build_sdist_only(tmp_context: WorkContext) -> None: built_sdist = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" sdist_root.mkdir(parents=True, exist_ok=True) - wi = bootstrapper.WorkItem( + wi = WorkItem( req=Requirement("testpkg"), req_type=RequirementType.TOP_LEVEL, why_snapshot=[], @@ -1065,7 +1087,7 @@ def test_build_item_do_build_sdist_only(tmp_context: WorkContext) -> None: sdist_root_dir=sdist_root, build_env=Mock(), ) - item = bootstrapper.BuildItem(wi) + item = BuildItem(wi) with ( patch.object(item, "_build_sdist", return_value=built_sdist) as mock_sdist, @@ -1085,7 +1107,7 @@ def test_build_item_do_build_builds_wheel(tmp_context: WorkContext) -> None: built_sdist = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" sdist_root.mkdir(parents=True, exist_ok=True) - wi = bootstrapper.WorkItem( + wi = WorkItem( req=Requirement("testpkg"), req_type=RequirementType.TOP_LEVEL, why_snapshot=[], @@ -1095,7 +1117,7 @@ def test_build_item_do_build_builds_wheel(tmp_context: WorkContext) -> None: sdist_root_dir=sdist_root, build_env=Mock(), ) - item = bootstrapper.BuildItem(wi) + item = BuildItem(wi) with patch.object( item, "_build_wheel", return_value=(built_wheel, built_sdist) diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index c17434d5..0d74ccf8 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -29,19 +29,20 @@ from packaging.version import Version from fromager import bootstrapper, build_environment -from fromager.bootstrapper import ( +from fromager.bootstrapper._build_item import BuildItem +from fromager.bootstrapper._complete_item import CompleteItem +from fromager.bootstrapper._phase_item import PhaseItem +from fromager.bootstrapper._prepare_build_item import PrepareBuildItem +from fromager.bootstrapper._prepare_source_item import PrepareSourceItem +from fromager.bootstrapper._process_install_deps_item import ProcessInstallDepsItem +from fromager.bootstrapper._resolve_item import ResolveItem +from fromager.bootstrapper._start_item import StartItem +from fromager.bootstrapper._types import ( BootstrapPhase, - BuildItem, - CompleteItem, - PhaseItem, - PrepareBuildItem, - PrepareSourceItem, - ProcessInstallDepsItem, - ResolveItem, + PreparedSourceData, SourceBuildResult, - StartItem, - WorkItem, ) +from fromager.bootstrapper._work_item import WorkItem from fromager.context import WorkContext from fromager.requirements_file import RequirementType, SourceType @@ -423,7 +424,9 @@ def mock_cache( return (tmp_context.work_dir / "pkg-2.0-py3-none-any.whl", None) return (None, None) - with patch("fromager.bootstrapper._find_cached_wheel", side_effect=mock_cache): + with patch( + "fromager.bootstrapper._cache._find_cached_wheel", side_effect=mock_cache + ): result = item.run(bt) assert len(result) == 2 @@ -443,7 +446,7 @@ def test_all_cached_keeps_highest_version(self, tmp_context: WorkContext) -> Non ) with patch( - "fromager.bootstrapper._find_cached_wheel", + "fromager.bootstrapper._cache._find_cached_wheel", return_value=(tmp_context.work_dir / "cached.whl", None), ): result = item.run(bt) @@ -459,7 +462,7 @@ def test_no_filtering_in_single_version_mode( item = _make_resolve_item() item.bg_future = _make_resolved_future([("url-1.0", Version("1.0"))]) - with patch("fromager.bootstrapper._find_cached_wheel") as mock_cache: + with patch("fromager.bootstrapper._cache._find_cached_wheel") as mock_cache: result = item.run(bt) assert len(result) == 1 @@ -496,7 +499,7 @@ def test_filters_failed_versions_in_multiple_versions_mode( ) with patch( - "fromager.bootstrapper._find_cached_wheel", return_value=(None, None) + "fromager.bootstrapper._cache._find_cached_wheel", return_value=(None, None) ): result = item.run(bt) @@ -1169,9 +1172,7 @@ def test_prebuilt_uses_background_result(self, tmp_context: WorkContext) -> None mock_wheel = tmp_context.work_dir / "testpkg-1.0-py3-none-any.whl" mock_unpack = tmp_context.work_dir / "testpkg-1.0" item.bg_future = _make_resolved_future( - bootstrapper.PreparedSourceData( - wheel_filename=mock_wheel, unpack_dir=mock_unpack - ) + PreparedSourceData(wheel_filename=mock_wheel, unpack_dir=mock_unpack) ) with patch.object(tmp_context.constraints, "get_constraint", return_value=None): @@ -1198,7 +1199,7 @@ def test_source_no_cache_uses_background_result( mock_env = Mock() mock_dep_item = _make_resolve_item(req="setuptools") item.bg_future = _make_resolved_future( - bootstrapper.PreparedSourceData(sdist_root_dir=sdist_root) + PreparedSourceData(sdist_root_dir=sdist_root) ) with ( @@ -1248,7 +1249,7 @@ def test_source_cached_wheel_uses_background_result( cached_wheel = tmp_context.work_dir / "testpkg-1.0-py3-none-any.whl" mock_env = Mock() item.bg_future = _make_resolved_future( - bootstrapper.PreparedSourceData( + PreparedSourceData( sdist_root_dir=unpacked / unpacked.stem, cached_wheel_filename=cached_wheel, ) @@ -1281,7 +1282,7 @@ def test_bad_sdist_root_raises_valueerror(self, tmp_context: WorkContext) -> Non bad_root = tmp_context.work_dir / "a" / "b" / "c" item.bg_future = _make_resolved_future( - bootstrapper.PreparedSourceData(sdist_root_dir=bad_root) + PreparedSourceData(sdist_root_dir=bad_root) ) with patch.object(tmp_context.constraints, "get_constraint", return_value=None): @@ -1298,7 +1299,7 @@ def test_constraint_logged_when_present( sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" mock_env = Mock() item.bg_future = _make_resolved_future( - bootstrapper.PreparedSourceData(sdist_root_dir=sdist_root) + PreparedSourceData(sdist_root_dir=sdist_root) ) with ( @@ -1699,7 +1700,7 @@ def test_normal_path_returns_item_and_dep_items( with ( patch("fromager.hooks.run_post_bootstrap_hooks"), patch( - "fromager.bootstrapper._get_install_dependencies", + "fromager.bootstrapper._process_install_deps_item._get_install_dependencies", return_value=[Requirement("dep-a")], ), patch.object( @@ -1739,7 +1740,8 @@ def test_hook_error_test_mode_records_and_continues( side_effect=RuntimeError("hook failed"), ), patch( - "fromager.bootstrapper._get_install_dependencies", return_value=[] + "fromager.bootstrapper._process_install_deps_item._get_install_dependencies", + return_value=[], ) as mock_get_deps, patch.object( tmp_context, @@ -1783,7 +1785,7 @@ def test_dep_extraction_error_test_mode_uses_empty_deps( with ( patch("fromager.hooks.run_post_bootstrap_hooks"), patch( - "fromager.bootstrapper._get_install_dependencies", + "fromager.bootstrapper._process_install_deps_item._get_install_dependencies", side_effect=RuntimeError("dep failed"), ), patch.object( @@ -1820,7 +1822,7 @@ def test_dep_extraction_error_normal_mode_raises( with ( patch("fromager.hooks.run_post_bootstrap_hooks"), patch( - "fromager.bootstrapper._get_install_dependencies", + "fromager.bootstrapper._process_install_deps_item._get_install_dependencies", side_effect=RuntimeError("dep failed"), ), ): @@ -1834,7 +1836,10 @@ def test_no_install_deps_returns_item_only(self, tmp_context: WorkContext) -> No with ( patch("fromager.hooks.run_post_bootstrap_hooks"), - patch("fromager.bootstrapper._get_install_dependencies", return_value=[]), + patch( + "fromager.bootstrapper._process_install_deps_item._get_install_dependencies", + return_value=[], + ), patch.object( tmp_context, "package_build_info", @@ -1860,7 +1865,10 @@ def test_build_order_called_with_correct_args( with ( patch("fromager.hooks.run_post_bootstrap_hooks"), - patch("fromager.bootstrapper._get_install_dependencies", return_value=[]), + patch( + "fromager.bootstrapper._process_install_deps_item._get_install_dependencies", + return_value=[], + ), patch.object( tmp_context, "package_build_info", From f98b5760db09f2bbff24722f4f62de1afda77582 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 2 Jul 2026 18:48:34 -0400 Subject: [PATCH 15/19] refactor(bootstrapper): rename phase classes and files, drop "Item" suffix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename all phase subclasses of `PhaseItem` to drop the redundant "Item" suffix, and rename the abstract base `PhaseItem` to `Phase`. File names follow the class names. Class renames: - PhaseItem → Phase - ResolveItem → Resolve - StartItem → Start - PrepareSourceItem → PrepareSource - PrepareBuildItem → PrepareBuild - BuildItem → Build - ProcessInstallDepsItem → ProcessInstallDeps - CompleteItem → Complete File renames (in src/fromager/bootstrapper/): - _phase_item.py → _phase.py - _resolve_item.py → _resolve.py - _start_item.py → _start.py - _prepare_source_item.py → _prepare_source.py - _prepare_build_item.py → _prepare_build.py - _build_item.py → _build.py - _process_install_deps_item.py → _process_install_deps.py - _complete_item.py → _complete.py `WorkItem` and `_work_item.py` are unchanged. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper/_bootstrapper.py | 50 +-- .../{_build_item.py => _build.py} | 12 +- .../{_complete_item.py => _complete.py} | 6 +- .../{_phase_item.py => _phase.py} | 4 +- ...repare_build_item.py => _prepare_build.py} | 16 +- ...pare_source_item.py => _prepare_source.py} | 20 +- ..._deps_item.py => _process_install_deps.py} | 14 +- .../{_resolve_item.py => _resolve.py} | 18 +- .../{_start_item.py => _start.py} | 16 +- src/fromager/bootstrapper/_work_item.py | 2 +- tests/test_bootstrapper.py | 90 +++-- tests/test_bootstrapper_iterative.py | 318 +++++++++--------- 12 files changed, 278 insertions(+), 288 deletions(-) rename src/fromager/bootstrapper/{_build_item.py => _build.py} (95%) rename src/fromager/bootstrapper/{_complete_item.py => _complete.py} (85%) rename src/fromager/bootstrapper/{_phase_item.py => _phase.py} (97%) rename src/fromager/bootstrapper/{_prepare_build_item.py => _prepare_build.py} (86%) rename src/fromager/bootstrapper/{_prepare_source_item.py => _prepare_source.py} (91%) rename src/fromager/bootstrapper/{_process_install_deps_item.py => _process_install_deps.py} (93%) rename src/fromager/bootstrapper/{_resolve_item.py => _resolve.py} (93%) rename src/fromager/bootstrapper/{_start_item.py => _start.py} (84%) diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py index b653397c..4d65f3f5 100644 --- a/src/fromager/bootstrapper/_bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -27,12 +27,12 @@ from ..log import req_ctxvar_context, requirement_ctxvar from ..requirements_file import RequirementType, SourceType from . import _cache -from ._build_item import BuildItem -from ._phase_item import PhaseItem -from ._prepare_build_item import PrepareBuildItem -from ._prepare_source_item import PrepareSourceItem -from ._process_install_deps_item import ProcessInstallDepsItem -from ._resolve_item import ResolveItem +from ._build import Build +from ._phase import Phase +from ._prepare_build import PrepareBuild +from ._prepare_source import PrepareSource +from ._process_install_deps import ProcessInstallDeps +from ._resolve import Resolve from ._types import ( _DEFAULT_BG_THREADS, FailureRecord, @@ -269,15 +269,15 @@ def bootstrap(self, requirements: list[Requirement]) -> None: # Use the token pattern (no try/finally) so that if resolution raises # in normal mode, the context var stays set for the top-level error # handler in __main__.py to include the package name in its log message. - stack: list[PhaseItem] = [] - initial_items: list[PhaseItem] = [] + stack: list[Phase] = [] + initial_items: list[Phase] = [] for req in requirements: token = requirement_ctxvar.set(req) result = self._resolve_and_add_top_level(req) requirement_ctxvar.reset(token) if result is not None: initial_items.append( - ResolveItem( + Resolve( WorkItem( req=req, req_type=RequirementType.TOP_LEVEL, @@ -290,7 +290,7 @@ def bootstrap(self, requirements: list[Requirement]) -> None: self._run_bootstrap_loop(stack) - def _run_bootstrap_loop(self, stack: list[PhaseItem]) -> None: + def _run_bootstrap_loop(self, stack: list[Phase]) -> None: """Run the iterative DFS bootstrap loop over a pre-built work stack. Pops items one at a time, dispatches each phase, and pushes any @@ -298,7 +298,7 @@ def _run_bootstrap_loop(self, stack: list[PhaseItem]) -> None: stack. Updates the progress bar as items complete. Args: - stack: Initial list of ``PhaseItem`` objects to process. Modified + stack: Initial list of ``Phase`` objects to process. Modified in-place; empty on return. """ while stack: @@ -322,7 +322,7 @@ def _run_bootstrap_loop(self, stack: list[PhaseItem]) -> None: except Exception as err: new_items = self._handle_phase_error(item, err) - new_dep_count = sum(1 for it in new_items if isinstance(it, ResolveItem)) + new_dep_count = sum(1 for it in new_items if isinstance(it, Resolve)) if new_dep_count > 0: self.progressbar.update_total(new_dep_count) if not new_items: @@ -359,8 +359,8 @@ def _bootstrap_one(self, req: Requirement, req_type: RequirementType) -> None: saved_why = list(self.why) # Single RESOLVE item — resolution, version expansion, and error - # handling all happen inside the loop via ResolveItem.run(). - initial_item = ResolveItem( + # handling all happen inside the loop via Resolve.run(). + initial_item = Resolve( WorkItem( req=req, req_type=req_type, @@ -368,7 +368,7 @@ def _bootstrap_one(self, req: Requirement, req_type: RequirementType) -> None: parent=parent, ) ) - stack: list[PhaseItem] = [] + stack: list[Phase] = [] self._push_items(stack, [initial_item]) self._run_bootstrap_loop(stack) @@ -383,7 +383,7 @@ def _bootstrap_one(self, req: Requirement, req_type: RequirementType) -> None: @contextlib.contextmanager def _track_why( self, - item: PhaseItem, + item: Phase, ) -> typing.Generator[None, None, None]: """Context manager to track dependency chain in self.why stack. @@ -917,7 +917,7 @@ def add_to_build_order( # converted to JSON without help. json.dump(self._build_stack, f, indent=2, default=str) - def _record_stack_state(self, stack: list[PhaseItem]) -> None: + def _record_stack_state(self, stack: list[Phase]) -> None: """Write the current bootstrap stack to `self._stack_filename`. Index 0 in the output corresponds to `stack[-1]`, the next item to be @@ -929,7 +929,7 @@ def _record_stack_state(self, stack: list[PhaseItem]) -> None: # ---- Iterative bootstrap: phase handlers and helpers ---- - def _push_items(self, stack: list[PhaseItem], items: list[PhaseItem]) -> None: + def _push_items(self, stack: list[Phase], items: list[Phase]) -> None: """Push items onto the stack and submit background tasks in LIFO order. Submits the item that will be processed first (top of stack) to the @@ -962,7 +962,7 @@ def create_unresolved_work_items( dep_req_type: RequirementType, parent_req: Requirement, parent_version: Version, - ) -> list[PhaseItem]: + ) -> list[Phase]: """Create RESOLVE-phase work items for dependencies. Called inside a parent's _track_why context so that why_snapshot @@ -970,7 +970,7 @@ def create_unresolved_work_items( handling happen later when each item's RESOLVE phase runs. """ return [ - ResolveItem( + Resolve( WorkItem( req=dep, req_type=dep_req_type, @@ -1086,9 +1086,9 @@ def filter_deps_satisfied_by_build_system( def _handle_phase_error( self, - item: PhaseItem, + item: Phase, err: Exception, - ) -> list[PhaseItem]: + ) -> list[Phase]: """Handle errors from phase processing. Returns work items to continue processing (e.g. prebuilt fallback), @@ -1096,7 +1096,7 @@ def _handle_phase_error( """ wi = item.work_item # Resolution failures: recoverable in test mode and multiple versions mode - if isinstance(item, ResolveItem): + if isinstance(item, Resolve): if self.test_mode: self.record_test_mode_failure(wi.req, None, err, "resolution") if self.multiple_versions: @@ -1120,7 +1120,7 @@ def _handle_phase_error( # Test mode: try prebuilt fallback for build-related phases if self.test_mode: if ( - isinstance(item, PrepareSourceItem | PrepareBuildItem | BuildItem) + isinstance(item, PrepareSource | PrepareBuild | Build) and not wi.pbi_pre_built ): assert wi.resolved_version is not None @@ -1132,7 +1132,7 @@ def _handle_phase_error( ) if fallback is not None: wi.build_result = fallback - return [ProcessInstallDepsItem(wi)] + return [ProcessInstallDeps(wi)] self.record_test_mode_failure( wi.req, str(wi.resolved_version), err, "bootstrap" ) diff --git a/src/fromager/bootstrapper/_build_item.py b/src/fromager/bootstrapper/_build.py similarity index 95% rename from src/fromager/bootstrapper/_build_item.py rename to src/fromager/bootstrapper/_build.py index d3f93b03..6c06059b 100644 --- a/src/fromager/bootstrapper/_build_item.py +++ b/src/fromager/bootstrapper/_build.py @@ -5,8 +5,8 @@ import typing from .. import finders, server, sources, wheels -from ._phase_item import PhaseItem -from ._process_install_deps_item import ProcessInstallDepsItem +from ._phase import Phase +from ._process_install_deps import ProcessInstallDeps from ._types import BootstrapPhase, SourceBuildResult if typing.TYPE_CHECKING: @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) -class BuildItem(PhaseItem): +class Build(Phase): """BUILD phase: install remaining deps, build wheel/sdist.""" phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.BUILD @@ -26,11 +26,11 @@ class BuildItem(PhaseItem): def requires_exclusive_run(self) -> bool: return self.work_item.exclusive_build - def run(self, bt: Bootstrapper) -> list[PhaseItem]: + def run(self, bt: Bootstrapper) -> list[Phase]: """BUILD phase: install remaining deps, build wheel/sdist. Returns: - [ProcessInstallDepsItem]. + [ProcessInstallDeps]. """ wi = self.work_item assert wi.resolved_version is not None @@ -55,7 +55,7 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: source_type=source_type, ) - return [ProcessInstallDepsItem(wi)] + return [ProcessInstallDeps(wi)] def _build_sdist(self, ctx: context.WorkContext) -> pathlib.Path: """Build or locate an sdist for this item's package. diff --git a/src/fromager/bootstrapper/_complete_item.py b/src/fromager/bootstrapper/_complete.py similarity index 85% rename from src/fromager/bootstrapper/_complete_item.py rename to src/fromager/bootstrapper/_complete.py index 574c8615..06559ae5 100644 --- a/src/fromager/bootstrapper/_complete_item.py +++ b/src/fromager/bootstrapper/_complete.py @@ -2,20 +2,20 @@ import typing -from ._phase_item import PhaseItem +from ._phase import Phase from ._types import BootstrapPhase if typing.TYPE_CHECKING: from ._bootstrapper import Bootstrapper -class CompleteItem(PhaseItem): +class Complete(Phase): """COMPLETE phase: clean up build directories.""" phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.COMPLETE tracks_why: typing.ClassVar[bool] = True - def run(self, bt: Bootstrapper) -> list[PhaseItem]: + def run(self, bt: Bootstrapper) -> list[Phase]: """COMPLETE phase: clean up build directories. Returns: diff --git a/src/fromager/bootstrapper/_phase_item.py b/src/fromager/bootstrapper/_phase.py similarity index 97% rename from src/fromager/bootstrapper/_phase_item.py rename to src/fromager/bootstrapper/_phase.py index 95379a6e..90f83458 100644 --- a/src/fromager/bootstrapper/_phase_item.py +++ b/src/fromager/bootstrapper/_phase.py @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) -class PhaseItem(abc.ABC): +class Phase(abc.ABC): """Abstract base for items pushed onto the bootstrap stack. Each subclass encodes one phase of the bootstrap pipeline. @@ -30,7 +30,7 @@ def __init__(self, work_item: WorkItem) -> None: self.bg_future: concurrent.futures.Future[typing.Any] | None = None @abc.abstractmethod - def run(self, bt: Bootstrapper) -> list[PhaseItem]: ... + def run(self, bt: Bootstrapper) -> list[Phase]: ... @property def requires_exclusive_run(self) -> bool: diff --git a/src/fromager/bootstrapper/_prepare_build_item.py b/src/fromager/bootstrapper/_prepare_build.py similarity index 86% rename from src/fromager/bootstrapper/_prepare_build_item.py rename to src/fromager/bootstrapper/_prepare_build.py index 87c3f89b..d673d61c 100644 --- a/src/fromager/bootstrapper/_prepare_build_item.py +++ b/src/fromager/bootstrapper/_prepare_build.py @@ -5,8 +5,8 @@ from .. import dependencies from ..requirements_file import RequirementType -from ._build_item import BuildItem -from ._phase_item import PhaseItem +from ._build import Build +from ._phase import Phase from ._types import BootstrapPhase if typing.TYPE_CHECKING: @@ -15,13 +15,13 @@ logger = logging.getLogger(__name__) -class PrepareBuildItem(PhaseItem): +class PrepareBuild(Phase): """PREPARE_BUILD phase: install system deps, get backend/sdist deps.""" phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PREPARE_BUILD tracks_why: typing.ClassVar[bool] = True - def run(self, bt: Bootstrapper) -> list[PhaseItem]: + def run(self, bt: Bootstrapper) -> list[Phase]: """PREPARE_BUILD phase: install system deps, get backend/sdist deps. Build-backend and build-sdist dependencies that are already satisfied @@ -29,7 +29,7 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: resolving independently (see :issue:`1194`). Returns: - [BuildItem, *backend_dep_items, *sdist_dep_items]. + [Build, *backend_dep_items, *sdist_dep_items]. """ wi = self.work_item assert wi.resolved_version is not None @@ -74,13 +74,13 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: parent, ) - backend_items: list[PhaseItem] = bt.create_unresolved_work_items( + backend_items: list[Phase] = bt.create_unresolved_work_items( wi.build_backend_deps, RequirementType.BUILD_BACKEND, wi.req, wi.resolved_version, ) - sdist_items: list[PhaseItem] = bt.create_unresolved_work_items( + sdist_items: list[Phase] = bt.create_unresolved_work_items( wi.build_sdist_deps, RequirementType.BUILD_SDIST, wi.req, @@ -88,4 +88,4 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: ) dep_items = backend_items + sdist_items - return [BuildItem(wi)] + dep_items + return [Build(wi)] + dep_items diff --git a/src/fromager/bootstrapper/_prepare_source_item.py b/src/fromager/bootstrapper/_prepare_source.py similarity index 91% rename from src/fromager/bootstrapper/_prepare_source_item.py rename to src/fromager/bootstrapper/_prepare_source.py index bc1cd8c6..c0957a49 100644 --- a/src/fromager/bootstrapper/_prepare_source_item.py +++ b/src/fromager/bootstrapper/_prepare_source.py @@ -10,9 +10,9 @@ from ..log import req_ctxvar_context from ..requirements_file import RequirementType, SourceType from . import _cache -from ._phase_item import PhaseItem -from ._prepare_build_item import PrepareBuildItem -from ._process_install_deps_item import ProcessInstallDepsItem +from ._phase import Phase +from ._prepare_build import PrepareBuild +from ._process_install_deps import ProcessInstallDeps from ._types import BootstrapPhase, PreparedSourceData, SourceBuildResult if typing.TYPE_CHECKING: @@ -52,7 +52,7 @@ def _bg_prepare_source( return PreparedSourceData(sdist_root_dir=sdist_root_dir) -class PrepareSourceItem(PhaseItem): +class PrepareSource(Phase): """PREPARE_SOURCE phase: download source or prebuilt, get build system deps.""" phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PREPARE_SOURCE @@ -90,15 +90,15 @@ def do_prepare_source() -> PreparedSourceData: return do_prepare_source - def run(self, bt: Bootstrapper) -> list[PhaseItem]: + def run(self, bt: Bootstrapper) -> list[Phase]: """PREPARE_SOURCE phase: download source or prebuilt, get build system deps. Uses background I/O result from ``self.bg_future`` when available, falling back to inline I/O otherwise. Returns: - Prebuilt: [ProcessInstallDepsItem] (skip build phases). - Source: [PrepareBuildItem, *build_system_dep_items]. + Prebuilt: [ProcessInstallDeps] (skip build phases). + Source: [PrepareBuild, *build_system_dep_items]. """ wi = self.work_item assert wi.resolved_version is not None @@ -128,7 +128,7 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: build_env=None, source_type=SourceType.PREBUILT, ) - return [ProcessInstallDepsItem(wi)] + return [ProcessInstallDeps(wi)] # Source build path: background task already downloaded and prepared the source assert prepared.sdist_root_dir is not None @@ -155,11 +155,11 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: sdist_root_dir=sdist_root_dir, ) - dep_items: list[PhaseItem] = bt.create_unresolved_work_items( + dep_items: list[Phase] = bt.create_unresolved_work_items( wi.build_system_deps, RequirementType.BUILD_SYSTEM, wi.req, wi.resolved_version, ) - return [PrepareBuildItem(wi)] + dep_items + return [PrepareBuild(wi)] + dep_items diff --git a/src/fromager/bootstrapper/_process_install_deps_item.py b/src/fromager/bootstrapper/_process_install_deps.py similarity index 93% rename from src/fromager/bootstrapper/_process_install_deps_item.py rename to src/fromager/bootstrapper/_process_install_deps.py index 5f79b549..42e04498 100644 --- a/src/fromager/bootstrapper/_process_install_deps_item.py +++ b/src/fromager/bootstrapper/_process_install_deps.py @@ -10,8 +10,8 @@ from .. import build_environment, dependencies, hooks from ..requirements_file import RequirementType -from ._complete_item import CompleteItem -from ._phase_item import PhaseItem +from ._complete import Complete +from ._phase import Phase from ._types import BootstrapPhase if typing.TYPE_CHECKING: @@ -72,17 +72,17 @@ def _get_install_dependencies( raise RuntimeError("wheel_filename and sdist_filename are None") -class ProcessInstallDepsItem(PhaseItem): +class ProcessInstallDeps(Phase): """PROCESS_INSTALL_DEPS phase: hooks, extract deps, build order.""" phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PROCESS_INSTALL_DEPS tracks_why: typing.ClassVar[bool] = True - def run(self, bt: Bootstrapper) -> list[PhaseItem]: + def run(self, bt: Bootstrapper) -> list[Phase]: """PROCESS_INSTALL_DEPS phase: hooks, extract deps, build order. Returns: - [CompleteItem, *install_dep_items]. + [Complete, *install_dep_items]. """ wi = self.work_item assert wi.resolved_version is not None @@ -150,11 +150,11 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: constraint=constraint, ) - dep_items: list[PhaseItem] = bt.create_unresolved_work_items( + dep_items: list[Phase] = bt.create_unresolved_work_items( install_dependencies, RequirementType.INSTALL, wi.req, wi.resolved_version, ) - return [CompleteItem(wi)] + dep_items + return [Complete(wi)] + dep_items diff --git a/src/fromager/bootstrapper/_resolve_item.py b/src/fromager/bootstrapper/_resolve.py similarity index 93% rename from src/fromager/bootstrapper/_resolve_item.py rename to src/fromager/bootstrapper/_resolve.py index c793ab25..02221843 100644 --- a/src/fromager/bootstrapper/_resolve_item.py +++ b/src/fromager/bootstrapper/_resolve.py @@ -11,8 +11,8 @@ from ..log import req_ctxvar_context from ..requirements_file import RequirementType from . import _cache -from ._phase_item import PhaseItem -from ._start_item import StartItem +from ._phase import Phase +from ._start import Start from ._types import BootstrapPhase from ._work_item import WorkItem @@ -39,8 +39,8 @@ def _bg_resolve( ) -class ResolveItem(PhaseItem): - """RESOLVE phase: resolve versions and expand into StartItems.""" +class Resolve(Phase): + """RESOLVE phase: resolve versions and expand into Start items.""" phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.RESOLVE tracks_why: typing.ClassVar[bool] = False @@ -63,8 +63,8 @@ def do_resolve() -> list[tuple[str, Version]]: return do_resolve - def run(self, bt: Bootstrapper) -> list[PhaseItem]: - """RESOLVE phase: resolve versions and expand into StartItems. + def run(self, bt: Bootstrapper) -> list[Phase]: + """RESOLVE phase: resolve versions and expand into Start items. Centralizes version resolution so all dependencies are expanded uniformly. In multiple_versions mode, filters out versions that @@ -73,7 +73,7 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: processing. Returns: - One StartItem per resolved version that needs building. + One Start item per resolved version that needs building. Empty list if all versions are already cached. """ assert self.bg_future is not None @@ -127,10 +127,10 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: # Build list so highest version ends up on top of the stack # (last element after extend) and is processed first. - items: list[PhaseItem] = [] + items: list[Phase] = [] for source_url, version in reversed(resolved_versions): items.append( - StartItem( + Start( WorkItem( req=self.work_item.req, req_type=self.work_item.req_type, diff --git a/src/fromager/bootstrapper/_start_item.py b/src/fromager/bootstrapper/_start.py similarity index 84% rename from src/fromager/bootstrapper/_start_item.py rename to src/fromager/bootstrapper/_start.py index 32561da9..1fca7ea6 100644 --- a/src/fromager/bootstrapper/_start_item.py +++ b/src/fromager/bootstrapper/_start.py @@ -4,8 +4,8 @@ import typing from ..requirements_file import RequirementType -from ._phase_item import PhaseItem -from ._prepare_source_item import PrepareSourceItem +from ._phase import Phase +from ._prepare_source import PrepareSource from ._types import BootstrapPhase if typing.TYPE_CHECKING: @@ -14,13 +14,13 @@ logger = logging.getLogger(__name__) -class StartItem(PhaseItem): +class Start(Phase): """START phase: add to graph, check if already seen.""" phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.START tracks_why: typing.ClassVar[bool] = False - def run(self, bt: Bootstrapper) -> list[PhaseItem]: + def run(self, bt: Bootstrapper) -> list[Phase]: """START phase: add to graph, check if already seen. _track_why is a no-op for this phase (tracks_why is False), @@ -29,7 +29,7 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: Returns: Empty list if already seen (nothing to do). - [PrepareSourceItem] if this is new work. + [PrepareSource] if this is new work. """ wi = self.work_item assert wi.resolved_version is not None @@ -60,9 +60,9 @@ def run(self, bt: Bootstrapper) -> list[PhaseItem]: f"new {wi.req_type} dependency {wi.req} resolves to {wi.resolved_version}" ) - # Must set pbi_pre_built before constructing PrepareSourceItem so that - # PrepareSourceItem.background_work() immediately sees the correct value. + # Must set pbi_pre_built before constructing PrepareSource so that + # PrepareSource.background_work() immediately sees the correct value. pbi = bt.ctx.package_build_info(wi.req) wi.pbi_pre_built = pbi.pre_built wi.exclusive_build = pbi.exclusive_build - return [PrepareSourceItem(wi)] + return [PrepareSource(wi)] diff --git a/src/fromager/bootstrapper/_work_item.py b/src/fromager/bootstrapper/_work_item.py index beb04973..30163394 100644 --- a/src/fromager/bootstrapper/_work_item.py +++ b/src/fromager/bootstrapper/_work_item.py @@ -20,7 +20,7 @@ class WorkItem: Carries identity fields set at creation time and accumulated state populated across phases as processing advances. The current phase is - encoded by the ``PhaseItem`` subclass wrapping this object. + encoded by the ``Phase`` subclass wrapping this object. Items enter at the RESOLVE phase with only req and req_type set. The RESOLVE phase populates source_url and resolved_version, then diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index b116eada..f18c6918 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -12,23 +12,23 @@ from resolvelib.resolvers import ResolverException from fromager import bootstrapper, log -from fromager.bootstrapper._build_item import BuildItem +from fromager.bootstrapper._build import Build from fromager.bootstrapper._cache import ( _bg_prepare_prebuilt, _download_wheel_from_cache, _find_cached_wheel, ) -from fromager.bootstrapper._phase_item import PhaseItem -from fromager.bootstrapper._prepare_source_item import ( - PrepareSourceItem, +from fromager.bootstrapper._phase import Phase +from fromager.bootstrapper._prepare_source import ( + PrepareSource, _bg_prepare_source, ) -from fromager.bootstrapper._process_install_deps_item import ( - ProcessInstallDepsItem, +from fromager.bootstrapper._process_install_deps import ( + ProcessInstallDeps, _get_install_dependencies, ) -from fromager.bootstrapper._resolve_item import ResolveItem -from fromager.bootstrapper._start_item import StartItem +from fromager.bootstrapper._resolve import Resolve +from fromager.bootstrapper._start import Start from fromager.bootstrapper._types import ( BootstrapPhase, SourceBuildResult, @@ -314,7 +314,7 @@ def test_get_install_dependencies_returns_list( def test_phase_build_produces_source_build_result(tmp_context: WorkContext) -> None: - """Verify BuildItem.run() produces a SourceBuildResult with correct values.""" + """Verify Build.run() produces a SourceBuildResult with correct values.""" bt = bootstrapper.Bootstrapper(tmp_context) mock_sdist_root = tmp_context.work_dir / "package-1.0.0" / "package-1.0.0" @@ -334,7 +334,7 @@ def test_phase_build_produces_source_build_result(tmp_context: WorkContext) -> N build_backend_deps=set(), build_sdist_deps=set(), ) - item = BuildItem(wi) + item = Build(wi) # Set up why stack so _track_why works bt.why = [] @@ -347,7 +347,7 @@ def test_phase_build_produces_source_build_result(tmp_context: WorkContext) -> N result_items = item.run(bt) assert len(result_items) == 1 - assert isinstance(result_items[0], ProcessInstallDepsItem) + assert isinstance(result_items[0], ProcessInstallDeps) result = result_items[0].work_item.build_result assert isinstance(result, SourceBuildResult) @@ -377,9 +377,9 @@ def test_multiple_versions_continues_on_error(tmp_context: WorkContext) -> None: build_phase_count = {"count": 0} def prepare_source_run( - self: PrepareSourceItem, + self: PrepareSource, bt_arg: bootstrapper.Bootstrapper, - ) -> list[PhaseItem]: + ) -> list[Phase]: build_phase_count["count"] += 1 if str(self.work_item.resolved_version) == "1.5": raise ValueError("Simulated failure for version 1.5") @@ -387,7 +387,7 @@ def prepare_source_run( req = Requirement("testpkg>=1.0") - with patch.object(PrepareSourceItem, "run", prepare_source_run): + with patch.object(PrepareSource, "run", prepare_source_run): with patch.object(bt, "has_been_seen", return_value=False): bt._bootstrap_one( req=req, @@ -613,8 +613,8 @@ def _make_resolve_item( req_type: RequirementType = RequirementType.TOP_LEVEL, why_snapshot: list[tuple[RequirementType, Requirement, Version]] | None = None, parent: tuple[Requirement, Version] | None = None, -) -> ResolveItem: - return ResolveItem( +) -> Resolve: + return Resolve( WorkItem( req=Requirement(req), req_type=req_type, @@ -625,7 +625,7 @@ def _make_resolve_item( def _record_and_load( - bt: bootstrapper.Bootstrapper, stack: list[PhaseItem] + bt: bootstrapper.Bootstrapper, stack: list[Phase] ) -> list[typing.Any]: bt._record_stack_state(stack) return typing.cast(list[typing.Any], json.loads(bt._stack_filename.read_text())) @@ -657,7 +657,7 @@ def test_record_stack_state_full_item(tmp_context: WorkContext) -> None: parent_version = Version("2.0") why_snapshot = [(RequirementType.INSTALL, parent_req, parent_version)] - item = BuildItem( + item = Build( WorkItem( req=Requirement("child-pkg>=1.0"), req_type=RequirementType.INSTALL, @@ -694,7 +694,7 @@ def test_record_stack_state_full_item(tmp_context: WorkContext) -> None: def test_record_stack_state_dep_sets_are_sorted(tmp_context: WorkContext) -> None: """Mixed-order dep sets come out alphabetically sorted.""" bt = bootstrapper.Bootstrapper(tmp_context) - item = BuildItem( + item = Build( WorkItem( req=Requirement("mypkg"), req_type=RequirementType.TOP_LEVEL, @@ -714,7 +714,7 @@ def test_record_stack_state_dep_sets_are_sorted(tmp_context: WorkContext) -> Non def test_record_stack_state_writes_file(tmp_context: WorkContext) -> None: """File is created; list length matches stack size.""" bt = bootstrapper.Bootstrapper(tmp_context) - stack: list[PhaseItem] = [ + stack: list[Phase] = [ _make_resolve_item("pkga"), _make_resolve_item("pkgb"), ] @@ -730,7 +730,7 @@ def test_record_stack_state_writes_file(tmp_context: WorkContext) -> None: def test_record_stack_state_ordering(tmp_context: WorkContext) -> None: """Index 0 = stack[-1] (next to pop); last index = stack[0].""" bt = bootstrapper.Bootstrapper(tmp_context) - stack: list[PhaseItem] = [ + stack: list[Phase] = [ _make_resolve_item("pkga"), _make_resolve_item("pkgb"), _make_resolve_item("pkgc"), @@ -765,7 +765,7 @@ def test_bootstrap_calls_record_stack_state(tmp_context: WorkContext) -> None: original = bt._record_stack_state - def counting_record(stack: list[PhaseItem]) -> None: + def counting_record(stack: list[Phase]) -> None: call_count["n"] += 1 original(stack) @@ -778,7 +778,7 @@ def counting_record(stack: list[PhaseItem]) -> None: "resolve", return_value=[("https://pypi.test/testpkg-1.0.tar.gz", Version("1.0"))], ), - patch.object(StartItem, "run", return_value=[]), + patch.object(Start, "run", return_value=[]), ): bt._bootstrap_one(req=req, req_type=RequirementType.TOP_LEVEL) @@ -788,7 +788,7 @@ def counting_record(stack: list[PhaseItem]) -> None: def test_bootstrap_with_empty_list(tmp_context: WorkContext) -> None: """bootstrap([]) completes without error and runs no phases.""" bt = bootstrapper.Bootstrapper(tmp_context) - with patch.object(ResolveItem, "run") as mock_run: + with patch.object(Resolve, "run") as mock_run: bt.bootstrap([]) mock_run.assert_not_called() @@ -797,11 +797,9 @@ def test_bootstrap_with_single_requirement(tmp_context: WorkContext) -> None: """bootstrap([req]) resolves and processes the requirement.""" bt = bootstrapper.Bootstrapper(tmp_context) req = Requirement("testpkg==1.0") - captured: list[ResolveItem] = [] + captured: list[Resolve] = [] - def capture_run( - self: ResolveItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + def capture_run(self: Resolve, bt_arg: bootstrapper.Bootstrapper) -> list[Phase]: captured.append(self) return [] @@ -811,13 +809,13 @@ def capture_run( "_resolve_and_add_top_level", return_value=("http://example.test/testpkg-1.0.tar.gz", Version("1.0")), ), - patch.object(ResolveItem, "run", capture_run), + patch.object(Resolve, "run", capture_run), patch.object(bt, "_record_stack_state"), ): bt.bootstrap([req]) assert len(captured) == 1 - assert isinstance(captured[0], ResolveItem) + assert isinstance(captured[0], Resolve) assert captured[0].work_item.req == req assert captured[0].work_item.req_type == RequirementType.TOP_LEVEL @@ -829,7 +827,7 @@ def test_bootstrap_skips_failed_resolution(tmp_context: WorkContext) -> None: with ( patch.object(bt, "_resolve_and_add_top_level", return_value=None), - patch.object(ResolveItem, "run") as mock_run, + patch.object(Resolve, "run") as mock_run, patch.object(bt, "_record_stack_state"), ): bt.bootstrap([req]) @@ -845,9 +843,7 @@ def test_bootstrap_two_requirements_both_processed(tmp_context: WorkContext) -> dispatch_calls: list = [] - def capture_run( - self: ResolveItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + def capture_run(self: Resolve, bt_arg: bootstrapper.Bootstrapper) -> list[Phase]: dispatch_calls.append(self.work_item.req.name) return [] @@ -857,7 +853,7 @@ def capture_run( "_resolve_and_add_top_level", return_value=("http://example.test/pkg-1.0.tar.gz", Version("1.0")), ), - patch.object(ResolveItem, "run", capture_run), + patch.object(Resolve, "run", capture_run), patch.object(bt, "_record_stack_state"), ): bt.bootstrap([req1, req2]) @@ -962,7 +958,7 @@ def test_bg_prepare_prebuilt_log_prefix_includes_version( def test_build_item_build_sdist_finds_existing(tmp_context: WorkContext) -> None: - """BuildItem._build_sdist returns cached sdist when finders.find_sdist hits.""" + """Build._build_sdist returns cached sdist when finders.find_sdist hits.""" sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" sdist_root.mkdir(parents=True, exist_ok=True) wi = WorkItem( @@ -973,7 +969,7 @@ def test_build_item_build_sdist_finds_existing(tmp_context: WorkContext) -> None sdist_root_dir=sdist_root, build_env=Mock(), ) - item = BuildItem(wi) + item = Build(wi) cached = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" cached.touch() @@ -986,7 +982,7 @@ def test_build_item_build_sdist_finds_existing(tmp_context: WorkContext) -> None def test_build_item_build_sdist_calls_build_when_not_cached( tmp_context: WorkContext, ) -> None: - """BuildItem._build_sdist calls sources.build_sdist when no cached sdist found.""" + """Build._build_sdist calls sources.build_sdist when no cached sdist found.""" sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" sdist_root.mkdir(parents=True, exist_ok=True) wi = WorkItem( @@ -997,7 +993,7 @@ def test_build_item_build_sdist_calls_build_when_not_cached( sdist_root_dir=sdist_root, build_env=Mock(), ) - item = BuildItem(wi) + item = Build(wi) built = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" with ( @@ -1017,7 +1013,7 @@ def test_build_item_build_sdist_calls_build_when_not_cached( def test_build_item_build_wheel(tmp_context: WorkContext) -> None: - """BuildItem._build_wheel builds sdist then wheel and updates mirror.""" + """Build._build_wheel builds sdist then wheel and updates mirror.""" sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" sdist_root.mkdir(parents=True, exist_ok=True) wi = WorkItem( @@ -1028,7 +1024,7 @@ def test_build_item_build_wheel(tmp_context: WorkContext) -> None: sdist_root_dir=sdist_root, build_env=Mock(), ) - item = BuildItem(wi) + item = Build(wi) built_wheel = tmp_context.wheels_build / "testpkg-1.0-py3-none-any.whl" built_sdist = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" @@ -1049,7 +1045,7 @@ def test_build_item_build_wheel(tmp_context: WorkContext) -> None: def test_build_item_do_build_returns_cached_wheel( tmp_context: WorkContext, ) -> None: - """BuildItem.do_build returns cached wheel immediately without building.""" + """Build.do_build returns cached wheel immediately without building.""" cached = tmp_context.wheels_downloads / "testpkg-1.0-py3-none-any.whl" cached.touch() wi = WorkItem( @@ -1062,7 +1058,7 @@ def test_build_item_do_build_returns_cached_wheel( sdist_root_dir=tmp_context.work_dir, build_env=Mock(), ) - item = BuildItem(wi) + item = Build(wi) with patch.object(item, "_build_wheel") as mock_wheel: wheel, sdist = item.do_build(tmp_context) @@ -1073,7 +1069,7 @@ def test_build_item_do_build_returns_cached_wheel( def test_build_item_do_build_sdist_only(tmp_context: WorkContext) -> None: - """BuildItem.do_build calls _build_sdist and returns (None, sdist) when build_sdist_only=True.""" + """Build.do_build calls _build_sdist and returns (None, sdist) when build_sdist_only=True.""" built_sdist = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" sdist_root.mkdir(parents=True, exist_ok=True) @@ -1087,7 +1083,7 @@ def test_build_item_do_build_sdist_only(tmp_context: WorkContext) -> None: sdist_root_dir=sdist_root, build_env=Mock(), ) - item = BuildItem(wi) + item = Build(wi) with ( patch.object(item, "_build_sdist", return_value=built_sdist) as mock_sdist, @@ -1102,7 +1098,7 @@ def test_build_item_do_build_sdist_only(tmp_context: WorkContext) -> None: def test_build_item_do_build_builds_wheel(tmp_context: WorkContext) -> None: - """BuildItem.do_build calls _build_wheel when no cache and not sdist_only.""" + """Build.do_build calls _build_wheel when no cache and not sdist_only.""" built_wheel = tmp_context.wheels_downloads / "testpkg-1.0-py3-none-any.whl" built_sdist = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" @@ -1117,7 +1113,7 @@ def test_build_item_do_build_builds_wheel(tmp_context: WorkContext) -> None: sdist_root_dir=sdist_root, build_env=Mock(), ) - item = BuildItem(wi) + item = Build(wi) with patch.object( item, "_build_wheel", return_value=(built_wheel, built_sdist) diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index 0d74ccf8..6761bcfa 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -1,17 +1,17 @@ """Tests for the iterative bootstrap implementation. Tests cover: -- PhaseItem class hierarchy: class variables and base class behavior +- Phase class hierarchy: class variables and base class behavior - WorkItem dataclass defaults and state accumulation - _track_why context manager behavior - create_unresolved_work_items helper -- ResolveItem.run() version expansion -- StartItem.run() graph addition and seen-check -- PrepareSourceItem.run() all branches (prebuilt, source, cached, bad path) -- PrepareBuildItem.run() dep installation and extraction -- BuildItem.run() conditional install and result construction -- ProcessInstallDepsItem.run() hooks, dep extraction, error modes -- CompleteItem.run() cleanup +- Resolve.run() version expansion +- Start.run() graph addition and seen-check +- PrepareSource.run() all branches (prebuilt, source, cached, bad path) +- PrepareBuild.run() dep installation and extraction +- Build.run() conditional install and result construction +- ProcessInstallDeps.run() hooks, dep extraction, error modes +- Complete.run() cleanup - _handle_phase_error for all three error modes - End-to-end iterative loop with LIFO ordering """ @@ -29,14 +29,14 @@ from packaging.version import Version from fromager import bootstrapper, build_environment -from fromager.bootstrapper._build_item import BuildItem -from fromager.bootstrapper._complete_item import CompleteItem -from fromager.bootstrapper._phase_item import PhaseItem -from fromager.bootstrapper._prepare_build_item import PrepareBuildItem -from fromager.bootstrapper._prepare_source_item import PrepareSourceItem -from fromager.bootstrapper._process_install_deps_item import ProcessInstallDepsItem -from fromager.bootstrapper._resolve_item import ResolveItem -from fromager.bootstrapper._start_item import StartItem +from fromager.bootstrapper._build import Build +from fromager.bootstrapper._complete import Complete +from fromager.bootstrapper._phase import Phase +from fromager.bootstrapper._prepare_build import PrepareBuild +from fromager.bootstrapper._prepare_source import PrepareSource +from fromager.bootstrapper._process_install_deps import ProcessInstallDeps +from fromager.bootstrapper._resolve import Resolve +from fromager.bootstrapper._start import Start from fromager.bootstrapper._types import ( BootstrapPhase, PreparedSourceData, @@ -101,8 +101,8 @@ def _make_resolve_item( req_type: RequirementType = RequirementType.INSTALL, why_snapshot: list | None = None, parent: tuple | None = None, -) -> ResolveItem: - return ResolveItem( +) -> Resolve: + return Resolve( WorkItem( req=Requirement(req), req_type=req_type, @@ -119,8 +119,8 @@ def _make_start_item( version: str = "1.0", why_snapshot: list | None = None, parent: tuple | None = None, -) -> StartItem: - return StartItem( +) -> Start: + return Start( WorkItem( req=Requirement(req), req_type=req_type, @@ -132,14 +132,14 @@ def _make_start_item( ) -_PHASE_TO_CLASS: dict[BootstrapPhase, type[PhaseItem]] = { - BootstrapPhase.RESOLVE: ResolveItem, - BootstrapPhase.START: StartItem, - BootstrapPhase.PREPARE_SOURCE: PrepareSourceItem, - BootstrapPhase.PREPARE_BUILD: PrepareBuildItem, - BootstrapPhase.BUILD: BuildItem, - BootstrapPhase.PROCESS_INSTALL_DEPS: ProcessInstallDepsItem, - BootstrapPhase.COMPLETE: CompleteItem, +_PHASE_TO_CLASS: dict[BootstrapPhase, type[Phase]] = { + BootstrapPhase.RESOLVE: Resolve, + BootstrapPhase.START: Start, + BootstrapPhase.PREPARE_SOURCE: PrepareSource, + BootstrapPhase.PREPARE_BUILD: PrepareBuild, + BootstrapPhase.BUILD: Build, + BootstrapPhase.PROCESS_INSTALL_DEPS: ProcessInstallDeps, + BootstrapPhase.COMPLETE: Complete, } @@ -158,7 +158,7 @@ def _make_build_item( pbi_pre_built: bool = False, cached_wheel_filename: pathlib.Path | None = None, build_sdist_only: bool = False, -) -> PhaseItem: +) -> Phase: wi = WorkItem( req=Requirement(req), req_type=RequirementType.INSTALL, @@ -181,25 +181,25 @@ def _make_build_item( return _PHASE_TO_CLASS[phase](wi) -class TestPhaseItemClassVariables: - """Verify class-variable declarations on each PhaseItem subclass.""" +class TestPhaseClassVariables: + """Verify class-variable declarations on each Phase subclass.""" @pytest.mark.parametrize( "cls, expected_phase, expected_tracks_why", [ - (ResolveItem, BootstrapPhase.RESOLVE, False), - (StartItem, BootstrapPhase.START, False), - (PrepareSourceItem, BootstrapPhase.PREPARE_SOURCE, True), - (PrepareBuildItem, BootstrapPhase.PREPARE_BUILD, True), - (BuildItem, BootstrapPhase.BUILD, True), - (ProcessInstallDepsItem, BootstrapPhase.PROCESS_INSTALL_DEPS, True), - (CompleteItem, BootstrapPhase.COMPLETE, True), + (Resolve, BootstrapPhase.RESOLVE, False), + (Start, BootstrapPhase.START, False), + (PrepareSource, BootstrapPhase.PREPARE_SOURCE, True), + (PrepareBuild, BootstrapPhase.PREPARE_BUILD, True), + (Build, BootstrapPhase.BUILD, True), + (ProcessInstallDeps, BootstrapPhase.PROCESS_INSTALL_DEPS, True), + (Complete, BootstrapPhase.COMPLETE, True), ], ) def test_class_variables( self, tmp_context: WorkContext, - cls: type[PhaseItem], + cls: type[Phase], expected_phase: BootstrapPhase, expected_tracks_why: bool, ) -> None: @@ -208,14 +208,14 @@ def test_class_variables( def test_str_default(self) -> None: wi = _make_work_item(req="mypkg") - item = BuildItem(wi) - assert str(item) == "BuildItem(mypkg)" + item = Build(wi) + assert str(item) == "Build(mypkg)" def test_background_work_returns_none_by_default( self, tmp_context: WorkContext ) -> None: bt = bootstrapper.Bootstrapper(tmp_context) - for cls in (PrepareBuildItem, BuildItem, ProcessInstallDepsItem, CompleteItem): + for cls in (PrepareBuild, Build, ProcessInstallDeps, Complete): wi = _make_work_item(req="testpkg", version="1.0") item = cls(wi) assert item.background_work(bt) is None, ( @@ -307,7 +307,7 @@ def test_creates_resolve_phase_items(self, tmp_context: WorkContext) -> None: assert len(items) == 2 for item in items: - assert isinstance(item, ResolveItem) + assert isinstance(item, Resolve) assert item.phase == BootstrapPhase.RESOLVE assert item.work_item.req_type == RequirementType.BUILD_SYSTEM assert item.work_item.parent == (Requirement("parent"), Version("1.0")) @@ -363,7 +363,7 @@ def test_single_version(self, tmp_context: WorkContext) -> None: result = item.run(bt) assert len(result) == 1 - assert isinstance(result[0], StartItem) + assert isinstance(result[0], Start) assert result[0].work_item.source_url == "https://pypi.org/testpkg-1.0.tar.gz" assert result[0].work_item.resolved_version == Version("1.0") assert result[0].work_item.parent == wi.parent @@ -406,7 +406,7 @@ def test_preserves_why_snapshot(self, tmp_context: WorkContext) -> None: def test_filters_cached_versions_in_multiple_versions_mode( self, tmp_context: WorkContext ) -> None: - """Cached versions are filtered out before creating StartItems.""" + """Cached versions are filtered out before creating Starts.""" bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) item = _make_resolve_item() item.bg_future = _make_resolved_future( @@ -483,7 +483,7 @@ def test_empty_resolution_raises_runtime_error( def test_filters_failed_versions_in_multiple_versions_mode( self, tmp_context: WorkContext ) -> None: - """Previously failed versions are excluded before creating StartItems.""" + """Previously failed versions are excluded before creating Starts.""" bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) item = _make_resolve_item() @@ -563,7 +563,7 @@ def test_new_item_advances_to_prepare_source( result = item.run(bt) assert len(result) == 1 - assert isinstance(result[0], PrepareSourceItem) + assert isinstance(result[0], PrepareSource) assert result[0].work_item is item.work_item def test_already_seen_returns_empty(self, tmp_context: WorkContext) -> None: @@ -636,7 +636,7 @@ def test_marks_as_seen(self, tmp_context: WorkContext) -> None: def test_sets_pbi_pre_built_before_prepare_source( self, tmp_context: WorkContext ) -> None: - """pbi_pre_built is set on work_item before PrepareSourceItem is constructed.""" + """pbi_pre_built is set on work_item before PrepareSource is constructed.""" bt = bootstrapper.Bootstrapper(tmp_context) bt.why = [] item = _make_start_item() @@ -649,11 +649,11 @@ def test_sets_pbi_pre_built_before_prepare_source( result = item.run(bt) assert len(result) == 1 - assert isinstance(result[0], PrepareSourceItem) + assert isinstance(result[0], PrepareSource) assert result[0].work_item.pbi_pre_built is True -class TestCompleteItem: +class TestComplete: def test_calls_clean_build_dirs(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) mock_sdist_root = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" @@ -756,7 +756,7 @@ def test_build_phase_test_mode_fallback_success( result = bt._handle_phase_error(item, err) assert len(result) == 1 - assert isinstance(result[0], ProcessInstallDepsItem) + assert isinstance(result[0], ProcessInstallDeps) assert result[0].work_item is item.work_item assert result[0].work_item.build_result is mock_fallback assert len(bt.failed_packages) == 0 @@ -873,39 +873,35 @@ def test_full_lifecycle_source_package(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) phases_visited: list[BootstrapPhase] = [] - original_resolve_run = ResolveItem.run - original_start_run = StartItem.run + original_resolve_run = Resolve.run + original_start_run = Start.run def resolve_run( - self: ResolveItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + self: Resolve, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: phases_visited.append(type(self).phase) return original_resolve_run(self, bt_arg) - def start_run( - self: StartItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + def start_run(self: Start, bt_arg: bootstrapper.Bootstrapper) -> list[Phase]: phases_visited.append(type(self).phase) return original_start_run(self, bt_arg) def prepare_source_run( - self: PrepareSourceItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + self: PrepareSource, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: phases_visited.append(type(self).phase) wi = self.work_item wi.build_env = Mock() wi.sdist_root_dir = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" - return [PrepareBuildItem(wi)] + return [PrepareBuild(wi)] def prepare_build_run( - self: PrepareBuildItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + self: PrepareBuild, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: phases_visited.append(type(self).phase) - return [BuildItem(self.work_item)] + return [Build(self.work_item)] - def build_run( - self: BuildItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + def build_run(self: Build, bt_arg: bootstrapper.Bootstrapper) -> list[Phase]: phases_visited.append(type(self).phase) wi = self.work_item wi.build_result = SourceBuildResult( @@ -916,28 +912,28 @@ def build_run( build_env=None, source_type=SourceType.SDIST, ) - return [ProcessInstallDepsItem(wi)] + return [ProcessInstallDeps(wi)] def process_install_deps_run( - self: ProcessInstallDepsItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + self: ProcessInstallDeps, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: phases_visited.append(type(self).phase) - return [CompleteItem(self.work_item)] + return [Complete(self.work_item)] def complete_run( - self: CompleteItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + self: Complete, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: phases_visited.append(type(self).phase) return [] with ( - patch.object(ResolveItem, "run", resolve_run), - patch.object(StartItem, "run", start_run), - patch.object(PrepareSourceItem, "run", prepare_source_run), - patch.object(PrepareBuildItem, "run", prepare_build_run), - patch.object(BuildItem, "run", build_run), - patch.object(ProcessInstallDepsItem, "run", process_install_deps_run), - patch.object(CompleteItem, "run", complete_run), + patch.object(Resolve, "run", resolve_run), + patch.object(Start, "run", start_run), + patch.object(PrepareSource, "run", prepare_source_run), + patch.object(PrepareBuild, "run", prepare_build_run), + patch.object(Build, "run", build_run), + patch.object(ProcessInstallDeps, "run", process_install_deps_run), + patch.object(Complete, "run", complete_run), patch.object( bt._resolver, "resolve", @@ -963,35 +959,33 @@ def test_lifo_ordering_deps_before_continuation( bt = bootstrapper.Bootstrapper(tmp_context) processing_order: list[tuple[str, str]] = [] - original_resolve_run = ResolveItem.run - original_start_run = StartItem.run + original_resolve_run = Resolve.run + original_start_run = Start.run def resolve_run( - self: ResolveItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + self: Resolve, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: processing_order.append( (str(self.work_item.req.name), str(type(self).phase)) ) return original_resolve_run(self, bt_arg) - def start_run( - self: StartItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + def start_run(self: Start, bt_arg: bootstrapper.Bootstrapper) -> list[Phase]: processing_order.append( (str(self.work_item.req.name), str(type(self).phase)) ) return original_start_run(self, bt_arg) def prepare_source_run( - self: PrepareSourceItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + self: PrepareSource, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: phase_name = str(type(self).phase) req_name = str(self.work_item.req.name) processing_order.append((req_name, phase_name)) wi = self.work_item if req_name == "parent": assert wi.resolved_version is not None - dep_item = ResolveItem( + dep_item = Resolve( WorkItem( req=Requirement("child"), req_type=RequirementType.BUILD_SYSTEM, @@ -999,12 +993,12 @@ def prepare_source_run( parent=(wi.req, wi.resolved_version), ) ) - return [CompleteItem(wi), dep_item] + return [Complete(wi), dep_item] return [] def complete_run( - self: CompleteItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + self: Complete, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: processing_order.append( (str(self.work_item.req.name), str(type(self).phase)) ) @@ -1022,10 +1016,10 @@ def complete_run( ) with ( - patch.object(ResolveItem, "run", resolve_run), - patch.object(StartItem, "run", start_run), - patch.object(PrepareSourceItem, "run", prepare_source_run), - patch.object(CompleteItem, "run", complete_run), + patch.object(Resolve, "run", resolve_run), + patch.object(Start, "run", start_run), + patch.object(PrepareSource, "run", prepare_source_run), + patch.object(Complete, "run", complete_run), patch.object( bt._resolver, "resolve", @@ -1057,14 +1051,14 @@ def test_multiple_versions_error_isolation(self, tmp_context: WorkContext) -> No bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) def prepare_source_run( - self: PrepareSourceItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + self: PrepareSource, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: if str(self.work_item.resolved_version) == "1.5": raise ValueError("1.5 broken") return [] with ( - patch.object(PrepareSourceItem, "run", prepare_source_run), + patch.object(PrepareSource, "run", prepare_source_run), patch.object( bt._resolver, "resolve", @@ -1090,25 +1084,25 @@ def test_multiple_versions_resolve_failure_continues( """RESOLVE failure for one dependency does not crash the loop.""" bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) - original_resolve_run = ResolveItem.run + original_resolve_run = Resolve.run completed: list[str] = [] def resolve_run( - self: ResolveItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + self: Resolve, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: if str(self.work_item.req.name) == "bad-dep": raise RuntimeError("Could not resolve any versions for bad-dep") return original_resolve_run(self, bt_arg) def prepare_source_run( - self: PrepareSourceItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + self: PrepareSource, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: completed.append(str(self.work_item.req.name)) return [] with ( - patch.object(ResolveItem, "run", resolve_run), - patch.object(PrepareSourceItem, "run", prepare_source_run), + patch.object(Resolve, "run", resolve_run), + patch.object(PrepareSource, "run", prepare_source_run), patch.object( bt._resolver, "resolve", @@ -1132,15 +1126,15 @@ def test_test_mode_continues_after_failure(self, tmp_context: WorkContext) -> No items_completed: list[str] = [] def prepare_source_run( - self: PrepareSourceItem, bt_arg: bootstrapper.Bootstrapper - ) -> list[PhaseItem]: + self: PrepareSource, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: if str(self.work_item.req.name) == "fail-pkg": raise RuntimeError("build error") items_completed.append(str(self.work_item.req.name)) return [] with ( - patch.object(PrepareSourceItem, "run", prepare_source_run), + patch.object(PrepareSource, "run", prepare_source_run), patch.object( bt._resolver, "resolve", @@ -1158,7 +1152,7 @@ def prepare_source_run( class TestPhasePrepareSource: - """Tests for PrepareSourceItem.run(): prebuilt, source, cache, and error paths.""" + """Tests for PrepareSource.run(): prebuilt, source, cache, and error paths.""" def test_prebuilt_uses_background_result(self, tmp_context: WorkContext) -> None: """Prebuilt package uses bg_future result and advances to PROCESS_INSTALL_DEPS.""" @@ -1179,7 +1173,7 @@ def test_prebuilt_uses_background_result(self, tmp_context: WorkContext) -> None result = item.run(bt) assert len(result) == 1 - assert isinstance(result[0], ProcessInstallDepsItem) + assert isinstance(result[0], ProcessInstallDeps) assert result[0].work_item is item.work_item wi = result[0].work_item assert wi.build_result is not None @@ -1219,7 +1213,7 @@ def test_source_no_cache_uses_background_result( result = item.run(bt) wi = item.work_item - assert isinstance(result[0], PrepareBuildItem) + assert isinstance(result[0], PrepareBuild) assert result[0].work_item is wi assert wi.build_env is mock_env assert wi.sdist_root_dir == sdist_root @@ -1272,7 +1266,7 @@ def test_source_cached_wheel_uses_background_result( wi = item.work_item assert wi.cached_wheel_filename == cached_wheel assert wi.sdist_root_dir == unpacked / unpacked.stem - assert isinstance(result[0], PrepareBuildItem) + assert isinstance(result[0], PrepareBuild) assert len(result) == 1 def test_bad_sdist_root_raises_valueerror(self, tmp_context: WorkContext) -> None: @@ -1320,12 +1314,12 @@ def test_constraint_logged_when_present( ): result = item.run(bt) - assert isinstance(result[0], PrepareBuildItem) + assert isinstance(result[0], PrepareBuild) assert "matches constraint" in caplog.text class TestPhasePrepareBuild: - """Tests for PrepareBuildItem.run(): dep installation and extraction.""" + """Tests for PrepareBuild.run(): dep installation and extraction.""" def test_installs_system_deps_and_returns_backend_sdist_items( self, tmp_context: WorkContext @@ -1363,7 +1357,7 @@ def test_installs_system_deps_and_returns_backend_sdist_items( ): result = item.run(bt) - assert isinstance(result[0], BuildItem) + assert isinstance(result[0], Build) assert result[0].work_item is wi assert wi.build_backend_deps == {Requirement("wheel")} assert wi.build_sdist_deps == {Requirement("flit-core")} @@ -1401,7 +1395,7 @@ def test_no_extra_deps_returns_item_only(self, tmp_context: WorkContext) -> None result = item.run(bt) assert len(result) == 1 - assert isinstance(result[0], BuildItem) + assert isinstance(result[0], Build) mock_env.install.assert_called_once_with(system_deps) def test_install_called_once_with_system_deps( @@ -1486,9 +1480,9 @@ def test_creates_items_with_correct_requirement_types( class TestPhaseBuild: - """Tests for BuildItem.run(): conditional dep install and result construction.""" + """Tests for Build.run(): conditional dep install and result construction.""" - def _make_build_phase_item(self, tmp_context: WorkContext) -> BuildItem: + def _make_build_phase_item(self, tmp_context: WorkContext) -> Build: mock_env = Mock() sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" sdist_root.parent.mkdir(parents=True, exist_ok=True) @@ -1500,7 +1494,7 @@ def _make_build_phase_item(self, tmp_context: WorkContext) -> BuildItem: build_backend_deps={Requirement("wheel")}, build_sdist_deps={Requirement("flit-core")}, ) - assert isinstance(item, BuildItem) + assert isinstance(item, Build) return item def test_disjoint_deps_installs_remaining(self, tmp_context: WorkContext) -> None: @@ -1529,7 +1523,7 @@ def test_disjoint_deps_installs_remaining(self, tmp_context: WorkContext) -> Non {Requirement("wheel"), Requirement("flit-core")} ) assert len(result) == 1 - assert isinstance(result[0], ProcessInstallDepsItem) + assert isinstance(result[0], ProcessInstallDeps) def test_overlapping_deps_skips_install(self, tmp_context: WorkContext) -> None: """Overlapping backend/sdist deps with system deps skips install.""" @@ -1637,7 +1631,7 @@ def test_build_result_uses_source_type_from_sources( def test_returns_single_process_install_deps_item( self, tmp_context: WorkContext ) -> None: - """BuildItem.run() returns exactly [ProcessInstallDepsItem].""" + """Build.run() returns exactly [ProcessInstallDeps].""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_build_phase_item(tmp_context) @@ -1648,13 +1642,13 @@ def test_returns_single_process_install_deps_item( result = item.run(bt) assert len(result) == 1 - assert isinstance(result[0], ProcessInstallDepsItem) + assert isinstance(result[0], ProcessInstallDeps) assert result[0].work_item is item.work_item def test_phase_advancement_preserves_work_item_identity( self, tmp_context: WorkContext ) -> None: - """When BuildItem.run() returns ProcessInstallDepsItem, work_item is same obj.""" + """When Build.run() returns ProcessInstallDeps, work_item is same obj.""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_build_phase_item(tmp_context) wi = item.work_item @@ -1669,9 +1663,9 @@ def test_phase_advancement_preserves_work_item_identity( class TestPhaseProcessInstallDeps: - """Tests for ProcessInstallDepsItem.run(): hooks, dep extraction, error modes.""" + """Tests for ProcessInstallDeps.run(): hooks, dep extraction, error modes.""" - def _make_process_item(self, tmp_context: WorkContext) -> ProcessInstallDepsItem: + def _make_process_item(self, tmp_context: WorkContext) -> ProcessInstallDeps: build_result = SourceBuildResult( wheel_filename=tmp_context.work_dir / "testpkg-1.0-py3-none-any.whl", sdist_filename=tmp_context.work_dir / "testpkg-1.0.tar.gz", @@ -1685,13 +1679,13 @@ def _make_process_item(self, tmp_context: WorkContext) -> ProcessInstallDepsItem build_result=build_result, source_url="https://pkg.test/testpkg-1.0.tar.gz", ) - assert isinstance(item, ProcessInstallDepsItem) + assert isinstance(item, ProcessInstallDeps) return item def test_normal_path_returns_item_and_dep_items( self, tmp_context: WorkContext ) -> None: - """Normal path: hooks, deps, build order, returns [CompleteItem, *dep_items].""" + """Normal path: hooks, deps, build order, returns [Complete, *dep_items].""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_process_item(tmp_context) wi = item.work_item @@ -1700,7 +1694,7 @@ def test_normal_path_returns_item_and_dep_items( with ( patch("fromager.hooks.run_post_bootstrap_hooks"), patch( - "fromager.bootstrapper._process_install_deps_item._get_install_dependencies", + "fromager.bootstrapper._process_install_deps._get_install_dependencies", return_value=[Requirement("dep-a")], ), patch.object( @@ -1716,7 +1710,7 @@ def test_normal_path_returns_item_and_dep_items( ): result = item.run(bt) - assert isinstance(result[0], CompleteItem) + assert isinstance(result[0], Complete) assert result[0].work_item is wi assert result[1] is dep_item mock_build_order.assert_called_once() @@ -1740,7 +1734,7 @@ def test_hook_error_test_mode_records_and_continues( side_effect=RuntimeError("hook failed"), ), patch( - "fromager.bootstrapper._process_install_deps_item._get_install_dependencies", + "fromager.bootstrapper._process_install_deps._get_install_dependencies", return_value=[], ) as mock_get_deps, patch.object( @@ -1754,7 +1748,7 @@ def test_hook_error_test_mode_records_and_continues( ): result = item.run(bt) - assert isinstance(result[0], CompleteItem) + assert isinstance(result[0], Complete) assert len(bt.failed_packages) == 1 assert bt.failed_packages[0]["failure_type"] == "hook" mock_get_deps.assert_called_once() @@ -1785,7 +1779,7 @@ def test_dep_extraction_error_test_mode_uses_empty_deps( with ( patch("fromager.hooks.run_post_bootstrap_hooks"), patch( - "fromager.bootstrapper._process_install_deps_item._get_install_dependencies", + "fromager.bootstrapper._process_install_deps._get_install_dependencies", side_effect=RuntimeError("dep failed"), ), patch.object( @@ -1801,7 +1795,7 @@ def test_dep_extraction_error_test_mode_uses_empty_deps( ): result = item.run(bt) - assert isinstance(result[0], CompleteItem) + assert isinstance(result[0], Complete) assert len(bt.failed_packages) == 1 assert bt.failed_packages[0]["failure_type"] == "dependency_extraction" mock_build_order.assert_called_once() @@ -1822,7 +1816,7 @@ def test_dep_extraction_error_normal_mode_raises( with ( patch("fromager.hooks.run_post_bootstrap_hooks"), patch( - "fromager.bootstrapper._process_install_deps_item._get_install_dependencies", + "fromager.bootstrapper._process_install_deps._get_install_dependencies", side_effect=RuntimeError("dep failed"), ), ): @@ -1830,14 +1824,14 @@ def test_dep_extraction_error_normal_mode_raises( item.run(bt) def test_no_install_deps_returns_item_only(self, tmp_context: WorkContext) -> None: - """When no install deps, returns [CompleteItem].""" + """When no install deps, returns [Complete].""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_process_item(tmp_context) with ( patch("fromager.hooks.run_post_bootstrap_hooks"), patch( - "fromager.bootstrapper._process_install_deps_item._get_install_dependencies", + "fromager.bootstrapper._process_install_deps._get_install_dependencies", return_value=[], ), patch.object( @@ -1852,7 +1846,7 @@ def test_no_install_deps_returns_item_only(self, tmp_context: WorkContext) -> No result = item.run(bt) assert len(result) == 1 - assert isinstance(result[0], CompleteItem) + assert isinstance(result[0], Complete) def test_build_order_called_with_correct_args( self, tmp_context: WorkContext @@ -1866,7 +1860,7 @@ def test_build_order_called_with_correct_args( with ( patch("fromager.hooks.run_post_bootstrap_hooks"), patch( - "fromager.bootstrapper._process_install_deps_item._get_install_dependencies", + "fromager.bootstrapper._process_install_deps._get_install_dependencies", return_value=[], ), patch.object( @@ -2176,7 +2170,7 @@ def test_returns_empty_when_parent_not_in_graph( class TestPhasePrepareBuildFiltering: - """Tests for PrepareBuildItem.run() with build-system satisfaction filtering.""" + """Tests for PrepareBuild.run() with build-system satisfaction filtering.""" def test_satisfied_backend_dep_skips_resolve( self, tmp_context: WorkContext @@ -2229,7 +2223,7 @@ def test_satisfied_backend_dep_skips_resolve( ): result = item.run(bt) - assert isinstance(result[0], BuildItem) + assert isinstance(result[0], Build) calls = mock_create.call_args_list assert calls[0][0][0] == set() assert calls[1][0][0] == set() @@ -2338,12 +2332,12 @@ def test_unsatisfied_backend_dep_creates_resolve_item( class TestBackgroundWork: - """Tests for background_work() dispatch on PhaseItem subclasses.""" + """Tests for background_work() dispatch on Phase subclasses.""" def test_resolve_item_background_work_returns_callable( self, tmp_context: WorkContext ) -> None: - """ResolveItem.background_work(bt) returns a non-None callable.""" + """Resolve.background_work(bt) returns a non-None callable.""" bt = bootstrapper.Bootstrapper(tmp_context) item = _make_resolve_item() bg = item.background_work(bt) @@ -2361,7 +2355,7 @@ def test_prepare_source_item_background_work_source( source_url="https://pypi.test/testpkg-1.0.tar.gz", ) wi.pbi_pre_built = False - item = PrepareSourceItem(wi) + item = PrepareSource(wi) bg = item.background_work(bt) assert bg is not None assert callable(bg) @@ -2377,29 +2371,29 @@ def test_prepare_source_item_background_work_prebuilt( source_url="https://pypi.test/testpkg-1.0-py3-none-any.whl", ) wi.pbi_pre_built = True - item = PrepareSourceItem(wi) + item = PrepareSource(wi) bg = item.background_work(bt) assert bg is not None assert callable(bg) class TestAsJson: - """Tests for PhaseItem.as_json() serialization.""" + """Tests for Phase.as_json() serialization.""" @pytest.mark.parametrize( "cls, expected_phase", [ - (ResolveItem, BootstrapPhase.RESOLVE), - (StartItem, BootstrapPhase.START), - (PrepareSourceItem, BootstrapPhase.PREPARE_SOURCE), - (PrepareBuildItem, BootstrapPhase.PREPARE_BUILD), - (BuildItem, BootstrapPhase.BUILD), - (ProcessInstallDepsItem, BootstrapPhase.PROCESS_INSTALL_DEPS), - (CompleteItem, BootstrapPhase.COMPLETE), + (Resolve, BootstrapPhase.RESOLVE), + (Start, BootstrapPhase.START), + (PrepareSource, BootstrapPhase.PREPARE_SOURCE), + (PrepareBuild, BootstrapPhase.PREPARE_BUILD), + (Build, BootstrapPhase.BUILD), + (ProcessInstallDeps, BootstrapPhase.PROCESS_INSTALL_DEPS), + (Complete, BootstrapPhase.COMPLETE), ], ) def test_phase_field_per_subclass( - self, cls: type[PhaseItem], expected_phase: BootstrapPhase + self, cls: type[Phase], expected_phase: BootstrapPhase ) -> None: wi = _make_work_item(req="testpkg") item = cls(wi) @@ -2419,11 +2413,11 @@ def test_push_items_sets_bg_future_when_pool_exists( with bootstrapper.Bootstrapper(tmp_context, num_bg_threads=1) as bt: with patch.object(bt._resolver, "resolve", return_value=expected): item = _make_resolve_item(req="testpkg") - stack: list[PhaseItem] = [] + stack: list[Phase] = [] bt._push_items(stack, [item]) - # bg_future must be set on the PhaseItem, not None + # bg_future must be set on the Phase, not None assert item.bg_future is not None assert isinstance(item.bg_future, concurrent.futures.Future) @@ -2431,11 +2425,11 @@ def test_push_items_sets_bg_future_when_pool_exists( result = item.bg_future.result(timeout=10) assert result == expected - # ResolveItem.run() processes the future correctly + # Resolve.run() processes the future correctly new_items = item.run(bt) assert len(new_items) == 1 - assert isinstance(new_items[0], StartItem) + assert isinstance(new_items[0], Start) assert new_items[0].work_item.resolved_version == Version("1.0") def test_push_items_no_bg_future_when_pool_is_none( @@ -2445,7 +2439,7 @@ def test_push_items_no_bg_future_when_pool_is_none( bt = bootstrapper.Bootstrapper(tmp_context) bt._bg_pool = None item = _make_resolve_item(req="testpkg") - stack: list[PhaseItem] = [] + stack: list[Phase] = [] bt._push_items(stack, [item]) @@ -2453,7 +2447,7 @@ def test_push_items_no_bg_future_when_pool_is_none( class TestCreateUnresolvedWorkItemsReturnType: - """create_unresolved_work_items returns ResolveItem instances.""" + """create_unresolved_work_items returns Resolve instances.""" def test_returns_resolve_items(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) @@ -2462,6 +2456,6 @@ def test_returns_resolve_items(self, tmp_context: WorkContext) -> None: deps, RequirementType.INSTALL, Requirement("parent"), Version("1.0") ) assert len(items) == 1 - assert isinstance(items[0], ResolveItem) + assert isinstance(items[0], Resolve) assert items[0].work_item.req == Requirement("dep-a") assert items[0].work_item.req_type == RequirementType.INSTALL From b29a0fb9aea8c639f4d1fa837f486788a34f3888 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 6 Jul 2026 14:54:25 -0400 Subject: [PATCH 16/19] refactor(bootstrapper): add docstrings, enforce phase ClassVar, fix log context - Add docstrings to `add_to_graph`, `has_been_seen`, and `add_to_build_order` in `Bootstrapper` - Move the "exclusive build, draining background pool" log and `_drain_background_pool()` call inside `req_ctxvar_context` so the message carries the per-requirement log prefix - Use the `_create_unpack_dir` helper in `_bg_prepare_prebuilt` instead of duplicating the inline path-and-mkdir logic - Add `Phase.__init_subclass__` to raise `TypeError` at class-definition time when a concrete subclass omits the required `phase` ClassVar; add two tests - Expand class docstrings on all seven `Phase` subclasses to describe each phase's role and its next-phase transitions Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper/_bootstrapper.py | 56 ++++++++++++++++--- src/fromager/bootstrapper/_build.py | 18 +++++- src/fromager/bootstrapper/_cache.py | 3 +- src/fromager/bootstrapper/_complete.py | 9 ++- src/fromager/bootstrapper/_phase.py | 17 ++++++ src/fromager/bootstrapper/_prepare_build.py | 12 +++- src/fromager/bootstrapper/_prepare_source.py | 13 ++++- .../bootstrapper/_process_install_deps.py | 10 +++- src/fromager/bootstrapper/_resolve.py | 11 +++- src/fromager/bootstrapper/_start.py | 11 +++- tests/test_bootstrapper.py | 21 +++++++ 11 files changed, 165 insertions(+), 16 deletions(-) diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py index 4d65f3f5..3a9d34cf 100644 --- a/src/fromager/bootstrapper/_bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -306,17 +306,16 @@ def _run_bootstrap_loop(self, stack: list[Phase]) -> None: item = stack.pop() self.why = list(item.work_item.why_snapshot) - if item.requires_exclusive_run: - logger.info( - "%s requires exclusive build, draining background pool", - item.work_item.req, - ) - self._drain_background_pool() - with ( req_ctxvar_context(item.work_item.req, item.work_item.resolved_version), self._track_why(item), ): + if item.requires_exclusive_run: + logger.info( + "%s requires exclusive build, draining background pool", + item.work_item.req, + ) + self._drain_background_pool() try: new_items = item.run(self) except Exception as err: @@ -819,6 +818,22 @@ def add_to_graph( download_url: str, parent: tuple[Requirement, Version] | None, ) -> None: + """Add a resolved requirement as a node/edge in the dependency graph. + + Records the dependency relationship between ``parent`` and ``req`` in + the in-memory graph, then writes the updated graph to disk. Must be + called after resolution but before the seen-check so that every edge + is captured even for packages that are not built again. + + Args: + req: The requirement being added. + req_type: Whether this is a build-system, build-backend, install, + or other dependency kind. + req_version: The resolved version of ``req``. + download_url: Source URL used to obtain the distribution. + parent: ``(req, version)`` of the package that introduced ``req``, + or ``None`` for top-level requirements. + """ parent_req, parent_version = parent if parent else (None, None) pbi = self.ctx.package_build_info(req) # Update the dependency graph after we determine that this requirement is @@ -877,6 +892,18 @@ def has_been_seen( version: Version, sdist_only: bool = False, ) -> bool: + """Return True if this requirement/version has already been processed. + + Uses the same sdist-vs-wheel distinction as ``mark_as_seen``: + when ``sdist_only`` is True, checks only for a prior sdist build; + otherwise checks for a prior wheel build. + + Args: + req: The requirement to check. + version: The resolved version to check. + sdist_only: If True, check for a sdist-only build instead of a + full wheel build. + """ typ: typing.Literal["sdist", "wheel"] = "sdist" if sdist_only else "wheel" return self._resolved_key(req, version, typ) in self._seen_requirements @@ -889,6 +916,21 @@ def add_to_build_order( prebuilt: bool = False, constraint: Requirement | None = None, ) -> None: + """Append a package to the build-order output file if not already present. + + Deduplicates by ``(canonicalized_name, version)`` so the same package + is never written twice, regardless of extras. On each new entry, the + full build-order list is written to ``self._build_order_filename``. + + Args: + req: The requirement being added to the build order. + version: The resolved version. + source_url: URL from which the source distribution was obtained. + source_type: The kind of source (sdist, wheel, git, etc.). + prebuilt: True if the wheel was taken from a prebuilt cache rather + than compiled locally. + constraint: The version constraint active for this package, if any. + """ # We only care if this version of this package has been built, # and don't want to trigger building it twice. The "extras" # value, included in the _resolved_key() output, can confuse diff --git a/src/fromager/bootstrapper/_build.py b/src/fromager/bootstrapper/_build.py index 6c06059b..250b99ac 100644 --- a/src/fromager/bootstrapper/_build.py +++ b/src/fromager/bootstrapper/_build.py @@ -17,7 +17,23 @@ class Build(Phase): - """BUILD phase: install remaining deps, build wheel/sdist.""" + """Install remaining build deps and produce a wheel or sdist for the package. + + Three execution paths based on ``WorkItem`` state: + + - **Cached wheel**: a matching wheel was found during source preparation; + no build is performed. + - **sdist-only**: ``sdist_only`` mode is active and this is not a build + requirement; only an sdist is built. + - **Full build**: an sdist is built first, then a wheel is compiled and + the local mirror is updated. + + When ``pbi.exclusive_build`` is set the bootstrap loop drains the + background thread pool before calling ``run()`` so the build has exclusive + access to the environment. + + Next phase: ``ProcessInstallDeps``. + """ phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.BUILD tracks_why: typing.ClassVar[bool] = True diff --git a/src/fromager/bootstrapper/_cache.py b/src/fromager/bootstrapper/_cache.py index 3b87ff1b..d1ca7f31 100644 --- a/src/fromager/bootstrapper/_cache.py +++ b/src/fromager/bootstrapper/_cache.py @@ -221,7 +221,6 @@ def _bg_prepare_prebuilt( # mkdir uses exist_ok=True (atomic), and update_wheel_mirror() is already locked. logger.info(f"using pre-built wheel for {req_type} requirement") wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt) - unpack_dir = ctx.work_dir / f"{req.name}-{resolved_version}" - unpack_dir.mkdir(parents=True, exist_ok=True) + unpack_dir = _create_unpack_dir(ctx.work_dir, req, resolved_version) server.update_wheel_mirror(ctx) return PreparedSourceData(wheel_filename=wheel_filename, unpack_dir=unpack_dir) diff --git a/src/fromager/bootstrapper/_complete.py b/src/fromager/bootstrapper/_complete.py index 06559ae5..e0a9ed8d 100644 --- a/src/fromager/bootstrapper/_complete.py +++ b/src/fromager/bootstrapper/_complete.py @@ -10,7 +10,14 @@ class Complete(Phase): - """COMPLETE phase: clean up build directories.""" + """Clean up build directories after a package has been fully processed. + + Removes the unpacked source tree and build environment created during + ``PrepareSource`` and ``PrepareBuild`` to free disk space. This is the + terminal phase for every package: ``run()`` always returns an empty list. + + Next phase: none (terminal). + """ phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.COMPLETE tracks_why: typing.ClassVar[bool] = True diff --git a/src/fromager/bootstrapper/_phase.py b/src/fromager/bootstrapper/_phase.py index 90f83458..f7f584c2 100644 --- a/src/fromager/bootstrapper/_phase.py +++ b/src/fromager/bootstrapper/_phase.py @@ -25,6 +25,23 @@ class Phase(abc.ABC): phase: typing.ClassVar[BootstrapPhase] tracks_why: typing.ClassVar[bool] = True + def __init_subclass__(cls, **kwargs: typing.Any) -> None: + super().__init_subclass__(**kwargs) + # Enforce that every concrete subclass defines `phase`. Abstract + # subclasses (those that still have unimplemented abstract methods) + # are allowed to omit it; they will be checked when their own concrete + # subclasses are defined. + # + # Note: ABCMeta sets __abstractmethods__ *after* __init_subclass__ + # runs, so we cannot rely on it here. Instead we inspect the class's + # MRO for any attribute still marked as abstract. + is_abstract = any( + getattr(getattr(cls, attr, None), "__isabstractmethod__", False) + for attr in dir(cls) + ) + if not is_abstract and "phase" not in cls.__dict__: + raise TypeError(f"{cls.__name__} must define the 'phase' class attribute") + def __init__(self, work_item: WorkItem) -> None: self.work_item = work_item self.bg_future: concurrent.futures.Future[typing.Any] | None = None diff --git a/src/fromager/bootstrapper/_prepare_build.py b/src/fromager/bootstrapper/_prepare_build.py index d673d61c..2426d2b9 100644 --- a/src/fromager/bootstrapper/_prepare_build.py +++ b/src/fromager/bootstrapper/_prepare_build.py @@ -16,7 +16,17 @@ class PrepareBuild(Phase): - """PREPARE_BUILD phase: install system deps, get backend/sdist deps.""" + """Install build-system deps and discover build-backend and build-sdist deps. + + Installs the build-system wheels (already available from DFS processing), + then queries the build backend for its own dependencies and the + sdist-generation dependencies. Dependencies already satisfied by the + installed build-system set are filtered out to avoid resolving a + conflicting version (see :issue:`1194`). + + Next phase: ``Build`` + one ``Resolve`` per build-backend dependency + + one ``Resolve`` per build-sdist dependency. + """ phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PREPARE_BUILD tracks_why: typing.ClassVar[bool] = True diff --git a/src/fromager/bootstrapper/_prepare_source.py b/src/fromager/bootstrapper/_prepare_source.py index c0957a49..454d3ca3 100644 --- a/src/fromager/bootstrapper/_prepare_source.py +++ b/src/fromager/bootstrapper/_prepare_source.py @@ -53,7 +53,18 @@ def _bg_prepare_source( class PrepareSource(Phase): - """PREPARE_SOURCE phase: download source or prebuilt, get build system deps.""" + """Download the source distribution or prebuilt wheel and set up build-system deps. + + The download runs in a background thread (via ``background_work``). + For prebuilt packages the background task fetches the wheel directly and + the build phases are skipped entirely. For source builds the background + task downloads and unpacks the sdist; ``run()`` then reads build-system + dependencies from the unpacked tree. + + Next phase: + - Prebuilt wheel: ``ProcessInstallDeps``. + - Source build: ``PrepareBuild`` + one ``Resolve`` per build-system dependency. + """ phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PREPARE_SOURCE tracks_why: typing.ClassVar[bool] = True diff --git a/src/fromager/bootstrapper/_process_install_deps.py b/src/fromager/bootstrapper/_process_install_deps.py index 42e04498..a6ed4b58 100644 --- a/src/fromager/bootstrapper/_process_install_deps.py +++ b/src/fromager/bootstrapper/_process_install_deps.py @@ -73,7 +73,15 @@ def _get_install_dependencies( class ProcessInstallDeps(Phase): - """PROCESS_INSTALL_DEPS phase: hooks, extract deps, build order.""" + """Run post-bootstrap hooks, extract install deps, and record the build order. + + Runs any configured post-bootstrap hooks for the completed package (errors + are fatal in normal mode; recorded as warnings in test mode). Extracts + install-time dependencies from the built wheel or sdist. Appends the + package to the persistent build-order file via ``Bootstrapper.add_to_build_order``. + + Next phase: ``Complete`` + one ``Resolve`` per install dependency. + """ phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PROCESS_INSTALL_DEPS tracks_why: typing.ClassVar[bool] = True diff --git a/src/fromager/bootstrapper/_resolve.py b/src/fromager/bootstrapper/_resolve.py index 02221843..51dc541c 100644 --- a/src/fromager/bootstrapper/_resolve.py +++ b/src/fromager/bootstrapper/_resolve.py @@ -40,7 +40,16 @@ def _bg_resolve( class Resolve(Phase): - """RESOLVE phase: resolve versions and expand into Start items.""" + """Resolve a requirement's available version(s) and fan out to ``Start`` items. + + Resolution runs in a background thread (via ``background_work``); ``run()`` + blocks until the result is ready. In normal mode the highest matching + version is used; in ``multiple_versions`` mode every candidate version is + returned, minus any that previously failed or whose wheels are already + cached. + + Next phase: one ``Start`` item per version that still needs processing. + """ phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.RESOLVE tracks_why: typing.ClassVar[bool] = False diff --git a/src/fromager/bootstrapper/_start.py b/src/fromager/bootstrapper/_start.py index 1fca7ea6..2db1eac9 100644 --- a/src/fromager/bootstrapper/_start.py +++ b/src/fromager/bootstrapper/_start.py @@ -15,7 +15,16 @@ class Start(Phase): - """START phase: add to graph, check if already seen.""" + """Record a resolved requirement in the dependency graph and deduplicate. + + Adds the ``(parent → req)`` edge to the dependency graph, then checks + whether this ``(req, version)`` pair has already been processed. Duplicate + requirements are silently dropped; new ones proceed to source preparation. + ``tracks_why`` is ``False`` so graph additions happen before the why-stack + is updated. + + Next phase: ``PrepareSource`` (new requirement) or ``[]`` (already seen). + """ phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.START tracks_why: typing.ClassVar[bool] = False diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index f18c6918..97cf6af0 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -38,6 +38,27 @@ from fromager.requirements_file import RequirementType, SourceType +def test_phase_subclass_without_phase_attribute_raises() -> None: + """Concrete Phase subclass missing 'phase' raises TypeError at class definition.""" + with pytest.raises(TypeError, match="must define the 'phase' class attribute"): + + class _BadPhase(Phase): + def run(self, bt: typing.Any) -> list[Phase]: # type: ignore[override] + return [] + + +def test_phase_abstract_subclass_without_phase_attribute_is_allowed() -> None: + """Abstract Phase subclass may omit 'phase' without error.""" + import abc + + class _AbstractMiddle(Phase): + @abc.abstractmethod + def helper(self) -> None: ... + + # Defining _AbstractMiddle itself must not raise. + assert _AbstractMiddle.__abstractmethods__ + + def test_seen(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) req = Requirement("testdist") From 9ece33731a60787fd8016e140f69a4a6ecb42edc Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 6 Jul 2026 14:55:08 -0400 Subject: [PATCH 17/19] (docs): remove phase classes plan from proposals The doc added to the proposals directory was the agent's plan, not really a proposal. Clean that up. Signed-off-by: Doug Hellmann --- docs/proposals/phase-classes-plan.md | 320 --------------------------- 1 file changed, 320 deletions(-) delete mode 100644 docs/proposals/phase-classes-plan.md diff --git a/docs/proposals/phase-classes-plan.md b/docs/proposals/phase-classes-plan.md deleted file mode 100644 index 9efc7009..00000000 --- a/docs/proposals/phase-classes-plan.md +++ /dev/null @@ -1,320 +0,0 @@ -# Plan: Refactor Bootstrapper Phase Handlers into PhaseItem Class Hierarchy - -## Context - -`Bootstrapper` in `src/fromager/bootstrapper.py` has grown large (~2268 lines). The seven `_phase_*` methods that implement the bootstrap loop's processing logic are co-mingled with orchestration, state tracking, and utility methods. The goal is to move each phase handler into its own class (`PhaseItem` subclass) that lives on the stack, wrapping a `WorkItem` data container. This makes each phase self-contained and easier to understand, test, and extend. - -## Design - -### New: `PhaseItem` abstract base class - -Each object pushed onto the bootstrap stack is a `PhaseItem`. It wraps a `WorkItem` (the accumulated per-package state) and implements the logic for one phase. - -```python -class PhaseItem(abc.ABC): - phase: typing.ClassVar[BootstrapPhase] # which phase this item represents - tracks_why: typing.ClassVar[bool] = True # whether to push onto why stack - - def __init__(self, work_item: WorkItem) -> None: - self.work_item = work_item - self.bg_future: concurrent.futures.Future[typing.Any] | None = None - - @abc.abstractmethod - def run(self, bt: Bootstrapper) -> list[PhaseItem]: ... - - def background_work(self, bt: Bootstrapper) -> typing.Callable[[], typing.Any] | None: - """Return a zero-argument callable for background I/O, or None. - Override in subclasses that need background prefetching. - ``bt`` is provided so subclasses can capture Bootstrapper state - (e.g. resolver, ctx) into the returned closure without storing - a circular reference on the item itself.""" - return None - - def __str__(self) -> str: - """Human-readable representation for logging. - Default: ``"()"``. Subclasses may override.""" - wi = self.work_item - return f"{type(self).__name__}({wi.req})" - - def as_json(self) -> dict[str, typing.Any]: - """Return a JSON-serialisable dict for stack-state recording. - The base implementation covers all common ``WorkItem`` fields. - Subclasses may override to add phase-specific entries.""" - wi = self.work_item - return { - "req": str(wi.req), - "req_type": str(wi.req_type), - "phase": str(self.phase), - "resolved_version": str(wi.resolved_version) if wi.resolved_version is not None else None, - "source_url": wi.source_url, - "build_sdist_only": wi.build_sdist_only, - "why": [ - {"req_type": str(rt), "req": str(r), "version": str(v)} - for rt, r, v in wi.why_snapshot - ], - "parent": ( - {"req": str(wi.parent[0]), "version": str(wi.parent[1])} - if wi.parent else None - ), - "build_system_deps": sorted(str(r) for r in wi.build_system_deps), - "build_backend_deps": sorted(str(r) for r in wi.build_backend_deps), - "build_sdist_deps": sorted(str(r) for r in wi.build_sdist_deps), - } -``` - -### New: 7 concrete `PhaseItem` subclasses - -| Class | phase | tracks_why | overrides background_work | -| -- | -- | -- | -- | -| `ResolveItem` | RESOLVE | False | Yes (calls `_bg_resolve`) | -| `StartItem` | START | False | No | -| `PrepareSourceItem` | PREPARE_SOURCE | True | Yes (calls `_bg_prepare_source` or `_bg_prepare_prebuilt`) | -| `PrepareBuildItem` | PREPARE_BUILD | True | No | -| `BuildItem` | BUILD | True | No | -| `ProcessInstallDepsItem` | PROCESS_INSTALL_DEPS | True | No | -| `CompleteItem` | COMPLETE | True | No | - -Each `run(self, bt: Bootstrapper) -> list[PhaseItem]` method contains the body of the corresponding `_phase_*` method. References to the old `self` (Bootstrapper) become `bt`; references to `item.*` become `self.work_item.*`. - -#### Phase advancement pattern - -The current handlers advance an item to the next phase by mutating `item.phase` then returning `[item]`. With `PhaseItem` classes the class encodes the phase, so mutation is impossible. Instead, each `run()` that continues the same package constructs a **new** `PhaseItem` subclass wrapping the same `work_item`: - -```python -# Old: mutation -item.phase = BootstrapPhase.PREPARE_BUILD -return [item] + dep_items - -# New: construction -return [PrepareBuildItem(self.work_item)] + dep_items -``` - -The expected return types per class are: - -| Class | Returns | -| -- | -- | -| `ResolveItem` | `list[StartItem]` (one per resolved version) | -| `StartItem` | `[]` (already seen) or `[PrepareSourceItem]` | -| `PrepareSourceItem` | `[ProcessInstallDepsItem]` (prebuilt) or `[PrepareBuildItem] + dep_items` (source) | -| `PrepareBuildItem` | `[BuildItem] + dep_items` | -| `BuildItem` | `[ProcessInstallDepsItem]` | -| `ProcessInstallDepsItem` | `[CompleteItem] + dep_items` | -| `CompleteItem` | `[]` | - -Assertions such as `assert item.bg_future is not None` in the current `_phase_resolve` and `_phase_prepare_source` bodies become `assert self.bg_future is not None` and must be retained in the corresponding `run()` methods. - -#### `background_work()` and module-level helpers - -`_bg_resolve`, `_bg_prepare_source`, and `_bg_prepare_prebuilt` are already **module-level functions** (not `Bootstrapper` methods). Closures built inside `background_work(self, bt)` capture them directly from module scope — `bt` is provided only to access `Bootstrapper` attributes (e.g. `bt._resolver`, `bt.ctx`) needed to construct the closure, not to store a reference to `bt` on the item. - -### Changes to `BootstrapPhase` - -The `BootstrapPhase` enum is **retained** — it is still referenced by `PhaseItem.phase: ClassVar[BootstrapPhase]` and used for its string values in `as_json()`, log messages, and error output. - -- **Remove** the `tracks_why` property — it becomes dead code once `tracks_why` moves to `PhaseItem` class variables. - -### Changes to `WorkItem` - -- **Remove** `phase: BootstrapPhase` field — phase is now encoded in the `PhaseItem` subclass type -- **Remove** `bg_future` field — moved to `PhaseItem` base class -- All other fields remain unchanged (they accumulate state across phases via `work_item`) - -### Changes to `Bootstrapper` - -| What | Change | -| -- | -- | -| `_phase_resolve`, `_phase_start`, `_phase_prepare_source`, `_phase_prepare_build`, `_phase_build`, `_phase_process_install_deps`, `_phase_complete` | **Delete** — logic moves into `PhaseItem` subclasses | -| `_get_background_work()` | **Delete** — replaced by `PhaseItem.background_work()` | -| `_dispatch_phase(item)` | Simplify to `return item.run(self)`; remove the `match/case` block and the `case _: raise ValueError(...)` fallthrough (unreachable once all subclasses are concrete) | -| `_push_items(stack, items)` | Accept `list[PhaseItem]`; retain the `if self._bg_pool is not None` guard; iterate `reversed(items)` (LIFO) to submit the highest-priority item first; call `item.background_work(self)` instead of `self._get_background_work(item)` and if not None, `item.bg_future = self._bg_pool.submit(bg_work)` | -| `_track_why(item)` | Accept `PhaseItem`; use `item.tracks_why` instead of `item.phase.tracks_why`; access `item.work_item.resolved_version`, `item.work_item.req_type`, `item.work_item.req` | -| `_record_stack_state(stack)` | Accept `list[PhaseItem]`; replace `[serialize(item) for item in reversed(stack)]` with `[item.as_json() for item in reversed(stack)]` — remove the inline `serialize()` helper entirely | -| `_handle_phase_error(item, err)` | Replace `item.phase == BootstrapPhase.RESOLVE` checks with `isinstance(item, ResolveItem)` etc.; replace phase-mutation fallback with `ProcessInstallDepsItem(item.work_item)` construction (see Error handling); update all `item.*` attribute accesses to `item.work_item.*` (see Attribute migration below) | -| `_create_unresolved_work_items(...)` | Return `list[ResolveItem]` instead of `list[WorkItem]`; remove `phase=BootstrapPhase.RESOLVE` from `WorkItem(...)` constructor; wrap each result: `ResolveItem(WorkItem(...))` | -| `bootstrap()` | Remove `phase=BootstrapPhase.RESOLVE` from `WorkItem(...)` calls; wrap each in `ResolveItem(WorkItem(...))` | -| `_bootstrap_one()` | Same — remove `phase=` from `WorkItem(...)`, wrap in `ResolveItem(WorkItem(...))` | -| `_run_bootstrap_loop(stack)` | Accept `list[PhaseItem]`; change `it.phase == BootstrapPhase.RESOLVE` progress-bar check to `isinstance(it, ResolveItem)`; change `self.why = list(item.why_snapshot)` to `self.why = list(item.work_item.why_snapshot)`; change `req_ctxvar_context(item.req, item.resolved_version)` to `req_ctxvar_context(item.work_item.req, item.work_item.resolved_version)` | - -#### Attribute migration in `Bootstrapper` methods - -After the refactor, `item` parameters that were `WorkItem` become `PhaseItem`. All attribute accesses must be updated: - -| Old (`WorkItem`) | New (`PhaseItem`) | -| -- | -- | -| `item.phase` | `type(item).phase` (ClassVar — still accessible on instance, but explicit is clearer) | -| `item.req` | `item.work_item.req` | -| `item.req_type` | `item.work_item.req_type` | -| `item.resolved_version` | `item.work_item.resolved_version` | -| `item.source_url` | `item.work_item.source_url` | -| `item.pbi_pre_built` | `item.work_item.pbi_pre_built` | -| `item.build_result` | `item.work_item.build_result` | -| `item.why_snapshot` | `item.work_item.why_snapshot` | -| `item.bg_future` | `item.bg_future` (moved to `PhaseItem` base — no change) | - -This applies to `_handle_phase_error`, `_run_bootstrap_loop`, `_track_why`, and any other `Bootstrapper` method that receives a `PhaseItem`. Inside each `PhaseItem.run()` method, references to the old `self` (Bootstrapper) become `bt` and references to the old `item.*` become `self.work_item.*`. - -#### `StartItem.run()` mutation ordering - -`StartItem.run()` must set `self.work_item.build_sdist_only` and `self.work_item.pbi_pre_built` **before** constructing `PrepareSourceItem(self.work_item)`, because `PrepareSourceItem.background_work(bt)` reads `work_item.pbi_pre_built` to choose between `_bg_prepare_source` and `_bg_prepare_prebuilt`: - -```python -# Inside StartItem.run(): -self.work_item.build_sdist_only = ( - bt.sdist_only and not bt._processing_build_requirement(self.work_item.req_type) -) -self.work_item.pbi_pre_built = bt.ctx.package_build_info(self.work_item.req).pre_built -# Now safe to hand work_item to the next phase: -return [PrepareSourceItem(self.work_item)] -``` - -### Error handling - -`_handle_phase_error` uses `isinstance` checks: - -```python -if isinstance(item, ResolveItem): ... -if isinstance(item, (PrepareSourceItem, PrepareBuildItem, BuildItem)): ... -``` - -The test-mode prebuilt fallback currently mutates `item.phase` before returning: - -```python -# Old: mutation -item.build_result = fallback -item.phase = BootstrapPhase.PROCESS_INSTALL_DEPS -return [item] -``` - -With `PhaseItem` classes this must instead construct a new item: - -```python -# New: construction -item.work_item.build_result = fallback -return [ProcessInstallDepsItem(item.work_item)] -``` - -## Files to Modify - -- `src/fromager/bootstrapper.py` — all changes (primary file) -- `tests/test_bootstrapper.py` — update affected tests - -## Test Changes - -- `test_phase_build_produces_source_build_result`: - - - Replace `WorkItem(phase=BootstrapPhase.BUILD, ...)` construction with `BuildItem(WorkItem(...))` (no `phase=` arg) - - Change `bt._phase_build(item)` → `item.run(bt)` inside the `bt._track_why(item)` context - - Change `result_items[0].phase == BootstrapPhase.PROCESS_INSTALL_DEPS` → `isinstance(result_items[0], ProcessInstallDepsItem)` - -- `_make_resolve_item()` helper: - - - Remove `phase=BootstrapPhase.RESOLVE` from `WorkItem(...)` call - - Wrap result in `ResolveItem`: `return ResolveItem(WorkItem(...))` - - Update return type annotation from `bootstrapper.WorkItem` to `bootstrapper.ResolveItem` - - Update all callers that compare `item.phase` directly to use `type(item).phase` or `isinstance(item, ResolveItem)` - -- `_record_and_load()` helper: - - - Change parameter type annotation from `list[bootstrapper.WorkItem]` to `list[bootstrapper.PhaseItem]` - -- `test_record_stack_state_full_item`: - - - Replace `WorkItem(phase=BootstrapPhase.BUILD, ...)` with `BuildItem(WorkItem(...))` (remove `phase=` arg) - -- `test_record_stack_state_dep_sets_are_sorted`: - - - Replace `WorkItem(phase=BootstrapPhase.BUILD, ...)` with `BuildItem(WorkItem(...))` (remove `phase=` arg) - -- `test_multiple_versions_continues_on_error` and similar tests that check `item.phase in (BootstrapPhase.RESOLVE, BootstrapPhase.START, ...)` inside mock dispatchers: - - - Replace phase equality checks with `isinstance(item, (ResolveItem, StartItem, ...))` - -- Tests for `_handle_phase_error`: - - - Any test that constructs a mid-pipeline `WorkItem` (e.g. with `phase=PREPARE_SOURCE`) and passes it to `_handle_phase_error` must instead construct the corresponding `PhaseItem` subclass (e.g. `PrepareSourceItem(WorkItem(...))`) - - Tests that check the prebuilt fallback return value must assert `isinstance(result[0], ProcessInstallDepsItem)` rather than checking a mutated `WorkItem.phase` - -- All remaining tests that import or reference `WorkItem` with a `phase=` keyword argument must drop the `phase=` argument - -## New Tests - -These test cases do not exist yet and should be added to cover the new class hierarchy and changed method behaviors. - -### `PhaseItem` base class - -- `test_phase_item_str_default` — `str(BuildItem(work_item))` returns `"BuildItem()"` using the default `__str__` implementation. -- `test_phase_item_background_work_returns_none_by_default` — phases that do not override `background_work()` (`PrepareBuildItem`, `BuildItem`, `ProcessInstallDepsItem`, `CompleteItem`) return `None`. - -### Class variable correctness - -- `test_phase_class_variables` — parametrize over all 7 subclasses; assert `MyClass.phase == expected_phase` and `MyClass.tracks_why == expected_bool` as listed in the design table. Verifies that class-variable declarations are not accidentally overridden by instance state. - -### `background_work()` dispatch - -- `test_resolve_item_background_work_returns_callable` — `ResolveItem(work_item).background_work(bt)` returns a non-`None` callable (does not call it; just verifies a callable is produced). -- `test_prepare_source_item_background_work_source` — when `work_item.pbi_pre_built=False`, `PrepareSourceItem.background_work(bt)` returns a callable that wraps `_bg_prepare_source`. -- `test_prepare_source_item_background_work_prebuilt` — when `work_item.pbi_pre_built=True`, `PrepareSourceItem.background_work(bt)` returns a callable that wraps `_bg_prepare_prebuilt`. - -### `StartItem` mutation ordering - -- `test_start_item_sets_pbi_pre_built_before_constructing_prepare_source` — `StartItem.run(bt)` must set `work_item.pbi_pre_built` before the returned `PrepareSourceItem` is constructed, so that `PrepareSourceItem.background_work(bt)` immediately sees the correct value. Verify by checking `result[0].work_item.pbi_pre_built` is set correctly and that calling `result[0].background_work(bt)` (with the same bootstrapper) returns the appropriate callable branch. - -### Phase advancement shares `work_item` identity - -- `test_phase_advancement_preserves_work_item_identity` — when `BuildItem(wi).run(bt)` returns `[ProcessInstallDepsItem(...)]`, assert `result[0].work_item is wi` (same object, not a copy). Applies to any phase that returns the next phase wrapping the same `work_item`. - -### `CompleteItem` - -- `test_complete_item_run_returns_empty_list` — `CompleteItem(work_item).run(bt)` returns `[]`. - -### `_dispatch_phase` delegation - -- `test_dispatch_phase_calls_item_run` — `bt._dispatch_phase(item)` calls `item.run(bt)` and returns its result without a `match/case` block. Patch `item.run` to return a sentinel list and assert the sentinel is returned. - -### `_track_why` with `PhaseItem` - -- `test_track_why_not_pushed_for_no_tracks_why_item` — inside `bt._track_why(resolve_item)`, `bt.why` is not modified (because `ResolveItem.tracks_why` is `False`). -- `test_track_why_pushed_for_tracks_why_item` — inside `bt._track_why(build_item)`, `bt.why` gains one entry while in the context and is restored on exit (because `BuildItem.tracks_why` is `True`). - -### `_push_items` background future submission - -- `test_push_items_sets_bg_future_when_pool_exists` — construct a `Bootstrapper` with a live `ThreadPoolExecutor` as `_bg_pool`; call `_push_items` with a `ResolveItem`; assert `item.bg_future` is not `None` after the call. -- `test_push_items_no_bg_future_when_pool_is_none` — when `bt._bg_pool` is `None`, calling `_push_items` with a `ResolveItem` leaves `item.bg_future` as `None`. - -### `as_json()` phase field per subclass - -- `test_as_json_phase_field_per_subclass` — parametrize over all 7 `(SubclassType, BootstrapPhase)` pairs; construct the subclass wrapping a minimal `WorkItem` and assert `item.as_json()["phase"] == str(expected_phase)`. Guards against a subclass accidentally inheriting the wrong `phase` ClassVar or the serialization using a stale field. - -### `_create_unresolved_work_items` return type - -- `test_create_unresolved_work_items_returns_resolve_items` — the returned items are `ResolveItem` instances (not bare `WorkItem` objects). Each `item.work_item` holds the expected `req` and `req_type`. - -## Verification - -```bash -# After each phase class is extracted: -hatch run mypy:check src/fromager/bootstrapper.py -hatch run test:test tests/test_bootstrapper.py - -# Full check at the end: -hatch run lint:fix src/fromager/bootstrapper.py tests/test_bootstrapper.py -hatch run mypy:check src/fromager/bootstrapper.py -hatch run test:test tests/test_bootstrapper.py -hatch run lint:check src/fromager/bootstrapper.py -``` - -## Implementation Order - -1. Add `abc` import; define `PhaseItem` abstract base class above `Bootstrapper` -2. Create each concrete subclass one at a time, moving `_phase_*` body into `run()`: - a. `ResolveItem` (with `background_work()` override) - b. `StartItem` - c. `PrepareSourceItem` (with `background_work()` override) - d. `PrepareBuildItem` - e. `BuildItem` - f. `ProcessInstallDepsItem` - g. `CompleteItem` -3. Update `WorkItem`: remove `phase` and `bg_future` fields -4. Update `Bootstrapper`: remove deleted methods, simplify `_dispatch_phase`, `_push_items`, `_track_why`, `_record_stack_state`, `_handle_phase_error`, `_create_unresolved_work_items`, `bootstrap()`, `_bootstrap_one()` -5. Update `tests/test_bootstrapper.py` -6. Run full verification From 4685a6ca686a51ff1fcceea5a42ab02d6cd8dd14 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 7 Jul 2026 11:49:33 -0400 Subject: [PATCH 18/19] fix(bootstrapper): create working_src_dir.parent before shutil.move in git clone flow On first-time git URL resolution, working_src_dir.parent did not exist yet, causing shutil.move to raise FileNotFoundError. The mkdir call was inside the `if working_src_dir.exists()` branch, so it was skipped when the destination was new. Move it unconditionally before shutil.move, and add an assert to satisfy the type checker. Adds a regression test that lets the real filesystem run (no shutil.move mock) to verify the parent directory is created before the move. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Doug Hellmann --- src/fromager/bootstrapper/_bootstrapper.py | 3 +- tests/test_bootstrap.py | 40 ++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py index 3a9d34cf..dfec81e2 100644 --- a/src/fromager/bootstrapper/_bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -756,7 +756,8 @@ def _resolve_version_from_git_url(self, req: Requirement) -> tuple[str, Version] # dynamically computed by something like setuptools-scm. logger.debug("cleaning up %s", working_src_dir) shutil.rmtree(working_src_dir) - working_src_dir.parent.mkdir(parents=True, exist_ok=True) + assert working_src_dir is not None + working_src_dir.parent.mkdir(parents=True, exist_ok=True) logger.info("moving cloned repo to %s", working_src_dir) shutil.move(clone_dir, str(working_src_dir)) diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 3a30ec6f..0a3c9f7c 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -894,3 +894,43 @@ def test_resolve_version_from_git_url_invalid_scheme( bs = bootstrapper.Bootstrapper(tmp_context) with pytest.raises(ValueError, match="unsupported VCS URL scheme"): bs._resolve_version_from_git_url(req) + + +def test_resolve_version_from_git_url_creates_parent_dir_on_first_clone( + tmp_context: context.WorkContext, +) -> None: + """working_src_dir.parent is created before shutil.move on first clone. + + When the URL has no version ref, the version is discovered after cloning, + so working_src_dir is computed inside the clone block. Its parent directory + does not exist yet and must be created before shutil.move is called. + """ + req = Requirement("test-pkg @ git+https://pkg.test/repo.git") + + def fake_download_git_source( + *, + ctx: context.WorkContext, + req: Requirement, + url_to_clone: str, + destination_dir: pathlib.Path, + ref: str, + ) -> None: + destination_dir.mkdir(parents=True, exist_ok=True) + + with ( + patch( + "fromager.sources.download_git_source", + side_effect=fake_download_git_source, + ), + patch( + "fromager.bootstrapper.Bootstrapper._get_version_from_package_metadata", + return_value=Version("1.0.0"), + ), + ): + bs = bootstrapper.Bootstrapper(tmp_context) + src_dir, version = bs._resolve_version_from_git_url(req) + + expected = tmp_context.work_dir / "test-pkg-1.0.0" / "test-pkg-1.0.0" + assert pathlib.Path(src_dir) == expected + assert expected.exists() + assert version == Version("1.0.0") From 8d117d2eae5dc35df2e8d1775c2fa28c2ae2e86e Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 7 Jul 2026 11:50:33 -0400 Subject: [PATCH 19/19] test(bootstrapper): assert update_wheel_mirror is called in _build_wheel test The wheel-build test patched `fromager.server.update_wheel_mirror` but never asserted the call, so removing the wiring would go undetected. Capture the mock and assert it is called once with the expected context. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Doug Hellmann --- tests/test_bootstrapper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index 97cf6af0..797121f5 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -1054,11 +1054,12 @@ def test_build_item_build_wheel(tmp_context: WorkContext) -> None: item, "_build_sdist", return_value=built_sdist ) as mock_build_sdist, patch("fromager.wheels.build_wheel", return_value=built_wheel), - patch("fromager.server.update_wheel_mirror"), + patch("fromager.server.update_wheel_mirror") as mock_update_mirror, ): wheel_filename, sdist_filename = item._build_wheel(tmp_context) mock_build_sdist.assert_called_once_with(tmp_context) + mock_update_mirror.assert_called_once_with(tmp_context) assert wheel_filename == tmp_context.wheels_downloads / built_wheel.name assert sdist_filename == built_sdist