diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py deleted file mode 100644 index 38a0bd5f..00000000 --- a/src/fromager/bootstrapper.py +++ /dev/null @@ -1,2278 +0,0 @@ -from __future__ import annotations - -import concurrent.futures -import contextlib -import dataclasses -import datetime -import json -import logging -import operator -import os -import pathlib -import shutil -import tempfile -import typing -import zipfile -from enum import StrEnum -from urllib.parse import urlparse - -import requests.exceptions -from packaging.requirements import Requirement -from packaging.utils import NormalizedName, canonicalize_name -from packaging.version import Version -from resolvelib.resolvers import ResolverException - -from . import ( - bootstrap_requirement_resolver, - build_environment, - dependencies, - finders, - gitutils, - hooks, - progress, - resolver, - server, - sources, - threading_utils, - wheels, -) -from .dependency_graph import DependencyGraph -from .log import req_ctxvar_context, requirement_ctxvar -from .requirements_file import RequirementType, SourceType - -if typing.TYPE_CHECKING: - from . import context - -logger = logging.getLogger(__name__) - -# package name, extras, version, sdist/wheel -SeenKey = tuple[NormalizedName, tuple[str, ...], str, typing.Literal["sdist", "wheel"]] - -_DEFAULT_BG_THREADS: int = max(1, threading_utils.get_cpu_count() // 2) - - -@dataclasses.dataclass -class SourceBuildResult: - """Result of building or downloading a package. - - Captures the output artifacts from either a source build or - prebuilt wheel download, used across bootstrap phases. - """ - - wheel_filename: pathlib.Path | None - sdist_filename: pathlib.Path | None - unpack_dir: pathlib.Path - sdist_root_dir: pathlib.Path | None - build_env: build_environment.BuildEnvironment | None - source_type: SourceType - - -@dataclasses.dataclass -class PreparedSourceData: - """Result of background I/O pre-fetching returned to the main thread. - - Fields are set in one of three combinations depending on the result type: - - - Source (no cache hit): only ``sdist_root_dir`` is set. - - Source (cache hit): both ``sdist_root_dir`` and ``cached_wheel_filename`` are set. - - Prebuilt wheel: both ``wheel_filename`` and ``unpack_dir`` are set. - """ - - # Source path: set after download+unpack OR cache hit - sdist_root_dir: pathlib.Path | None = None - # Source path: set when the result came from the wheel cache - cached_wheel_filename: pathlib.Path | None = None - # Prebuilt path: downloaded wheel file - wheel_filename: pathlib.Path | None = None - # Prebuilt path: unpack directory (created by mkdir) - unpack_dir: pathlib.Path | None = None - - -# Valid failure types for test mode error recording -FailureType = typing.Literal["resolution", "bootstrap", "hook", "dependency_extraction"] - - -class FailureRecord(typing.TypedDict): - """Record of a package that failed during bootstrap in test mode. - - Attributes: - package: The package name that failed. - version: The resolved version (None if resolution failed). - exception_type: The exception class name. - exception_message: The exception message string. - failure_type: Category of failure for analysis. - """ - - package: str - version: str | None - exception_type: str - exception_message: str - failure_type: FailureType - - -class BootstrapPhase(StrEnum): - """Processing phases for iterative bootstrap. - - All packages: RESOLVE -> START -> ... - Source packages: ... -> PREPARE_SOURCE -> PREPARE_BUILD -> BUILD - -> PROCESS_INSTALL_DEPS -> COMPLETE. - Prebuilt packages: ... -> PREPARE_SOURCE -> PROCESS_INSTALL_DEPS -> COMPLETE. - """ - - RESOLVE = "resolve" - START = "start" - PREPARE_SOURCE = "prepare-source" - PREPARE_BUILD = "prepare-build" - BUILD = "build" - PROCESS_INSTALL_DEPS = "process-install-deps" - COMPLETE = "complete" - - @property - def tracks_why(self) -> bool: - """Whether this phase pushes onto the dependency-chain (why) stack.""" - return self not in (BootstrapPhase.RESOLVE, BootstrapPhase.START) - - -@dataclasses.dataclass -class WorkItem: - """A unit of work in the iterative bootstrap loop. - - Carries identity fields set at creation time and accumulated state - populated across phases as processing advances. - - Items enter at the RESOLVE phase with only req and req_type set. - The RESOLVE phase populates source_url and resolved_version, then - creates new items at the START phase for each resolved version. - """ - - # Identity (set at creation) - req: Requirement - req_type: RequirementType - phase: BootstrapPhase - why_snapshot: list[tuple[RequirementType, Requirement, Version]] - parent: tuple[Requirement, Version] | None = None - - # Populated by RESOLVE phase (None until then) - source_url: str | None = None - resolved_version: Version | None = None - - build_sdist_only: bool = False - - # Accumulated state (populated during phases) - build_env: build_environment.BuildEnvironment | None = None - sdist_root_dir: pathlib.Path | None = None - unpack_dir: pathlib.Path | None = None - cached_wheel_filename: pathlib.Path | None = None - build_result: SourceBuildResult | None = None - pbi_pre_built: bool = False - build_system_deps: set[Requirement] = dataclasses.field(default_factory=set) - build_backend_deps: set[Requirement] = dataclasses.field(default_factory=set) - build_sdist_deps: set[Requirement] = dataclasses.field(default_factory=set) - - # Background future: set by _push_items for RESOLVE and PREPARE_SOURCE items. - # None for all other phases or before _push_items has been called. - bg_future: concurrent.futures.Future[typing.Any] | None = dataclasses.field( - default=None, compare=False, repr=False - ) - - -def _create_unpack_dir( - work_dir: pathlib.Path, - req: Requirement, - resolved_version: Version, -) -> pathlib.Path: - unpack_dir = work_dir / f"{req.name}-{resolved_version}" - unpack_dir.mkdir(parents=True, exist_ok=True) - return unpack_dir - - -def _extract_build_reqs_from_wheel( - work_dir: pathlib.Path, - req: Requirement, - resolved_version: Version, - wheel_filename: pathlib.Path, -) -> pathlib.Path | None: - """Extract fromager build requirement files from a wheel archive. - - Looks for files prefixed with `FROMAGER_BUILD_REQ_PREFIX` inside the - wheel's `.dist-info` directory and extracts them to a local unpack - directory. Returns the unpack directory on success, or None if the - files are not present or extraction fails. - """ - dist_name, dist_version, _, _ = wheels.extract_info_from_wheel_file( - req, wheel_filename - ) - unpack_dir = _create_unpack_dir(work_dir, req, resolved_version) - dist_filename = f"{dist_name}-{dist_version}" - dist_info_path = pathlib.Path(f"{dist_filename}.dist-info") - req_filenames: list[str] = [ - dependencies.BUILD_BACKEND_REQ_FILE_NAME, - dependencies.BUILD_SDIST_REQ_FILE_NAME, - dependencies.BUILD_SYSTEM_REQ_FILE_NAME, - ] - try: - with zipfile.ZipFile(wheel_filename) as archive: - for filename in req_filenames: - zipinfo = archive.getinfo( - str( - dist_info_path - / f"{wheels.FROMAGER_BUILD_REQ_PREFIX}-{filename}" - ) - ) - if os.path.isabs(zipinfo.filename) or ".." in zipinfo.filename: - raise ValueError(f"Unsafe path in wheel: {zipinfo.filename}") - zipinfo.filename = filename - output_file = archive.extract(zipinfo, unpack_dir) - logger.info(f"extracted {output_file}") - logger.info(f"extracted build requirements from wheel into {unpack_dir}") - return unpack_dir - except Exception as e: - logger.info(f"could not extract build requirements from wheel: {e}") - for filename in req_filenames: - unpack_dir.joinpath(filename).unlink(missing_ok=True) - return None - - -def _look_for_existing_wheel( - ctx: context.WorkContext, - req: Requirement, - resolved_version: Version, - search_in: pathlib.Path, -) -> tuple[pathlib.Path | None, pathlib.Path | None]: - pbi = ctx.package_build_info(req) - expected_build_tag = pbi.build_tag(resolved_version) - logger.info( - f"looking for existing wheel for version {resolved_version} with build tag {expected_build_tag} in {search_in}" - ) - wheel_filename = finders.find_wheel( - downloads_dir=search_in, - req=req, - dist_version=str(resolved_version), - build_tag=expected_build_tag, - ) - if not wheel_filename: - return None, None - _, _, build_tag, _ = wheels.extract_info_from_wheel_file(req, wheel_filename) - if expected_build_tag and expected_build_tag != build_tag: - logger.info( - f"found wheel for {resolved_version} in {wheel_filename} but build tag does not match. Got {build_tag} but expected {expected_build_tag}" - ) - return None, None - logger.info(f"found existing wheel {wheel_filename}") - build_reqs_dir = _extract_build_reqs_from_wheel( - ctx.work_dir, req, resolved_version, wheel_filename - ) - return wheel_filename, build_reqs_dir - - -def _download_wheel_from_cache( - ctx: context.WorkContext, - cache_wheel_server_url: str | None, - req: Requirement, - resolved_version: Version, -) -> tuple[pathlib.Path | None, pathlib.Path | None]: - if not cache_wheel_server_url: - return None, None - logger.info(f"checking if wheel was already uploaded to {cache_wheel_server_url}") - try: - pinned_req = Requirement(f"{req.name}=={resolved_version}") - provider = finders.PyPICacheProvider( - cache_server_url=cache_wheel_server_url, - constraints=ctx.constraints, - ) - results = resolver.find_all_matching_from_provider(provider, pinned_req) - wheel_url, _ = results[0] - wheelfile_name = pathlib.Path(urlparse(wheel_url).path) - pbi = ctx.package_build_info(req) - expected_build_tag = pbi.build_tag(resolved_version) - logger.info(f"has expected build tag {expected_build_tag}") - changelogs = pbi.get_changelog(resolved_version) - logger.debug(f"has change logs {changelogs}") - - _, _, build_tag, _ = wheels.extract_info_from_wheel_file(req, wheelfile_name) - if expected_build_tag and expected_build_tag != build_tag: - logger.info( - f"found wheel for {resolved_version} in cache but build tag does not match. Got {build_tag} but expected {expected_build_tag}" - ) - return None, None - - cached_wheel = wheels.download_wheel( - req=req, wheel_url=wheel_url, output_directory=ctx.wheels_downloads - ) - if cache_wheel_server_url != ctx.wheel_server_url: - server.update_wheel_mirror(ctx) - logger.info("found built wheel on cache server") - unpack_dir = _extract_build_reqs_from_wheel( - ctx.work_dir, req, resolved_version, cached_wheel - ) - return cached_wheel, unpack_dir - except ResolverException: - logger.info( - f"did not find wheel for {resolved_version} in {cache_wheel_server_url}" - ) - return None, None - except requests.exceptions.RequestException as err: - logger.warning( - f"network error checking wheel cache for {resolved_version} " - f"at {cache_wheel_server_url}: {err}" - ) - return None, None - except Exception as err: - logger.warning( - f"unexpected error checking wheel cache for {resolved_version} " - f"at {cache_wheel_server_url}: {err}" - ) - return None, None - - -def _find_cached_wheel( - ctx: context.WorkContext, - cache_wheel_server_url: str | None, - req: Requirement, - resolved_version: Version, -) -> tuple[pathlib.Path | None, pathlib.Path | None]: - """Look for cached wheel in 3 locations (thread-safe, no Bootstrapper state). - - Checks for cached wheels in order: - 1. wheels_build directory (previously built) - 2. wheels_downloads directory (previously downloaded) - 3. Cache server (remote cache) - - Returns: - Tuple of (cached_wheel_filename, unpacked_cached_wheel). - Both None if no cache hit. - """ - cached_wheel, unpacked = _look_for_existing_wheel( - ctx, req, resolved_version, ctx.wheels_build - ) - if cached_wheel: - return cached_wheel, unpacked - - cached_wheel, unpacked = _look_for_existing_wheel( - ctx, req, resolved_version, ctx.wheels_downloads - ) - if cached_wheel: - return cached_wheel, unpacked - - cached_wheel, unpacked = _download_wheel_from_cache( - ctx, cache_wheel_server_url, req, resolved_version - ) - if cached_wheel: - return cached_wheel, unpacked - - return None, None - - -def _bg_resolve( - bg_resolver: bootstrap_requirement_resolver.BootstrapRequirementResolver, - req: Requirement, - req_type: RequirementType, - parent_req: Requirement | None, - return_all_versions: bool, -) -> list[tuple[str, Version]]: - """Background-safe resolution: no Bootstrapper state accessed.""" - logger.info(f"{BootstrapPhase.RESOLVE} for {req_type} requirement") - return bg_resolver.resolve( - req=req, - req_type=req_type, - parent_req=parent_req, - return_all_versions=return_all_versions, - ) - - -def _bg_prepare_source( - ctx: context.WorkContext, - cache_wheel_server_url: str | None, - req: Requirement, - resolved_version: Version, - source_url: str, -) -> PreparedSourceData: - """Background-safe source download+unpack: no Bootstrapper state accessed.""" - # Thread-safe: _seen_requirements in the main thread prevents the same - # package from being submitted to the thread pool more than once. - # Paths from download_source() and prepare_source() include {name}-{version}, - # making them unique across concurrent threads processing different packages. - logger.info("preparing source") - cached_wheel, unpacked = _find_cached_wheel( - ctx, cache_wheel_server_url, req, resolved_version - ) - if unpacked is not None: - return PreparedSourceData( - sdist_root_dir=unpacked / unpacked.stem, - cached_wheel_filename=cached_wheel, - ) - source_filename = sources.download_source( - ctx=ctx, req=req, version=resolved_version, download_url=source_url - ) - sdist_root_dir = sources.prepare_source( - ctx=ctx, req=req, source_filename=source_filename, version=resolved_version - ) - return PreparedSourceData(sdist_root_dir=sdist_root_dir) - - -def _bg_prepare_prebuilt( - ctx: context.WorkContext, - req: Requirement, - req_type: RequirementType, - resolved_version: Version, - wheel_url: str, -) -> PreparedSourceData: - """Background-safe prebuilt download: no Bootstrapper state accessed.""" - # Thread-safe: paths include {req.name}-{resolved_version} (unique per package), - # mkdir uses exist_ok=True (atomic), and update_wheel_mirror() is already locked. - logger.info(f"using pre-built wheel for {req_type} requirement") - wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt) - unpack_dir = ctx.work_dir / f"{req.name}-{resolved_version}" - unpack_dir.mkdir(parents=True, exist_ok=True) - server.update_wheel_mirror(ctx) - return PreparedSourceData(wheel_filename=wheel_filename, unpack_dir=unpack_dir) - - -class Bootstrapper: - def __init__( - self, - ctx: context.WorkContext, - progressbar: progress.Progressbar | None = None, - prev_graph: DependencyGraph | None = None, - cache_wheel_server_url: str | None = None, - sdist_only: bool = False, - test_mode: bool = False, - multiple_versions: bool = False, - num_bg_threads: int = _DEFAULT_BG_THREADS, - ) -> None: - if test_mode and sdist_only: - raise ValueError( - "--test-mode requires full wheel builds; incompatible with --sdist-only" - ) - - self.ctx = ctx - self.progressbar = progressbar or progress.Progressbar(None) - self.prev_graph = prev_graph - self.cache_wheel_server_url = cache_wheel_server_url or ctx.wheel_server_url - self.sdist_only = sdist_only - self.test_mode = test_mode - self.multiple_versions = multiple_versions - self.why: list[tuple[RequirementType, Requirement, Version]] = [] - self._num_bg_threads = max(1, num_bg_threads) - self._bg_pool: concurrent.futures.ThreadPoolExecutor | None = ( - concurrent.futures.ThreadPoolExecutor( - max_workers=self._num_bg_threads, thread_name_prefix="fromager-bg" - ) - ) - - # Delegate resolution to BootstrapRequirementResolver - self._resolver = bootstrap_requirement_resolver.BootstrapRequirementResolver( - ctx=ctx, - prev_graph=prev_graph, - multiple_versions=multiple_versions, - cache_wheel_server_url=self.cache_wheel_server_url, - ) - # Push items onto the stack as we start to resolve their - # dependencies so at the end we have a list of items that need to - # be built in order. - self._build_stack: list[typing.Any] = [] - self._build_requirements: set[tuple[NormalizedName, str]] = set() - - # Track requirements we've seen before so we don't resolve the - # same dependencies over and over and so we can break cycles in - # the dependency list. The key is the requirements spec, rather - # than the package, in case we do have multiple rules for the same - # package. - self._seen_requirements: set[SeenKey] = set() - - self._build_order_filename = self.ctx.work_dir / "build-order.json" - self._stack_filename = self.ctx.work_dir / "bootstrap-stack.json" - logger.info("recording bootstrap stack state to %s", self._stack_filename) - - # Track failed packages in test mode (list of typed dicts for JSON export) - self.failed_packages: list[FailureRecord] = [] - - # Track failed versions in multiple_versions mode - self._failed_versions: dict[tuple[str, str], Exception] = {} - - def _resolve_and_add_top_level( - self, - req: Requirement, - ) -> tuple[str, Version] | None: - """Resolve a top-level requirement and add it to the dependency graph. - - Private method called only by ``bootstrap()``. - - This is the pre-resolution phase before recursive bootstrapping begins. - In test mode, catches resolution errors and records them as failures. - - When multiple_versions is enabled, resolves and adds all matching versions - to the graph, but still returns only the first (highest) version for - backward compatibility. - - Args: - req: The top-level requirement to resolve. - - Returns: - Tuple of (source_url, version) if resolution succeeded, None if it - failed in test mode. - - Raises: - Exception: In normal mode, re-raises any resolution error. - """ - try: - pbi = self.ctx.package_build_info(req) - results = self.resolve_versions( - req=req, - req_type=RequirementType.TOP_LEVEL, - parent_req=None, - return_all_versions=self.multiple_versions, - ) - if self.multiple_versions: - logger.info(f"resolved {len(results)} version(s) for {req}") - - # Add all resolved versions to the graph - for source_url, version in results: - logger.info("%s resolves to %s", req, version) - self.ctx.dependency_graph.add_dependency( - parent_name=None, - parent_version=None, - req_type=RequirementType.TOP_LEVEL, - req=req, - req_version=version, - download_url=source_url, - pre_built=pbi.pre_built, - constraint=self.ctx.constraints.get_constraint(req.name), - ) - - if not results: - if self.multiple_versions: - err = RuntimeError(f"no versions found for {req}") - self._record_failed_version( - req, "unresolved", err, "no versions resolved, skipping" - ) - if self.test_mode: - self._record_test_mode_failure(req, None, err, "resolution") - return None - raise RuntimeError(f"Could not resolve any versions for {req}") - - # Return first result for backward compatibility - return results[0] - except Exception as err: - if self.multiple_versions: - self._record_failed_version(req, "unresolved", err, "failed to resolve") - if self.test_mode: - self._record_test_mode_failure(req, None, err, "resolution") - return None - if not self.test_mode: - raise - self._record_test_mode_failure(req, None, err, "resolution") - return None - - def resolve_versions( - self, - req: Requirement, - req_type: RequirementType, - parent_req: Requirement | None = None, - return_all_versions: bool = False, - ) -> list[tuple[str, Version]]: - """Resolve version(s) of a requirement. - - Returns list of (source URL, version) tuples, sorted by version (highest first). - - Git URL resolution stays in Bootstrapper because it requires - build orchestration (BuildEnvironment, build dependencies). - Delegates PyPI/graph resolution to BootstrapRequirementResolver. - - Args: - req: Package requirement to resolve - req_type: Type of requirement - parent_req: Explicit parent requirement from dependency chain. - Callers must pass this explicitly; do not read ``self.why`` here. - return_all_versions: If True, return all matching versions. - If False, return only highest version. - - Returns: - List of (url, version) tuples. Contains one item when - return_all_versions=False, or all matching versions when True. - """ - if req.url: - if req_type != RequirementType.TOP_LEVEL: - raise ValueError( - f"{req} includes a URL, but is not a top-level dependency" - ) - - # Check cache first to avoid re-resolving - # Git URLs are always source (not prebuilt) - cached_result = self._resolver.get_matching_versions(req, pre_built=False) - if cached_result: - logger.debug(f"resolved {req} from cache") - return cached_result if return_all_versions else [cached_result[0]] - - logger.info("resolving source via URL, ignoring any plugins") - source_url, resolved_version = self._resolve_version_from_git_url(req=req) - # Cache the git URL resolution (always source, not prebuilt) - # Store as list for consistency with cache structure - result = [(source_url, resolved_version)] - self._resolver.extend_known_versions(req, pre_built=False, result=result) - return result # Git URLs always return single version - - # Delegate to RequirementResolver - return self._resolver.resolve( - req=req, - req_type=req_type, - parent_req=parent_req, - return_all_versions=return_all_versions, - ) - - def bootstrap(self, requirements: list[Requirement]) -> None: - """Bootstrap all top-level requirements and their transitive dependencies. - - .. versionadded:: 0.89 - Replaces the former ``bootstrap(req, req_type)`` signature. - - Resolves each requirement, adds it to the dependency graph, and processes - the full dependency tree using an iterative DFS loop. Handles - ``requirement_ctxvar`` context internally; callers do not need to manage it. - - In test mode, records failures and continues instead of raising. In - ``multiple_versions`` mode, processes all matching versions per requirement. - - Args: - requirements: Top-level requirements to resolve and bootstrap. - """ - # Resolve all top-level reqs and build initial stack. - # Use the token pattern (no try/finally) so that if resolution raises - # in normal mode, the context var stays set for the top-level error - # handler in __main__.py to include the package name in its log message. - stack: list[WorkItem] = [] - initial_items: list[WorkItem] = [] - for req in requirements: - token = requirement_ctxvar.set(req) - result = self._resolve_and_add_top_level(req) - requirement_ctxvar.reset(token) - if result is not None: - initial_items.append( - WorkItem( - req=req, - req_type=RequirementType.TOP_LEVEL, - phase=BootstrapPhase.RESOLVE, - why_snapshot=[], - parent=None, - ) - ) - self._push_items(stack, initial_items) - - self._run_bootstrap_loop(stack) - - def _run_bootstrap_loop(self, stack: list[WorkItem]) -> None: - """Run the iterative DFS bootstrap loop over a pre-built work stack. - - Pops items one at a time, dispatches each phase, and pushes any - follow-on items (continuations and new dependencies) back onto the - stack. Updates the progress bar as items complete. - - Args: - stack: Initial list of ``WorkItem`` objects to process. Modified - in-place; empty on return. - """ - while stack: - self._record_stack_state(stack) - item = stack.pop() - self.why = list(item.why_snapshot) - - with ( - req_ctxvar_context(item.req, item.resolved_version), - self._track_why(item), - ): - try: - new_items = self._dispatch_phase(item) - except Exception as err: - new_items = self._handle_phase_error(item, err) - - new_dep_count = sum( - 1 for it in new_items if it.phase == BootstrapPhase.RESOLVE - ) - if new_dep_count > 0: - self.progressbar.update_total(new_dep_count) - if not new_items: - self.progressbar.update() - - self._push_items(stack, new_items) - - def _processing_build_requirement(self, current_req_type: RequirementType) -> bool: - """Are we currently processing a build requirement? - - We determine that a package is a build dependency if its requirement - type is build_system, build_backend, or build_sdist OR if it is an - installation requirement of something that is a build dependency. We - use a verbose loop to determine the status so we can log the reason - something is treated as a build dependency. - """ - if current_req_type.is_build_requirement: - logger.debug(f"is itself a build requirement: {current_req_type}") - return True - if not current_req_type.is_install_requirement: - logger.debug( - "is not an install requirement, not checking dependency chain for a build requirement" - ) - return False - for req_type, req, resolved_version in reversed(self.why): - if req_type.is_build_requirement: - logger.debug( - f"is a build requirement because {req_type} dependency {req} ({resolved_version}) depends on it" - ) - return True - logger.debug("is not a build requirement") - return False - - def _bootstrap_one(self, req: Requirement, req_type: RequirementType) -> None: - """Bootstrap a single requirement using an iterative DFS loop. - - Internal method used only by the git URL resolution path - (``_handle_build_requirements``). All other callers should use - ``bootstrap(requirements)`` instead. - - Uses an explicit LIFO stack instead of recursion to handle arbitrarily - deep dependency graphs without hitting Python's recursion limit. - - In test mode, catches build exceptions, records package name, and continues. - In normal mode, raises exceptions immediately (fail-fast). - - When multiple_versions is enabled, bootstraps all matching versions instead - of just the highest version. - """ - logger.info(f"bootstrapping {req} as {req_type} dependency of {self.why[-1:]}") - - # Capture parent from current why stack before creating work items - parent: tuple[Requirement, Version] | None = None - if self.why: - _, parent_req, parent_version = self.why[-1] - parent = (parent_req, parent_version) - - # Save the why stack so we can restore it after the iterative loop - # (the loop modifies self.why for each work item) - saved_why = list(self.why) - - # Single RESOLVE item — resolution, version expansion, and error - # handling all happen inside the loop via _phase_resolve. - initial_item = WorkItem( - req=req, - req_type=req_type, - phase=BootstrapPhase.RESOLVE, - why_snapshot=list(self.why), - parent=parent, - ) - stack: list[WorkItem] = [] - self._push_items(stack, [initial_item]) - - self._run_bootstrap_loop(stack) - - # empty the stack state file to show watchers are are done - # with bootstrapping - self._record_stack_state([]) - - # Restore why stack for the caller - self.why = saved_why - - @contextlib.contextmanager - def _track_why( - self, - item: WorkItem, - ) -> typing.Generator[None, None, None]: - """Context manager to track dependency chain in self.why stack. - - No-op for phases where tracks_why is False (RESOLVE and START). - For all other phases, pushes the item onto the why stack and - ensures it is popped even if an exception occurs. - """ - if not item.phase.tracks_why: - yield - return - assert item.resolved_version is not None - self.why.append((item.req_type, item.req, item.resolved_version)) - try: - yield - finally: - self.why.pop() - - def _record_test_mode_failure( - self, - req: Requirement, - version: str | None, - err: Exception, - failure_type: FailureType, - log_level: typing.Literal["error", "warning"] = "error", - ) -> None: - """Record a failure in test mode. Call this after checking test_mode. - - Args: - req: The requirement that failed. - version: The version being processed (None if not yet resolved). - err: The exception that was raised. - failure_type: Category of failure for analysis. - log_level: Log at error (fatal) or warning (non-fatal, continuing). - """ - version_str = f"=={version}" if version else "" - msg = f"test mode: {failure_type} failed for {req.name}{version_str}" - if log_level == "warning": - logger.warning("%s: %s (continuing)", msg, err) - else: - logger.error("%s: %s", msg, err, exc_info=True) - - self.failed_packages.append( - { - "package": str(req.name), - "version": version, - "exception_type": err.__class__.__name__, - "exception_message": str(err), - "failure_type": failure_type, - } - ) - - @property - def _explain(self) -> str: - """Return message formatting current version of why stack.""" - return " for ".join( - f"{req_type} dependency {req} ({resolved_version})" - for req_type, req, resolved_version in reversed(self.why) - ) - - def _build_sdist( - self, - req: Requirement, - resolved_version: Version, - sdist_root_dir: pathlib.Path, - build_env: build_environment.BuildEnvironment, - ) -> pathlib.Path: - find_sdist_result = finders.find_sdist( - self.ctx, self.ctx.sdists_builds, req, str(resolved_version) - ) - if not find_sdist_result: - sdist_filename: pathlib.Path = sources.build_sdist( - ctx=self.ctx, - req=req, - version=resolved_version, - sdist_root_dir=sdist_root_dir, - build_env=build_env, - ) - else: - sdist_filename = find_sdist_result - logger.info(f"have sdist version {resolved_version}: {find_sdist_result}") - return sdist_filename - - def _build_wheel( - self, - req: Requirement, - resolved_version: Version, - sdist_root_dir: pathlib.Path, - build_env: build_environment.BuildEnvironment, - ) -> tuple[pathlib.Path, pathlib.Path]: - sdist_filename = self._build_sdist( - req, resolved_version, sdist_root_dir, build_env - ) - - logger.info(f"starting build of {self._explain} for {self.ctx.variant}") - built_filename = wheels.build_wheel( - ctx=self.ctx, - req=req, - sdist_root_dir=sdist_root_dir, - version=resolved_version, - build_env=build_env, - ) - server.update_wheel_mirror(self.ctx) - # When we update the mirror, the built file moves to the - # downloads directory. - wheel_filename = self.ctx.wheels_downloads / built_filename.name - logger.info(f"built wheel for version {resolved_version}: {wheel_filename}") - return wheel_filename, sdist_filename - - def _prepare_build_dependencies( - self, - req: Requirement, - resolved_version: Version | None, - sdist_root_dir: pathlib.Path, - build_env: build_environment.BuildEnvironment, - ) -> set[Requirement]: - """Prepare build dependencies for a package. - - Only used by the git URL resolution path - (_resolve_version_from_git_url -> _get_version_from_package_metadata). - The main iterative bootstrap loop handles build deps via phase handlers. - """ - # build system - build_system_dependencies = dependencies.get_build_system_dependencies( - ctx=self.ctx, - req=req, - version=resolved_version, - sdist_root_dir=sdist_root_dir, - ) - self._handle_build_requirements( - req, - RequirementType.BUILD_SYSTEM, - build_system_dependencies, - ) - # The next hooks need build system requirements. - build_env.install(build_system_dependencies) - - # build backend - build_backend_dependencies = dependencies.get_build_backend_dependencies( - ctx=self.ctx, - req=req, - version=resolved_version, - sdist_root_dir=sdist_root_dir, - build_env=build_env, - ) - - # build sdist - build_sdist_dependencies = dependencies.get_build_sdist_dependencies( - ctx=self.ctx, - req=req, - version=resolved_version, - sdist_root_dir=sdist_root_dir, - build_env=build_env, - ) - - # Filter out deps already satisfied by build-system dependencies - resolved_build_sys = self._resolve_build_system_versions_by_name( - build_system_dependencies, - ) - build_backend_dependencies = self._filter_deps_satisfied_by_build_system( - build_backend_dependencies, - resolved_build_sys, - RequirementType.BUILD_BACKEND, - ) - build_sdist_dependencies = self._filter_deps_satisfied_by_build_system( - build_sdist_dependencies, - resolved_build_sys, - RequirementType.BUILD_SDIST, - ) - - self._handle_build_requirements( - req, - RequirementType.BUILD_BACKEND, - build_backend_dependencies, - ) - self._handle_build_requirements( - req, - RequirementType.BUILD_SDIST, - build_sdist_dependencies, - ) - - build_dependencies = build_sdist_dependencies | build_backend_dependencies - if build_dependencies.isdisjoint(build_system_dependencies): - build_env.install(build_dependencies) - - return ( - build_system_dependencies - | build_backend_dependencies - | build_sdist_dependencies - ) - - def _handle_build_requirements( - self, - req: Requirement, - build_type: RequirementType, - build_dependencies: set[Requirement], - ) -> None: - """Bootstrap build dependencies. - - Only used by the git URL resolution path - (_resolve_version_from_git_url -> _get_version_from_package_metadata). - The main iterative bootstrap loop handles build deps via phase handlers. - """ - self.progressbar.update_total(len(build_dependencies)) - - for dep in self._sort_requirements(build_dependencies): - with req_ctxvar_context(dep): - # Save/restore self.why because the iterative bootstrap() - # modifies it internally for each work item. - saved_why = list(self.why) - self._bootstrap_one(req=dep, req_type=build_type) - self.why = saved_why - self.progressbar.update() - - def _download_prebuilt( - self, - req: Requirement, - req_type: RequirementType, - resolved_version: Version, - wheel_url: str, - ) -> tuple[pathlib.Path, pathlib.Path]: - result = _bg_prepare_prebuilt( - self.ctx, req, req_type, resolved_version, wheel_url - ) - assert result.wheel_filename is not None - assert result.unpack_dir is not None - return (result.wheel_filename, result.unpack_dir) - - def _get_install_dependencies( - self, - req: Requirement, - resolved_version: Version, - wheel_filename: pathlib.Path | None, - sdist_filename: pathlib.Path | None, - sdist_root_dir: pathlib.Path | None, - build_env: build_environment.BuildEnvironment | None, - unpack_dir: pathlib.Path | None, - ) -> list[Requirement]: - """Extract install dependencies from wheel or sdist. - - Returns: - List of install requirements. - - Raises: - RuntimeError: If both wheel_filename and sdist_filename are None. - """ - if wheel_filename is not None: - assert unpack_dir is not None - logger.debug( - "get install dependencies of wheel %s", - wheel_filename.name, - ) - return list( - dependencies.get_install_dependencies_of_wheel( - req=req, - wheel_filename=wheel_filename, - requirements_file_dir=unpack_dir, - ) - ) - elif sdist_filename is not None: - assert sdist_root_dir is not None - assert build_env is not None - logger.debug( - "get install dependencies of sdist from directory %s", - sdist_root_dir, - ) - return list( - dependencies.get_install_dependencies_of_sdist( - ctx=self.ctx, - req=req, - version=resolved_version, - sdist_root_dir=sdist_root_dir, - build_env=build_env, - ) - ) - else: - raise RuntimeError("wheel_filename and sdist_filename are None") - - def _download_source( - self, - req: Requirement, - resolved_version: Version, - source_url: str, - ) -> pathlib.Path: - """Download source for a package.""" - result: pathlib.Path = sources.download_source( - ctx=self.ctx, - req=req, - version=resolved_version, - download_url=source_url, - ) - return result - - def _prepare_source( - self, - req: Requirement, - resolved_version: Version, - source_filename: pathlib.Path, - ) -> pathlib.Path: - """Prepare (unpack/patch) source for building.""" - result: pathlib.Path = sources.prepare_source( - ctx=self.ctx, - req=req, - source_filename=source_filename, - version=resolved_version, - ) - return result - - def _create_build_env( - self, - req: Requirement, - resolved_version: Version, - parent_dir: pathlib.Path, - ) -> build_environment.BuildEnvironment: - """Create isolated build environment.""" - return build_environment.BuildEnvironment( - ctx=self.ctx, - parent_dir=parent_dir, - ) - - def _do_build( - self, - req: Requirement, - resolved_version: Version, - sdist_root_dir: pathlib.Path, - build_env: build_environment.BuildEnvironment, - build_sdist_only: bool, - cached_wheel_filename: pathlib.Path | None, - ) -> tuple[pathlib.Path | None, pathlib.Path | None]: - """Build wheel or sdist from prepared source.""" - if cached_wheel_filename: - logger.debug( - f"getting install requirements from cached wheel {cached_wheel_filename.name}" - ) - return cached_wheel_filename, None - elif build_sdist_only: - logger.debug( - f"getting install requirements from sdist {req.name}=={resolved_version}" - ) - return None, self._build_sdist( - req, resolved_version, sdist_root_dir, build_env - ) - else: - logger.debug( - f"building wheel {req.name}=={resolved_version} to get install requirements" - ) - return self._build_wheel(req, resolved_version, sdist_root_dir, build_env) - - def _handle_test_mode_failure( - self, - req: Requirement, - resolved_version: Version, - req_type: RequirementType, - build_error: Exception, - ) -> SourceBuildResult | None: - """Handle build failure in test mode by attempting pre-built fallback. - - Args: - req: The requirement that failed to build. - resolved_version: The version that was attempted. - req_type: The type of requirement (for fallback resolution). - build_error: The original exception from the build attempt. - - Returns: - SourceBuildResult if fallback succeeded, None if fallback also failed. - """ - logger.warning( - "test mode: build failed for %s==%s, attempting pre-built fallback: %s", - req.name, - resolved_version, - build_error, - ) - - try: - parent_req = self.why[-1][1] if self.why else None - results = self._resolver.resolve( - req=req, - req_type=req_type, - parent_req=parent_req, - pre_built=True, # Force prebuilt for test mode fallback - ) - wheel_url, fallback_version = results[0] - - if fallback_version != resolved_version: - logger.warning( - "test mode: version mismatch for %s - requested %s, fallback %s", - req.name, - resolved_version, - fallback_version, - ) - - wheel_filename, unpack_dir = self._download_prebuilt( - req=req, - req_type=req_type, - resolved_version=fallback_version, - wheel_url=wheel_url, - ) - - logger.info( - "test mode: successfully used pre-built wheel for %s==%s", - req.name, - fallback_version, - ) - # Package succeeded via fallback - no failure to record - - return SourceBuildResult( - wheel_filename=wheel_filename, - sdist_filename=None, - unpack_dir=unpack_dir, - sdist_root_dir=None, - build_env=None, - source_type=SourceType.PREBUILT, - ) - - except Exception as fallback_error: - logger.error( - "test mode: pre-built fallback also failed for %s: %s", - req.name, - fallback_error, - exc_info=True, - ) - # Return None to signal failure; bootstrap() will record via re-raised exception - return None - - def _resolve_version_from_git_url(self, req: Requirement) -> tuple[str, Version]: - """Resolve source path and version from a ``git+`` URL. - - Parses the URL for an ``@ref`` version hint. If the ref is a valid - version, reuses an existing clone when possible. Otherwise, clones - the repo and extracts the version from package metadata. - """ - - if not req.url: - raise ValueError(f"unable to resolve from URL with no URL in {req}") - - # We start by not knowing where we would put the source because we don't - # know the version. - working_src_dir: pathlib.Path | None = None - version: Version | None = None - - url_to_clone, git_ref = gitutils.parse_vcs_url(req.url, require_ref=False) - need_to_clone = False - - if git_ref == gitutils.GIT_HEAD: - # No ref in URL, clone to discover the version. - logger.debug("no reference in URL, will clone") - need_to_clone = True - else: - # If we have a reference, it might be a valid python version - # string, or not. It _must_ be a valid git reference. If it can - # be parsed as a valid python version, we assume the tag points - # to source that will think that is its version, so we allow - # reusing an existing cloned repo if there is one. - try: - version = Version(git_ref) - except ValueError: - logger.info( - "could not parse %r as a version, cloning to get the version", - git_ref, - ) - need_to_clone = True - else: - logger.info("URL %s includes version %s", req.url, version) - working_src_dir = ( - self.ctx.work_dir - / f"{req.name}-{version}" - / f"{req.name}-{version}" - ) - if not working_src_dir.exists(): - need_to_clone = True - else: - if self.ctx.cleanup: - logger.debug("cleaning up %s to reclone", working_src_dir) - shutil.rmtree(working_src_dir) - need_to_clone = True - else: - logger.info("reusing %s", working_src_dir) - - if need_to_clone: - with tempfile.TemporaryDirectory() as tmpdir: - clone_dir = pathlib.Path(tmpdir) / "src" - sources.download_git_source( - ctx=self.ctx, - req=req, - url_to_clone=url_to_clone, - destination_dir=clone_dir, - ref=git_ref, - ) - if not version: - # If we still do not have a version, get it from the package - # metadata. - version = self._get_version_from_package_metadata(req, clone_dir) - logger.info("found version %s", version) - working_src_dir = ( - self.ctx.work_dir - / f"{req.name}-{version}" - / f"{req.name}-{version}" - ) - if working_src_dir.exists(): - # We have to check if the destination directory exists - # because if we were not given a version we did not - # clean it up earlier. We do not use ctx.cleanup to - # control this action because we cannot trust that the - # destination directory is reusable because we have had - # to compute the version and we cannot be sure that the - # version is dynamic. Two different commits in the repo - # could have the same version if that version is set - # with static data in the repo instead of via a tag or - # dynamically computed by something like setuptools-scm. - logger.debug("cleaning up %s", working_src_dir) - shutil.rmtree(working_src_dir) - working_src_dir.parent.mkdir(parents=True, exist_ok=True) - logger.info("moving cloned repo to %s", working_src_dir) - shutil.move(clone_dir, str(working_src_dir)) - - if not version: - raise ValueError(f"unable to determine version for {req}") - - if not working_src_dir: - raise ValueError(f"unable to determine working source directory for {req}") - - logging.info("resolved from git URL to %s, %s", working_src_dir, version) - return (str(working_src_dir), version) - - def _get_version_from_package_metadata( - self, - req: Requirement, - source_dir: pathlib.Path, - ) -> Version: - pbi = self.ctx.package_build_info(req) - build_dir = pbi.build_dir(source_dir) - - logger.info( - "preparing build dependencies so we can access the metadata to get the version" - ) - build_env = build_environment.BuildEnvironment( - ctx=self.ctx, - parent_dir=source_dir.parent, - ) - build_dependencies = self._prepare_build_dependencies( - req=req, - resolved_version=None, - sdist_root_dir=source_dir, - build_env=build_env, - ) - build_env.install(build_dependencies) - - logger.info("generating metadata to get version") - hook_caller = dependencies.get_build_backend_hook_caller( - ctx=self.ctx, - req=req, - build_dir=build_dir, - override_environ={}, - build_env=build_env, - ) - metadata_dir_base = hook_caller.prepare_metadata_for_build_wheel( - metadata_directory=str(source_dir.parent), - config_settings=pbi.config_settings, - ) - metadata_filename = source_dir.parent / metadata_dir_base / "METADATA" - # Disable validation because some packages have metadata version mismatches - # (e.g., declaring Metadata-Version: 2.2 but using fields from 2.4). - metadata = dependencies.parse_metadata(metadata_filename, validate=False) - return metadata.version - - def _add_to_graph( - self, - req: Requirement, - req_type: RequirementType, - req_version: Version, - download_url: str, - parent: tuple[Requirement, Version] | None, - ) -> None: - parent_req, parent_version = parent if parent else (None, None) - pbi = self.ctx.package_build_info(req) - # Update the dependency graph after we determine that this requirement is - # useful but before we determine if it is redundant so that we capture all - # edges to use for building a valid constraints file. - self.ctx.dependency_graph.add_dependency( - parent_name=canonicalize_name(parent_req.name) if parent_req else None, - parent_version=parent_version, - req_type=req_type, - req=req, - req_version=req_version, - download_url=download_url, - pre_built=pbi.pre_built, - constraint=self.ctx.constraints.get_constraint(req.name), - ) - self.ctx.write_to_graph_to_file() - - def _sort_requirements( - self, - requirements: typing.Iterable[Requirement], - ) -> typing.Iterable[Requirement]: - return sorted(requirements, key=operator.attrgetter("name")) - - def _resolved_key( - self, req: Requirement, version: Version, typ: typing.Literal["sdist", "wheel"] - ) -> SeenKey: - return ( - canonicalize_name(req.name), - tuple(sorted(req.extras)), - str(version), - typ, - ) - - def _mark_as_seen( - self, - req: Requirement, - version: Version, - sdist_only: bool = False, - ) -> None: - """Track sdist and wheel builds - - A sdist-only build just contains as an sdist. - A wheel build counts as wheel and sdist, because the presence of a - either implies we have built a wheel from an sdist or we have a - prebuilt wheel that will never have an sdist. - """ - # Mark sdist seen for sdist-only build and wheel build - self._seen_requirements.add(self._resolved_key(req, version, "sdist")) - if not sdist_only: - # Mark wheel seen only for wheel build - self._seen_requirements.add(self._resolved_key(req, version, "wheel")) - - def _has_been_seen( - self, - req: Requirement, - version: Version, - sdist_only: bool = False, - ) -> bool: - typ: typing.Literal["sdist", "wheel"] = "sdist" if sdist_only else "wheel" - return self._resolved_key(req, version, typ) in self._seen_requirements - - def _add_to_build_order( - self, - req: Requirement, - version: Version, - source_url: str, - source_type: SourceType, - prebuilt: bool = False, - constraint: Requirement | None = None, - ) -> None: - # We only care if this version of this package has been built, - # and don't want to trigger building it twice. The "extras" - # value, included in the _resolved_key() output, can confuse - # that so we ignore itand build our own key using just the - # name and version. - key = (canonicalize_name(req.name), str(version)) - if key in self._build_requirements: - return - logger.info(f"adding {key} to build order") - self._build_requirements.add(key) - info = { - "req": str(req), - "constraint": str(constraint) if constraint else "", - "dist": canonicalize_name(req.name), - "version": str(version), - "prebuilt": prebuilt, - "source_url": source_url, - "source_url_type": str(source_type), - } - if req.url: - info["source_url"] = req.url - self._build_stack.append(info) - with open(self._build_order_filename, "w") as f: - # Set default=str because the why value includes - # Requirement and Version instances that can't be - # converted to JSON without help. - json.dump(self._build_stack, f, indent=2, default=str) - - def _record_stack_state(self, stack: list[WorkItem]) -> None: - """Write the current bootstrap stack to `self._stack_filename`. - - Index 0 in the output corresponds to `stack[-1]`, the next item to be - processed. Overwrites the file on each call. - """ - - def serialize(item: WorkItem) -> dict[str, typing.Any]: - return { - "req": str(item.req), - "req_type": str(item.req_type), - "phase": str(item.phase), - "resolved_version": str(item.resolved_version) - if item.resolved_version is not None - else None, - "source_url": item.source_url, - "build_sdist_only": item.build_sdist_only, - "why": [ - {"req_type": str(rt), "req": str(r), "version": str(v)} - for rt, r, v in item.why_snapshot - ], - "parent": ( - {"req": str(item.parent[0]), "version": str(item.parent[1])} - if item.parent - else None - ), - "build_system_deps": sorted(str(r) for r in item.build_system_deps), - "build_backend_deps": sorted(str(r) for r in item.build_backend_deps), - "build_sdist_deps": sorted(str(r) for r in item.build_sdist_deps), - } - - records = [serialize(item) for item in reversed(stack)] - with open(self._stack_filename, "w") as f: - json.dump(records, f, indent=2, default=str) - - # ---- Iterative bootstrap: phase handlers and helpers ---- - - def _push_items(self, stack: list[WorkItem], items: list[WorkItem]) -> None: - """Push items onto the stack and submit background tasks in LIFO order. - - Submits the item that will be processed first (top of stack) to the - background pool first, maximising overlap between background I/O and - main-thread serial work. - """ - stack.extend(items) - if self._bg_pool is not None: - for item in reversed(items): - bg_work = self._get_background_work(item) - if bg_work is not None: - item.bg_future = self._bg_pool.submit(bg_work) - - def _get_background_work( - self, item: WorkItem - ) -> typing.Callable[[], typing.Any] | None: - """Return a zero-argument callable for background submission, or None. - - Uses closures that capture local variables rather than ``self``, so - the returned callable cannot access mutable ``Bootstrapper`` state. - Each closure sets up ``req_ctxvar_context`` so log messages include - the package name (and version when known). - """ - if item.phase == BootstrapPhase.RESOLVE: - bg_resolver = self._resolver - req = item.req - req_type = item.req_type - parent_req = item.why_snapshot[-1][1] if item.why_snapshot else None - return_all = self.multiple_versions - - def do_resolve() -> list[tuple[str, Version]]: - with req_ctxvar_context(req): - return _bg_resolve( - bg_resolver, req, req_type, parent_req, return_all - ) - - return do_resolve - - if item.phase == BootstrapPhase.PREPARE_SOURCE: - assert item.resolved_version is not None - assert item.source_url is not None - ctx = self.ctx - cache_wheel_server_url = self.cache_wheel_server_url - req = item.req - req_type = item.req_type - resolved_version = item.resolved_version - source_url = item.source_url - - if item.pbi_pre_built: - - def do_prepare_prebuilt() -> PreparedSourceData: - with req_ctxvar_context(req, resolved_version): - return _bg_prepare_prebuilt( - ctx, req, req_type, resolved_version, source_url - ) - - return do_prepare_prebuilt - - def do_prepare_source() -> PreparedSourceData: - with req_ctxvar_context(req, resolved_version): - return _bg_prepare_source( - ctx, cache_wheel_server_url, req, resolved_version, source_url - ) - - return do_prepare_source - - return None - - def _drain_background_pool(self) -> None: - """Drain all in-flight background tasks and recreate the pool. - - Used as an exclusive-build barrier: ensures all background I/O completes - before an exclusive build starts. ``cancel_futures=False`` guarantees - every submitted task runs to completion. - """ - if self._bg_pool is not None: - self._bg_pool.shutdown(wait=True, cancel_futures=False) - self._bg_pool = concurrent.futures.ThreadPoolExecutor( - max_workers=self._num_bg_threads, thread_name_prefix="fromager-bg" - ) - - def _create_unresolved_work_items( - self, - deps: typing.Iterable[Requirement], - dep_req_type: RequirementType, - parent_req: Requirement, - parent_version: Version, - ) -> list[WorkItem]: - """Create RESOLVE-phase work items for dependencies. - - Called inside a parent's _track_why context so that why_snapshot - captures the parent's dependency chain. Resolution and error - handling happen later when each item's RESOLVE phase runs. - """ - return [ - WorkItem( - req=dep, - req_type=dep_req_type, - phase=BootstrapPhase.RESOLVE, - why_snapshot=list(self.why), - parent=(parent_req, parent_version), - ) - for dep in self._sort_requirements(deps) - ] - - def _phase_resolve(self, item: WorkItem) -> list[WorkItem]: - """RESOLVE phase: resolve versions and expand into START-phase items. - - Centralizes version resolution so all dependencies are expanded - uniformly. In multiple_versions mode, filters out versions that - already failed in this run and versions whose wheels are already - cached to avoid redundant builds and transitive dependency - processing. - - Returns: - One START-phase item per resolved version that needs building. - Empty list if all versions are already cached. - """ - assert item.bg_future is not None - # bg_future.result() blocks until the background resolution completes, - # then returns the result or re-raises any exception from the background. - resolved_versions = item.bg_future.result() - if not resolved_versions: - raise RuntimeError(f"Could not resolve any versions for {item.req}") - - if self.multiple_versions: - pkg_name = canonicalize_name(item.req.name) - resolved_versions = [ - (url, ver) - for url, ver in resolved_versions - if (pkg_name, str(ver)) not in self._failed_versions - ] - if not resolved_versions: - raise RuntimeError( - f"Could not resolve any versions for {item.req}" - f" (all candidates failed previously)" - ) - - logger.info(f"resolved {len(resolved_versions)} version(s) for {item.req}") - filtered: list[tuple[str, Version]] = [] - for source_url, version in resolved_versions: - cached_wheel, _ = _find_cached_wheel( - self.ctx, self.cache_wheel_server_url, item.req, version - ) - if cached_wheel: - logger.info( - f"{item.req.name}=={version}: wheel already cached " - f"at {cached_wheel.name}, skipping" - ) - else: - filtered.append((source_url, version)) - if not filtered: - # Always process the highest version (first in - # resolved_versions) so new transitive dependencies - # are discovered even when every wheel is cached. - logger.info( - f"all versions of {item.req.name} already cached, " - f"keeping highest version {resolved_versions[0][1]} " - f"for dependency discovery" - ) - filtered.append(resolved_versions[0]) - resolved_versions = filtered - - # Build list so highest version ends up on top of the stack - # (last element after extend) and is processed first. - items: list[WorkItem] = [] - for source_url, version in reversed(resolved_versions): - items.append( - WorkItem( - req=item.req, - req_type=item.req_type, - phase=BootstrapPhase.START, - why_snapshot=list(item.why_snapshot), - parent=item.parent, - source_url=source_url, - resolved_version=version, - ) - ) - return items - - def _phase_start(self, item: WorkItem) -> list[WorkItem]: - """START phase: add to graph, check if already seen. - - _track_why is a no-op for this phase (tracks_why is False), - matching the original behavior where graph addition and - seen-check happen before pushing onto the why stack. - - Returns: - Empty list if already seen (nothing to do). - [item] advanced to PREPARE_SOURCE if this is new work. - """ - assert item.resolved_version is not None - assert item.source_url is not None - - # Add to graph (skip TOP_LEVEL, already added in _resolve_and_add_top_level) - if item.req_type != RequirementType.TOP_LEVEL: - self._add_to_graph( - item.req, - item.req_type, - item.resolved_version, - item.source_url, - item.parent, - ) - - item.build_sdist_only = ( - self.sdist_only and not self._processing_build_requirement(item.req_type) - ) - - if self._has_been_seen(item.req, item.resolved_version, item.build_sdist_only): - logger.debug( - f"redundant {item.req_type} dependency {item.req} " - f"({item.resolved_version}, sdist_only={item.build_sdist_only}) " - f"for {self._explain}" - ) - return [] - self._mark_as_seen(item.req, item.resolved_version, item.build_sdist_only) - - logger.info( - f"new {item.req_type} dependency {item.req} " - f"resolves to {item.resolved_version}" - ) - - item.pbi_pre_built = self.ctx.package_build_info(item.req).pre_built - item.phase = BootstrapPhase.PREPARE_SOURCE - return [item] - - def _phase_prepare_source(self, item: WorkItem) -> list[WorkItem]: - """PREPARE_SOURCE phase: download source or prebuilt, get build system deps. - - Uses background I/O result from ``item.bg_future`` when available, - falling back to inline I/O otherwise. - - Returns: - Prebuilt: [item] advanced to PROCESS_INSTALL_DEPS (skip build phases). - Source: [item advanced to PREPARE_BUILD, *build_system_dep_items]. - """ - assert item.resolved_version is not None - assert item.source_url is not None - - # bg_future is always set for PREPARE_SOURCE items (see _push_items / _get_background_work). - # bg_future.result() blocks until done and re-raises any background exception. - assert item.bg_future is not None - prepared: PreparedSourceData = item.bg_future.result() - - constraint = self.ctx.constraints.get_constraint(item.req.name) - if constraint: - logger.info( - f"incoming requirement {item.req} matches constraint " - f"{constraint}. Will apply both." - ) - - if item.pbi_pre_built: - # Background task already downloaded the prebuilt wheel - assert prepared.wheel_filename is not None - assert prepared.unpack_dir is not None - item.build_result = SourceBuildResult( - wheel_filename=prepared.wheel_filename, - sdist_filename=None, - unpack_dir=prepared.unpack_dir, - sdist_root_dir=None, - build_env=None, - source_type=SourceType.PREBUILT, - ) - item.phase = BootstrapPhase.PROCESS_INSTALL_DEPS - return [item] - - # Source build path: background task already downloaded and prepared the source - assert prepared.sdist_root_dir is not None - sdist_root_dir = prepared.sdist_root_dir - item.cached_wheel_filename = prepared.cached_wheel_filename - - assert sdist_root_dir is not None - - if sdist_root_dir.parent.parent != self.ctx.work_dir: - raise ValueError(f"'{sdist_root_dir}/../..' should be {self.ctx.work_dir}") - item.sdist_root_dir = sdist_root_dir - item.unpack_dir = sdist_root_dir.parent - - item.build_env = self._create_build_env( - req=item.req, - resolved_version=item.resolved_version, - parent_dir=sdist_root_dir.parent, - ) - - # Get build system dependencies - item.build_system_deps = dependencies.get_build_system_dependencies( - ctx=self.ctx, - req=item.req, - version=item.resolved_version, - sdist_root_dir=sdist_root_dir, - ) - - dep_items = self._create_unresolved_work_items( - item.build_system_deps, - RequirementType.BUILD_SYSTEM, - item.req, - item.resolved_version, - ) - - item.phase = BootstrapPhase.PREPARE_BUILD - return [item] + dep_items - - def _resolve_build_system_versions_by_name( - self, - build_system_deps: set[Requirement], - ) -> dict[NormalizedName, tuple[Version, str]]: - """Build a mapping of resolved build-system versions by looking up graph nodes. - - Used by the git URL path where the parent node may not yet exist - in the dependency graph. - """ - resolved: dict[NormalizedName, tuple[Version, str]] = {} - for dep in build_system_deps: - dep_name = canonicalize_name(dep.name) - nodes = self.ctx.dependency_graph.get_nodes_by_name(str(dep_name)) - for node in nodes: - if node.version in dep.specifier: - resolved[dep_name] = (node.version, node.download_url) - break - return resolved - - def _get_resolved_build_system_versions( - self, - item: WorkItem, - ) -> dict[NormalizedName, tuple[Version, str]]: - """Build a mapping of resolved build-system dependency versions. - - Looks up the parent node's ``BUILD_SYSTEM`` edges in the dependency - graph to find what versions were resolved for each build-system - dependency. - - Returns: - Mapping of canonicalized package name to ``(version, download_url)``. - """ - assert item.resolved_version is not None - parent_key = f"{canonicalize_name(item.req.name)}=={item.resolved_version}" - parent_node = self.ctx.dependency_graph.nodes.get(parent_key) - if parent_node is None: - return {} - resolved: dict[NormalizedName, tuple[Version, str]] = {} - for edge in parent_node.children: - if edge.req_type == RequirementType.BUILD_SYSTEM: - child = edge.destination_node - resolved[child.canonicalized_name] = ( - child.version, - child.download_url, - ) - return resolved - - def _filter_deps_satisfied_by_build_system( - self, - deps: set[Requirement], - resolved_build_sys: dict[NormalizedName, tuple[Version, str]], - dep_req_type: RequirementType, - parent: tuple[Requirement, Version] | None = None, - ) -> set[Requirement]: - """Filter out deps already satisfied by build-system dependencies. - - For each dep whose resolved build-system version satisfies the - requirement specifier, excludes the dep from the returned set. - When *parent* is provided, also adds a graph edge reusing that - version. Remaining deps need independent resolution. - - Logs a warning when the same package appears in both build-system - and build-backend/sdist with incompatible version specifiers. - """ - unsatisfied: set[Requirement] = set() - for dep in deps: - if dep.extras: - unsatisfied.add(dep) - continue - dep_name = canonicalize_name(dep.name) - if dep_name in resolved_build_sys: - version, download_url = resolved_build_sys[dep_name] - if version in dep.specifier: - logger.info( - "%s dependency %s is already satisfied by " - "build-system dependency %s==%s", - dep_req_type, - dep, - dep_name, - version, - ) - if parent is not None: - self._add_to_graph( - req=dep, - req_type=dep_req_type, - req_version=version, - download_url=download_url, - parent=parent, - ) - continue - else: - logger.warning( - "%s dependency %s conflicts with " - "build-system dependency %s==%s; " - "resolving independently", - dep_req_type, - dep, - dep_name, - version, - ) - unsatisfied.add(dep) - return unsatisfied - - def _phase_prepare_build(self, item: WorkItem) -> list[WorkItem]: - """PREPARE_BUILD phase: install system deps, get backend/sdist deps. - - Build-backend and build-sdist dependencies that are already satisfied - by a resolved build-system dependency reuse that version instead of - resolving independently (see :issue:`1194`). - - Returns: - [item advanced to BUILD, *backend_dep_items, *sdist_dep_items]. - """ - assert item.resolved_version is not None - assert item.build_env is not None - assert item.sdist_root_dir is not None - - # Install build system deps (their wheels exist from DFS processing) - item.build_env.install(item.build_system_deps) - - # Get build backend dependencies - item.build_backend_deps = dependencies.get_build_backend_dependencies( - ctx=self.ctx, - req=item.req, - version=item.resolved_version, - sdist_root_dir=item.sdist_root_dir, - build_env=item.build_env, - ) - - # Get build sdist dependencies - item.build_sdist_deps = dependencies.get_build_sdist_dependencies( - ctx=self.ctx, - req=item.req, - version=item.resolved_version, - sdist_root_dir=item.sdist_root_dir, - build_env=item.build_env, - ) - - # Filter out deps already satisfied by build-system dependencies - # to avoid resolving to a different (typically newer) version. - resolved_build_sys = self._get_resolved_build_system_versions(item) - parent = (item.req, item.resolved_version) - item.build_backend_deps = self._filter_deps_satisfied_by_build_system( - item.build_backend_deps, - resolved_build_sys, - RequirementType.BUILD_BACKEND, - parent, - ) - item.build_sdist_deps = self._filter_deps_satisfied_by_build_system( - item.build_sdist_deps, - resolved_build_sys, - RequirementType.BUILD_SDIST, - parent, - ) - - backend_items = self._create_unresolved_work_items( - item.build_backend_deps, - RequirementType.BUILD_BACKEND, - item.req, - item.resolved_version, - ) - sdist_items = self._create_unresolved_work_items( - item.build_sdist_deps, - RequirementType.BUILD_SDIST, - item.req, - item.resolved_version, - ) - dep_items = backend_items + sdist_items - - item.phase = BootstrapPhase.BUILD - return [item] + dep_items - - def _phase_build(self, item: WorkItem) -> list[WorkItem]: - """BUILD phase: install remaining deps, build wheel/sdist. - - Returns: - [item] advanced to PROCESS_INSTALL_DEPS. - """ - assert item.resolved_version is not None - assert item.build_env is not None - assert item.sdist_root_dir is not None - - # Drain all in-flight background I/O before an exclusive build starts. - # This ensures no parallel background tasks interfere with a build that - # must run without resource contention. - pbi = self.ctx.package_build_info(item.req) - if pbi.exclusive_build: - logger.info( - "%s requires exclusive build, draining background pool", item.req - ) - self._drain_background_pool() - - # Install backend+sdist deps if disjoint from system deps - remaining_deps = item.build_backend_deps | item.build_sdist_deps - if remaining_deps.isdisjoint(item.build_system_deps): - item.build_env.install(remaining_deps) - - wheel_filename, sdist_filename = self._do_build( - req=item.req, - resolved_version=item.resolved_version, - sdist_root_dir=item.sdist_root_dir, - build_env=item.build_env, - build_sdist_only=item.build_sdist_only, - cached_wheel_filename=item.cached_wheel_filename, - ) - - source_type = sources.get_source_type(self.ctx, item.req) - - item.build_result = SourceBuildResult( - wheel_filename=wheel_filename, - sdist_filename=sdist_filename, - unpack_dir=item.sdist_root_dir.parent, - sdist_root_dir=item.sdist_root_dir, - build_env=item.build_env, - source_type=source_type, - ) - - item.phase = BootstrapPhase.PROCESS_INSTALL_DEPS - return [item] - - def _phase_process_install_deps(self, item: WorkItem) -> list[WorkItem]: - """PROCESS_INSTALL_DEPS phase: hooks, extract deps, build order. - - Returns: - [item advanced to COMPLETE, *install_dep_items]. - """ - assert item.resolved_version is not None - assert item.source_url is not None - assert item.build_result is not None - - # Run post-bootstrap hooks (non-fatal in test mode) - try: - hooks.run_post_bootstrap_hooks( - ctx=self.ctx, - req=item.req, - dist_name=canonicalize_name(item.req.name), - dist_version=str(item.resolved_version), - sdist_filename=item.build_result.sdist_filename, - wheel_filename=item.build_result.wheel_filename, - ) - except Exception as hook_error: - if not self.test_mode: - raise - self._record_test_mode_failure( - item.req, - str(item.resolved_version), - hook_error, - "hook", - "warning", - ) - - # Extract install dependencies (non-fatal in test mode) - try: - install_dependencies = self._get_install_dependencies( - req=item.req, - resolved_version=item.resolved_version, - wheel_filename=item.build_result.wheel_filename, - sdist_filename=item.build_result.sdist_filename, - sdist_root_dir=item.build_result.sdist_root_dir, - build_env=item.build_result.build_env, - unpack_dir=item.build_result.unpack_dir, - ) - except Exception as dep_error: - if not self.test_mode: - raise - self._record_test_mode_failure( - item.req, - str(item.resolved_version), - dep_error, - "dependency_extraction", - "warning", - ) - install_dependencies = [] - - logger.debug( - "install dependencies: %s", - ", ".join(sorted(str(r) for r in install_dependencies)), - ) - - pbi = self.ctx.package_build_info(item.req) - constraint = self.ctx.constraints.get_constraint(item.req.name) - self._add_to_build_order( - req=item.req, - version=item.resolved_version, - source_url=item.source_url, - source_type=item.build_result.source_type, - prebuilt=pbi.pre_built, - constraint=constraint, - ) - - dep_items = self._create_unresolved_work_items( - install_dependencies, - RequirementType.INSTALL, - item.req, - item.resolved_version, - ) - - item.phase = BootstrapPhase.COMPLETE - return [item] + dep_items - - def _phase_complete(self, item: WorkItem) -> list[WorkItem]: - """COMPLETE phase: clean up build directories. - - Returns: - Empty list (processing finished for this item). - """ - if item.build_result is not None: - self.ctx.clean_build_dirs( - item.build_result.sdist_root_dir, - item.build_result.build_env, - ) - return [] - - def _dispatch_phase(self, item: WorkItem) -> list[WorkItem]: - """Route a work item to the appropriate phase handler.""" - match item.phase: - case BootstrapPhase.RESOLVE: - return self._phase_resolve(item) - case BootstrapPhase.START: - return self._phase_start(item) - case BootstrapPhase.PREPARE_SOURCE: - return self._phase_prepare_source(item) - case BootstrapPhase.PREPARE_BUILD: - return self._phase_prepare_build(item) - case BootstrapPhase.BUILD: - return self._phase_build(item) - case BootstrapPhase.PROCESS_INSTALL_DEPS: - return self._phase_process_install_deps(item) - case BootstrapPhase.COMPLETE: - return self._phase_complete(item) - case _: - raise ValueError(f"unexpected phase: {item.phase}") - - def _handle_phase_error( - self, - item: WorkItem, - err: Exception, - ) -> list[WorkItem]: - """Handle errors from phase processing. - - Returns work items to continue processing (e.g. prebuilt fallback), - or empty list to skip this item. Raises in normal mode (fail-fast). - """ - # Resolution failures: recoverable in test mode and multiple versions mode - if item.phase == BootstrapPhase.RESOLVE: - if self.test_mode: - self._record_test_mode_failure(item.req, None, err, "resolution") - if self.multiple_versions: - self._record_failed_version( - item.req, - "unresolved", - err, - f"failed during {item.phase} phase", - ) - return [] - if self.multiple_versions: - self._record_failed_version( - item.req, "unresolved", err, f"failed during {item.phase} phase" - ) - return [] - raise - - # Test mode: try prebuilt fallback for build-related phases - if self.test_mode: - if ( - item.phase - in ( - BootstrapPhase.PREPARE_SOURCE, - BootstrapPhase.PREPARE_BUILD, - BootstrapPhase.BUILD, - ) - and not item.pbi_pre_built - ): - assert item.resolved_version is not None - fallback = self._handle_test_mode_failure( - req=item.req, - resolved_version=item.resolved_version, - req_type=item.req_type, - build_error=err, - ) - if fallback is not None: - item.build_result = fallback - item.phase = BootstrapPhase.PROCESS_INSTALL_DEPS - return [item] - self._record_test_mode_failure( - item.req, str(item.resolved_version), err, "bootstrap" - ) - return [] - - # Multiple versions mode: record failure, remove from graph, continue - if self.multiple_versions: - assert item.resolved_version is not None - pkg_name = canonicalize_name(item.req.name) - self._record_failed_version( - item.req, - str(item.resolved_version), - err, - f"failed during {item.phase} phase", - ) - self.ctx.dependency_graph.remove_dependency(pkg_name, item.resolved_version) - self._seen_requirements.discard( - self._resolved_key(item.req, item.resolved_version, "sdist") - ) - self._seen_requirements.discard( - self._resolved_key(item.req, item.resolved_version, "wheel") - ) - self.ctx.write_to_graph_to_file() - return [] - - # Normal mode: fail-fast - raise - - def _record_failed_version( - self, - req: Requirement, - version: str, - err: Exception, - detail: str, - ) -> None: - """Record a version failure in multiple versions mode.""" - pkg_name = canonicalize_name(req.name) - self._failed_versions[(pkg_name, version)] = err - logger.warning( - "%s==%s: %s: %s: %s", - req.name, - version, - detail, - type(err).__name__, - err, - ) - - def _log_failed_versions_table(self) -> None: - """Log a summary table of all failed versions.""" - logger.warning("%d version(s) failed to bootstrap:", len(self._failed_versions)) - for (name, ver), exc in self._failed_versions.items(): - logger.warning(" %s==%s: %s: %s", name, ver, type(exc).__name__, exc) - - def finalize(self) -> int: - """Finalize bootstrap and return exit code. - - Reports failed versions in multiple versions mode. - In test mode, writes failure report and returns non-zero if there were failures. - - Returns: - 0 if all packages built successfully (or not in test/multiple versions mode) - 1 if any packages failed in test mode - """ - if self._bg_pool is not None: - # cancel_futures=True cancels pending (not-yet-started) futures immediately - # so we don't block waiting for work whose result will never be used. - # Already-running futures still complete naturally. - self._bg_pool.shutdown(wait=True, cancel_futures=True) - self._bg_pool = None - - if self.multiple_versions and self._failed_versions: - self._log_failed_versions_table() - - if not self.test_mode: - return 0 - - if not self.failed_packages: - logger.info("test mode: all packages processed successfully") - return 0 - - # Write JSON failure report with timestamp for uniqueness - timestamp = datetime.datetime.now(tz=datetime.UTC).strftime("%Y%m%d-%H%M%S-%f") - failures_file = self.ctx.work_dir / f"test-mode-failures-{timestamp}.json" - with open(failures_file, "w") as f: - json.dump({"failures": self.failed_packages}, f, indent=2) - logger.info("test mode: wrote failure report to %s", failures_file) - - # Log summary - failed_names = [f["package"] for f in self.failed_packages] - logger.error( - "test mode: %d package(s) failed: %s", - len(self.failed_packages), - ", ".join(failed_names), - ) - return 1 - - def __enter__(self) -> Bootstrapper: - return self - - def __exit__( - self, - exc_type: type | None, - exc_val: BaseException | None, - exc_tb: object, - ) -> None: - if self._bg_pool is not None: - self._bg_pool.shutdown(wait=False, cancel_futures=True) - self._bg_pool = None diff --git a/src/fromager/bootstrapper/__init__.py b/src/fromager/bootstrapper/__init__.py new file mode 100644 index 00000000..fda5fb83 --- /dev/null +++ b/src/fromager/bootstrapper/__init__.py @@ -0,0 +1,4 @@ +from ._bootstrapper import Bootstrapper +from ._types import _DEFAULT_BG_THREADS + +__all__ = ["_DEFAULT_BG_THREADS", "Bootstrapper"] diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py new file mode 100644 index 00000000..dfec81e2 --- /dev/null +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -0,0 +1,1286 @@ +from __future__ import annotations + +import concurrent.futures +import contextlib +import datetime +import json +import logging +import operator +import pathlib +import shutil +import tempfile +import typing + +from packaging.requirements import Requirement +from packaging.utils import NormalizedName, canonicalize_name +from packaging.version import Version + +from .. import ( + bootstrap_requirement_resolver, + build_environment, + dependencies, + gitutils, + progress, + sources, +) +from ..dependency_graph import DependencyGraph +from ..log import req_ctxvar_context, requirement_ctxvar +from ..requirements_file import RequirementType, SourceType +from . import _cache +from ._build import Build +from ._phase import Phase +from ._prepare_build import PrepareBuild +from ._prepare_source import PrepareSource +from ._process_install_deps import ProcessInstallDeps +from ._resolve import Resolve +from ._types import ( + _DEFAULT_BG_THREADS, + FailureRecord, + FailureType, + SeenKey, + SourceBuildResult, +) +from ._work_item import WorkItem + +if typing.TYPE_CHECKING: + from .. import context + +logger = logging.getLogger(__name__) + + +class Bootstrapper: + def __init__( + self, + ctx: context.WorkContext, + progressbar: progress.Progressbar | None = None, + prev_graph: DependencyGraph | None = None, + cache_wheel_server_url: str | None = None, + sdist_only: bool = False, + test_mode: bool = False, + multiple_versions: bool = False, + num_bg_threads: int = _DEFAULT_BG_THREADS, + ) -> None: + if test_mode and sdist_only: + raise ValueError( + "--test-mode requires full wheel builds; incompatible with --sdist-only" + ) + + self.ctx = ctx + self.progressbar = progressbar or progress.Progressbar(None) + self.prev_graph = prev_graph + self.cache_wheel_server_url = cache_wheel_server_url or ctx.wheel_server_url + self.sdist_only = sdist_only + self.test_mode = test_mode + self.multiple_versions = multiple_versions + self.why: list[tuple[RequirementType, Requirement, Version]] = [] + self._num_bg_threads = max(1, num_bg_threads) + self._bg_pool: concurrent.futures.ThreadPoolExecutor | None = ( + concurrent.futures.ThreadPoolExecutor( + max_workers=self._num_bg_threads, thread_name_prefix="fromager-bg" + ) + ) + + # Delegate resolution to BootstrapRequirementResolver + self._resolver = bootstrap_requirement_resolver.BootstrapRequirementResolver( + ctx=ctx, + prev_graph=prev_graph, + multiple_versions=multiple_versions, + cache_wheel_server_url=self.cache_wheel_server_url, + ) + # Push items onto the stack as we start to resolve their + # dependencies so at the end we have a list of items that need to + # be built in order. + self._build_stack: list[typing.Any] = [] + self._build_requirements: set[tuple[NormalizedName, str]] = set() + + # Track requirements we've seen before so we don't resolve the + # same dependencies over and over and so we can break cycles in + # the dependency list. The key is the requirements spec, rather + # than the package, in case we do have multiple rules for the same + # package. + self._seen_requirements: set[SeenKey] = set() + + self._build_order_filename = self.ctx.work_dir / "build-order.json" + self._stack_filename = self.ctx.work_dir / "bootstrap-stack.json" + logger.info("recording bootstrap stack state to %s", self._stack_filename) + + # Track failed packages in test mode (list of typed dicts for JSON export) + self.failed_packages: list[FailureRecord] = [] + + # Track failed versions in multiple_versions mode + self._failed_versions: dict[tuple[str, str], Exception] = {} + + @property + def resolver(self) -> bootstrap_requirement_resolver.BootstrapRequirementResolver: + """Public accessor for the version resolver.""" + return self._resolver + + def has_failed_version(self, pkg_name: NormalizedName, version: Version) -> bool: + """Return True if this (pkg_name, version) has previously failed resolution.""" + return (pkg_name, str(version)) in self._failed_versions + + def _resolve_and_add_top_level( + self, + req: Requirement, + ) -> tuple[str, Version] | None: + """Resolve a top-level requirement and add it to the dependency graph. + + Private method called only by ``bootstrap()``. + + This is the pre-resolution phase before recursive bootstrapping begins. + In test mode, catches resolution errors and records them as failures. + + When multiple_versions is enabled, resolves and adds all matching versions + to the graph, but still returns only the first (highest) version for + backward compatibility. + + Args: + req: The top-level requirement to resolve. + + Returns: + Tuple of (source_url, version) if resolution succeeded, None if it + failed in test mode. + + Raises: + Exception: In normal mode, re-raises any resolution error. + """ + try: + pbi = self.ctx.package_build_info(req) + results = self.resolve_versions( + req=req, + req_type=RequirementType.TOP_LEVEL, + parent_req=None, + return_all_versions=self.multiple_versions, + ) + if self.multiple_versions: + logger.info(f"resolved {len(results)} version(s) for {req}") + + # Add all resolved versions to the graph + for source_url, version in results: + logger.info("%s resolves to %s", req, version) + self.ctx.dependency_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=req, + req_version=version, + download_url=source_url, + pre_built=pbi.pre_built, + constraint=self.ctx.constraints.get_constraint(req.name), + ) + + if not results: + if self.multiple_versions: + err = RuntimeError(f"no versions found for {req}") + self._record_failed_version( + req, "unresolved", err, "no versions resolved, skipping" + ) + if self.test_mode: + self.record_test_mode_failure(req, None, err, "resolution") + return None + raise RuntimeError(f"Could not resolve any versions for {req}") + + # Return first result for backward compatibility + return results[0] + except Exception as err: + if self.multiple_versions: + self._record_failed_version(req, "unresolved", err, "failed to resolve") + if self.test_mode: + self.record_test_mode_failure(req, None, err, "resolution") + return None + if not self.test_mode: + raise + self.record_test_mode_failure(req, None, err, "resolution") + return None + + def resolve_versions( + self, + req: Requirement, + req_type: RequirementType, + parent_req: Requirement | None = None, + return_all_versions: bool = False, + ) -> list[tuple[str, Version]]: + """Resolve version(s) of a requirement. + + Returns list of (source URL, version) tuples, sorted by version (highest first). + + Git URL resolution stays in Bootstrapper because it requires + build orchestration (BuildEnvironment, build dependencies). + Delegates PyPI/graph resolution to BootstrapRequirementResolver. + + Args: + req: Package requirement to resolve + req_type: Type of requirement + parent_req: Explicit parent requirement from dependency chain. + Callers must pass this explicitly; do not read ``self.why`` here. + return_all_versions: If True, return all matching versions. + If False, return only highest version. + + Returns: + List of (url, version) tuples. Contains one item when + return_all_versions=False, or all matching versions when True. + """ + if req.url: + if req_type != RequirementType.TOP_LEVEL: + raise ValueError( + f"{req} includes a URL, but is not a top-level dependency" + ) + + # Check cache first to avoid re-resolving + # Git URLs are always source (not prebuilt) + cached_result = self._resolver.get_matching_versions(req, pre_built=False) + if cached_result: + logger.debug(f"resolved {req} from cache") + return cached_result if return_all_versions else [cached_result[0]] + + logger.info("resolving source via URL, ignoring any plugins") + source_url, resolved_version = self._resolve_version_from_git_url(req=req) + # Cache the git URL resolution (always source, not prebuilt) + # Store as list for consistency with cache structure + result = [(source_url, resolved_version)] + self._resolver.extend_known_versions(req, pre_built=False, result=result) + return result # Git URLs always return single version + + # Delegate to RequirementResolver + return self._resolver.resolve( + req=req, + req_type=req_type, + parent_req=parent_req, + return_all_versions=return_all_versions, + ) + + def bootstrap(self, requirements: list[Requirement]) -> None: + """Bootstrap all top-level requirements and their transitive dependencies. + + .. versionadded:: 0.89 + Replaces the former ``bootstrap(req, req_type)`` signature. + + Resolves each requirement, adds it to the dependency graph, and processes + the full dependency tree using an iterative DFS loop. Handles + ``requirement_ctxvar`` context internally; callers do not need to manage it. + + In test mode, records failures and continues instead of raising. In + ``multiple_versions`` mode, processes all matching versions per requirement. + + Args: + requirements: Top-level requirements to resolve and bootstrap. + """ + # Resolve all top-level reqs and build initial stack. + # Use the token pattern (no try/finally) so that if resolution raises + # in normal mode, the context var stays set for the top-level error + # handler in __main__.py to include the package name in its log message. + stack: list[Phase] = [] + initial_items: list[Phase] = [] + for req in requirements: + token = requirement_ctxvar.set(req) + result = self._resolve_and_add_top_level(req) + requirement_ctxvar.reset(token) + if result is not None: + initial_items.append( + Resolve( + WorkItem( + req=req, + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + parent=None, + ) + ) + ) + self._push_items(stack, initial_items) + + self._run_bootstrap_loop(stack) + + def _run_bootstrap_loop(self, stack: list[Phase]) -> None: + """Run the iterative DFS bootstrap loop over a pre-built work stack. + + Pops items one at a time, dispatches each phase, and pushes any + follow-on items (continuations and new dependencies) back onto the + stack. Updates the progress bar as items complete. + + Args: + stack: Initial list of ``Phase`` objects to process. Modified + in-place; empty on return. + """ + while stack: + self._record_stack_state(stack) + item = stack.pop() + self.why = list(item.work_item.why_snapshot) + + with ( + req_ctxvar_context(item.work_item.req, item.work_item.resolved_version), + self._track_why(item), + ): + if item.requires_exclusive_run: + logger.info( + "%s requires exclusive build, draining background pool", + item.work_item.req, + ) + self._drain_background_pool() + try: + new_items = item.run(self) + except Exception as err: + new_items = self._handle_phase_error(item, err) + + new_dep_count = sum(1 for it in new_items if isinstance(it, Resolve)) + if new_dep_count > 0: + self.progressbar.update_total(new_dep_count) + if not new_items: + self.progressbar.update() + + self._push_items(stack, new_items) + + def _bootstrap_one(self, req: Requirement, req_type: RequirementType) -> None: + """Bootstrap a single requirement using an iterative DFS loop. + + Internal method used only by the git URL resolution path + (``_handle_build_requirements``). All other callers should use + ``bootstrap(requirements)`` instead. + + Uses an explicit LIFO stack instead of recursion to handle arbitrarily + deep dependency graphs without hitting Python's recursion limit. + + In test mode, catches build exceptions, records package name, and continues. + In normal mode, raises exceptions immediately (fail-fast). + + When multiple_versions is enabled, bootstraps all matching versions instead + of just the highest version. + """ + logger.info(f"bootstrapping {req} as {req_type} dependency of {self.why[-1:]}") + + # Capture parent from current why stack before creating work items + parent: tuple[Requirement, Version] | None = None + if self.why: + _, parent_req, parent_version = self.why[-1] + parent = (parent_req, parent_version) + + # Save the why stack so we can restore it after the iterative loop + # (the loop modifies self.why for each work item) + saved_why = list(self.why) + + # Single RESOLVE item — resolution, version expansion, and error + # handling all happen inside the loop via Resolve.run(). + initial_item = Resolve( + WorkItem( + req=req, + req_type=req_type, + why_snapshot=list(self.why), + parent=parent, + ) + ) + stack: list[Phase] = [] + self._push_items(stack, [initial_item]) + + self._run_bootstrap_loop(stack) + + # empty the stack state file to show watchers are are done + # with bootstrapping + self._record_stack_state([]) + + # Restore why stack for the caller + self.why = saved_why + + @contextlib.contextmanager + def _track_why( + self, + item: Phase, + ) -> typing.Generator[None, None, None]: + """Context manager to track dependency chain in self.why stack. + + No-op for phases where tracks_why is False (RESOLVE and START). + For all other phases, pushes the item onto the why stack and + ensures it is popped even if an exception occurs. + """ + if not item.tracks_why: + yield + return + wi = item.work_item + assert wi.resolved_version is not None + self.why.append((wi.req_type, wi.req, wi.resolved_version)) + try: + yield + finally: + self.why.pop() + + def record_test_mode_failure( + self, + req: Requirement, + version: str | None, + err: Exception, + failure_type: FailureType, + log_level: typing.Literal["error", "warning"] = "error", + ) -> None: + """Record a failure in test mode. Call this after checking test_mode. + + Args: + req: The requirement that failed. + version: The version being processed (None if not yet resolved). + err: The exception that was raised. + failure_type: Category of failure for analysis. + log_level: Log at error (fatal) or warning (non-fatal, continuing). + """ + version_str = f"=={version}" if version else "" + msg = f"test mode: {failure_type} failed for {req.name}{version_str}" + if log_level == "warning": + logger.warning("%s: %s (continuing)", msg, err) + else: + logger.error("%s: %s", msg, err, exc_info=True) + + self.failed_packages.append( + { + "package": str(req.name), + "version": version, + "exception_type": err.__class__.__name__, + "exception_message": str(err), + "failure_type": failure_type, + } + ) + + @property + def explain(self) -> str: + """Return message formatting current version of why stack.""" + return " for ".join( + f"{req_type} dependency {req} ({resolved_version})" + for req_type, req, resolved_version in reversed(self.why) + ) + + def _prepare_build_dependencies( + self, + req: Requirement, + resolved_version: Version | None, + sdist_root_dir: pathlib.Path, + build_env: build_environment.BuildEnvironment, + ) -> set[Requirement]: + """Prepare build dependencies for a package. + + Only used by the git URL resolution path + (_resolve_version_from_git_url -> _get_version_from_package_metadata). + The main iterative bootstrap loop handles build deps via phase handlers. + """ + # build system + build_system_dependencies = dependencies.get_build_system_dependencies( + ctx=self.ctx, + req=req, + version=resolved_version, + sdist_root_dir=sdist_root_dir, + ) + self._handle_build_requirements( + req, + RequirementType.BUILD_SYSTEM, + build_system_dependencies, + ) + # The next hooks need build system requirements. + build_env.install(build_system_dependencies) + + # build backend + build_backend_dependencies = dependencies.get_build_backend_dependencies( + ctx=self.ctx, + req=req, + version=resolved_version, + sdist_root_dir=sdist_root_dir, + build_env=build_env, + ) + + # build sdist + build_sdist_dependencies = dependencies.get_build_sdist_dependencies( + ctx=self.ctx, + req=req, + version=resolved_version, + sdist_root_dir=sdist_root_dir, + build_env=build_env, + ) + + # Filter out deps already satisfied by build-system dependencies + resolved_build_sys = self._resolve_build_system_versions_by_name( + build_system_dependencies, + ) + build_backend_dependencies = self.filter_deps_satisfied_by_build_system( + build_backend_dependencies, + resolved_build_sys, + RequirementType.BUILD_BACKEND, + ) + build_sdist_dependencies = self.filter_deps_satisfied_by_build_system( + build_sdist_dependencies, + resolved_build_sys, + RequirementType.BUILD_SDIST, + ) + + self._handle_build_requirements( + req, + RequirementType.BUILD_BACKEND, + build_backend_dependencies, + ) + self._handle_build_requirements( + req, + RequirementType.BUILD_SDIST, + build_sdist_dependencies, + ) + + build_dependencies = build_sdist_dependencies | build_backend_dependencies + if build_dependencies.isdisjoint(build_system_dependencies): + build_env.install(build_dependencies) + + return ( + build_system_dependencies + | build_backend_dependencies + | build_sdist_dependencies + ) + + def _handle_build_requirements( + self, + req: Requirement, + build_type: RequirementType, + build_dependencies: set[Requirement], + ) -> None: + """Bootstrap build dependencies. + + Only used by the git URL resolution path + (_resolve_version_from_git_url -> _get_version_from_package_metadata). + The main iterative bootstrap loop handles build deps via phase handlers. + """ + self.progressbar.update_total(len(build_dependencies)) + + for dep in self._sort_requirements(build_dependencies): + with req_ctxvar_context(dep): + # Save/restore self.why because the iterative bootstrap() + # modifies it internally for each work item. + saved_why = list(self.why) + self._bootstrap_one(req=dep, req_type=build_type) + self.why = saved_why + self.progressbar.update() + + def _download_prebuilt( + self, + req: Requirement, + req_type: RequirementType, + resolved_version: Version, + wheel_url: str, + ) -> tuple[pathlib.Path, pathlib.Path]: + result = _cache._bg_prepare_prebuilt( + self.ctx, req, req_type, resolved_version, wheel_url + ) + assert result.wheel_filename is not None + assert result.unpack_dir is not None + return (result.wheel_filename, result.unpack_dir) + + def _download_source( + self, + req: Requirement, + resolved_version: Version, + source_url: str, + ) -> pathlib.Path: + """Download source for a package.""" + result: pathlib.Path = sources.download_source( + ctx=self.ctx, + req=req, + version=resolved_version, + download_url=source_url, + ) + return result + + def _prepare_source( + self, + req: Requirement, + resolved_version: Version, + source_filename: pathlib.Path, + ) -> pathlib.Path: + """Prepare (unpack/patch) source for building.""" + result: pathlib.Path = sources.prepare_source( + ctx=self.ctx, + req=req, + source_filename=source_filename, + version=resolved_version, + ) + return result + + def _handle_test_mode_failure( + self, + req: Requirement, + resolved_version: Version, + req_type: RequirementType, + build_error: Exception, + ) -> SourceBuildResult | None: + """Handle build failure in test mode by attempting pre-built fallback. + + Args: + req: The requirement that failed to build. + resolved_version: The version that was attempted. + req_type: The type of requirement (for fallback resolution). + build_error: The original exception from the build attempt. + + Returns: + SourceBuildResult if fallback succeeded, None if fallback also failed. + """ + logger.warning( + "test mode: build failed for %s==%s, attempting pre-built fallback: %s", + req.name, + resolved_version, + build_error, + ) + + try: + parent_req = self.why[-1][1] if self.why else None + results = self._resolver.resolve( + req=req, + req_type=req_type, + parent_req=parent_req, + pre_built=True, # Force prebuilt for test mode fallback + ) + wheel_url, fallback_version = results[0] + + if fallback_version != resolved_version: + logger.warning( + "test mode: version mismatch for %s - requested %s, fallback %s", + req.name, + resolved_version, + fallback_version, + ) + + wheel_filename, unpack_dir = self._download_prebuilt( + req=req, + req_type=req_type, + resolved_version=fallback_version, + wheel_url=wheel_url, + ) + + logger.info( + "test mode: successfully used pre-built wheel for %s==%s", + req.name, + fallback_version, + ) + + # Package succeeded via fallback - no failure to record + + return SourceBuildResult( + wheel_filename=wheel_filename, + sdist_filename=None, + unpack_dir=unpack_dir, + sdist_root_dir=None, + build_env=None, + source_type=SourceType.PREBUILT, + ) + + except Exception as fallback_error: + logger.error( + "test mode: pre-built fallback also failed for %s: %s", + req.name, + fallback_error, + exc_info=True, + ) + # Return None to signal failure; bootstrap() will record via re-raised exception + return None + + def _resolve_version_from_git_url(self, req: Requirement) -> tuple[str, Version]: + """Resolve source path and version from a ``git+`` URL. + + Parses the URL for an ``@ref`` version hint. If the ref is a valid + version, reuses an existing clone when possible. Otherwise, clones + the repo and extracts the version from package metadata. + """ + + if not req.url: + raise ValueError(f"unable to resolve from URL with no URL in {req}") + + # We start by not knowing where we would put the source because we don't + # know the version. + working_src_dir: pathlib.Path | None = None + version: Version | None = None + + url_to_clone, git_ref = gitutils.parse_vcs_url(req.url, require_ref=False) + need_to_clone = False + + if git_ref == gitutils.GIT_HEAD: + # No ref in URL, clone to discover the version. + logger.debug("no reference in URL, will clone") + need_to_clone = True + else: + # If we have a reference, it might be a valid python version + # string, or not. It _must_ be a valid git reference. If it can + # be parsed as a valid python version, we assume the tag points + # to source that will think that is its version, so we allow + # reusing an existing cloned repo if there is one. + try: + version = Version(git_ref) + except ValueError: + logger.info( + "could not parse %r as a version, cloning to get the version", + git_ref, + ) + need_to_clone = True + else: + logger.info("URL %s includes version %s", req.url, version) + working_src_dir = ( + self.ctx.work_dir + / f"{req.name}-{version}" + / f"{req.name}-{version}" + ) + if not working_src_dir.exists(): + need_to_clone = True + else: + if self.ctx.cleanup: + logger.debug("cleaning up %s to reclone", working_src_dir) + shutil.rmtree(working_src_dir) + need_to_clone = True + else: + logger.info("reusing %s", working_src_dir) + + if need_to_clone: + with tempfile.TemporaryDirectory() as tmpdir: + clone_dir = pathlib.Path(tmpdir) / "src" + sources.download_git_source( + ctx=self.ctx, + req=req, + url_to_clone=url_to_clone, + destination_dir=clone_dir, + ref=git_ref, + ) + if not version: + # If we still do not have a version, get it from the package + # metadata. + version = self._get_version_from_package_metadata(req, clone_dir) + logger.info("found version %s", version) + working_src_dir = ( + self.ctx.work_dir + / f"{req.name}-{version}" + / f"{req.name}-{version}" + ) + if working_src_dir.exists(): + # We have to check if the destination directory exists + # because if we were not given a version we did not + # clean it up earlier. We do not use ctx.cleanup to + # control this action because we cannot trust that the + # destination directory is reusable because we have had + # to compute the version and we cannot be sure that the + # version is dynamic. Two different commits in the repo + # could have the same version if that version is set + # with static data in the repo instead of via a tag or + # dynamically computed by something like setuptools-scm. + logger.debug("cleaning up %s", working_src_dir) + shutil.rmtree(working_src_dir) + assert working_src_dir is not None + working_src_dir.parent.mkdir(parents=True, exist_ok=True) + logger.info("moving cloned repo to %s", working_src_dir) + shutil.move(clone_dir, str(working_src_dir)) + + if not version: + raise ValueError(f"unable to determine version for {req}") + + if not working_src_dir: + raise ValueError(f"unable to determine working source directory for {req}") + + logging.info("resolved from git URL to %s, %s", working_src_dir, version) + return (str(working_src_dir), version) + + def _get_version_from_package_metadata( + self, + req: Requirement, + source_dir: pathlib.Path, + ) -> Version: + pbi = self.ctx.package_build_info(req) + build_dir = pbi.build_dir(source_dir) + + logger.info( + "preparing build dependencies so we can access the metadata to get the version" + ) + build_env = build_environment.BuildEnvironment( + ctx=self.ctx, + parent_dir=source_dir.parent, + ) + build_dependencies = self._prepare_build_dependencies( + req=req, + resolved_version=None, + sdist_root_dir=source_dir, + build_env=build_env, + ) + build_env.install(build_dependencies) + + logger.info("generating metadata to get version") + hook_caller = dependencies.get_build_backend_hook_caller( + ctx=self.ctx, + req=req, + build_dir=build_dir, + override_environ={}, + build_env=build_env, + ) + metadata_dir_base = hook_caller.prepare_metadata_for_build_wheel( + metadata_directory=str(source_dir.parent), + config_settings=pbi.config_settings, + ) + metadata_filename = source_dir.parent / metadata_dir_base / "METADATA" + # Disable validation because some packages have metadata version mismatches + # (e.g., declaring Metadata-Version: 2.2 but using fields from 2.4). + metadata = dependencies.parse_metadata(metadata_filename, validate=False) + return metadata.version + + def add_to_graph( + self, + req: Requirement, + req_type: RequirementType, + req_version: Version, + download_url: str, + parent: tuple[Requirement, Version] | None, + ) -> None: + """Add a resolved requirement as a node/edge in the dependency graph. + + Records the dependency relationship between ``parent`` and ``req`` in + the in-memory graph, then writes the updated graph to disk. Must be + called after resolution but before the seen-check so that every edge + is captured even for packages that are not built again. + + Args: + req: The requirement being added. + req_type: Whether this is a build-system, build-backend, install, + or other dependency kind. + req_version: The resolved version of ``req``. + download_url: Source URL used to obtain the distribution. + parent: ``(req, version)`` of the package that introduced ``req``, + or ``None`` for top-level requirements. + """ + parent_req, parent_version = parent if parent else (None, None) + pbi = self.ctx.package_build_info(req) + # Update the dependency graph after we determine that this requirement is + # useful but before we determine if it is redundant so that we capture all + # edges to use for building a valid constraints file. + self.ctx.dependency_graph.add_dependency( + parent_name=canonicalize_name(parent_req.name) if parent_req else None, + parent_version=parent_version, + req_type=req_type, + req=req, + req_version=req_version, + download_url=download_url, + pre_built=pbi.pre_built, + constraint=self.ctx.constraints.get_constraint(req.name), + ) + self.ctx.write_to_graph_to_file() + + def _sort_requirements( + self, + requirements: typing.Iterable[Requirement], + ) -> typing.Iterable[Requirement]: + return sorted(requirements, key=operator.attrgetter("name")) + + def _resolved_key( + self, req: Requirement, version: Version, typ: typing.Literal["sdist", "wheel"] + ) -> SeenKey: + return ( + canonicalize_name(req.name), + tuple(sorted(req.extras)), + str(version), + typ, + ) + + def mark_as_seen( + self, + req: Requirement, + version: Version, + sdist_only: bool = False, + ) -> None: + """Track sdist and wheel builds + + A sdist-only build just contains as an sdist. + A wheel build counts as wheel and sdist, because the presence of a + either implies we have built a wheel from an sdist or we have a + prebuilt wheel that will never have an sdist. + """ + # Mark sdist seen for sdist-only build and wheel build + self._seen_requirements.add(self._resolved_key(req, version, "sdist")) + if not sdist_only: + # Mark wheel seen only for wheel build + self._seen_requirements.add(self._resolved_key(req, version, "wheel")) + + def has_been_seen( + self, + req: Requirement, + version: Version, + sdist_only: bool = False, + ) -> bool: + """Return True if this requirement/version has already been processed. + + Uses the same sdist-vs-wheel distinction as ``mark_as_seen``: + when ``sdist_only`` is True, checks only for a prior sdist build; + otherwise checks for a prior wheel build. + + Args: + req: The requirement to check. + version: The resolved version to check. + sdist_only: If True, check for a sdist-only build instead of a + full wheel build. + """ + typ: typing.Literal["sdist", "wheel"] = "sdist" if sdist_only else "wheel" + return self._resolved_key(req, version, typ) in self._seen_requirements + + def add_to_build_order( + self, + req: Requirement, + version: Version, + source_url: str, + source_type: SourceType, + prebuilt: bool = False, + constraint: Requirement | None = None, + ) -> None: + """Append a package to the build-order output file if not already present. + + Deduplicates by ``(canonicalized_name, version)`` so the same package + is never written twice, regardless of extras. On each new entry, the + full build-order list is written to ``self._build_order_filename``. + + Args: + req: The requirement being added to the build order. + version: The resolved version. + source_url: URL from which the source distribution was obtained. + source_type: The kind of source (sdist, wheel, git, etc.). + prebuilt: True if the wheel was taken from a prebuilt cache rather + than compiled locally. + constraint: The version constraint active for this package, if any. + """ + # We only care if this version of this package has been built, + # and don't want to trigger building it twice. The "extras" + # value, included in the _resolved_key() output, can confuse + # that so we ignore itand build our own key using just the + # name and version. + key = (canonicalize_name(req.name), str(version)) + if key in self._build_requirements: + return + logger.info(f"adding {key} to build order") + self._build_requirements.add(key) + info = { + "req": str(req), + "constraint": str(constraint) if constraint else "", + "dist": canonicalize_name(req.name), + "version": str(version), + "prebuilt": prebuilt, + "source_url": source_url, + "source_url_type": str(source_type), + } + if req.url: + info["source_url"] = req.url + self._build_stack.append(info) + with open(self._build_order_filename, "w") as f: + # Set default=str because the why value includes + # Requirement and Version instances that can't be + # converted to JSON without help. + json.dump(self._build_stack, f, indent=2, default=str) + + def _record_stack_state(self, stack: list[Phase]) -> None: + """Write the current bootstrap stack to `self._stack_filename`. + + Index 0 in the output corresponds to `stack[-1]`, the next item to be + processed. Overwrites the file on each call. + """ + records = [item.as_json() for item in reversed(stack)] + with open(self._stack_filename, "w") as f: + json.dump(records, f, indent=2, default=str) + + # ---- Iterative bootstrap: phase handlers and helpers ---- + + def _push_items(self, stack: list[Phase], items: list[Phase]) -> None: + """Push items onto the stack and submit background tasks in LIFO order. + + Submits the item that will be processed first (top of stack) to the + background pool first, maximising overlap between background I/O and + main-thread serial work. + """ + stack.extend(items) + if self._bg_pool is not None: + for item in reversed(items): + bg_work = item.background_work(self) + if bg_work is not None: + item.bg_future = self._bg_pool.submit(bg_work) + + def _drain_background_pool(self) -> None: + """Drain all in-flight background tasks and recreate the pool. + + Used as an exclusive-build barrier: ensures all background I/O completes + before an exclusive build starts. ``cancel_futures=False`` guarantees + every submitted task runs to completion. + """ + if self._bg_pool is not None: + self._bg_pool.shutdown(wait=True, cancel_futures=False) + self._bg_pool = concurrent.futures.ThreadPoolExecutor( + max_workers=self._num_bg_threads, thread_name_prefix="fromager-bg" + ) + + def create_unresolved_work_items( + self, + deps: typing.Iterable[Requirement], + dep_req_type: RequirementType, + parent_req: Requirement, + parent_version: Version, + ) -> list[Phase]: + """Create RESOLVE-phase work items for dependencies. + + Called inside a parent's _track_why context so that why_snapshot + captures the parent's dependency chain. Resolution and error + handling happen later when each item's RESOLVE phase runs. + """ + return [ + Resolve( + WorkItem( + req=dep, + req_type=dep_req_type, + why_snapshot=list(self.why), + parent=(parent_req, parent_version), + ) + ) + for dep in self._sort_requirements(deps) + ] + + def _resolve_build_system_versions_by_name( + self, + build_system_deps: set[Requirement], + ) -> dict[NormalizedName, tuple[Version, str]]: + """Build a mapping of resolved build-system versions by looking up graph nodes. + + Used by the git URL path where the parent node may not yet exist + in the dependency graph. + """ + resolved: dict[NormalizedName, tuple[Version, str]] = {} + for dep in build_system_deps: + dep_name = canonicalize_name(dep.name) + nodes = self.ctx.dependency_graph.get_nodes_by_name(str(dep_name)) + for node in nodes: + if node.version in dep.specifier: + resolved[dep_name] = (node.version, node.download_url) + break + return resolved + + def get_resolved_build_system_versions( + self, + item: WorkItem, + ) -> dict[NormalizedName, tuple[Version, str]]: + """Build a mapping of resolved build-system dependency versions. + + Looks up the parent node's ``BUILD_SYSTEM`` edges in the dependency + graph to find what versions were resolved for each build-system + dependency. + + Returns: + Mapping of canonicalized package name to ``(version, download_url)``. + """ + assert item.resolved_version is not None + parent_key = f"{canonicalize_name(item.req.name)}=={item.resolved_version}" + parent_node = self.ctx.dependency_graph.nodes.get(parent_key) + if parent_node is None: + return {} + resolved: dict[NormalizedName, tuple[Version, str]] = {} + for edge in parent_node.children: + if edge.req_type == RequirementType.BUILD_SYSTEM: + child = edge.destination_node + resolved[child.canonicalized_name] = ( + child.version, + child.download_url, + ) + return resolved + + def filter_deps_satisfied_by_build_system( + self, + deps: set[Requirement], + resolved_build_sys: dict[NormalizedName, tuple[Version, str]], + dep_req_type: RequirementType, + parent: tuple[Requirement, Version] | None = None, + ) -> set[Requirement]: + """Filter out deps already satisfied by build-system dependencies. + + For each dep whose resolved build-system version satisfies the + requirement specifier, excludes the dep from the returned set. + When *parent* is provided, also adds a graph edge reusing that + version. Remaining deps need independent resolution. + + Logs a warning when the same package appears in both build-system + and build-backend/sdist with incompatible version specifiers. + """ + unsatisfied: set[Requirement] = set() + for dep in deps: + if dep.extras: + unsatisfied.add(dep) + continue + dep_name = canonicalize_name(dep.name) + if dep_name in resolved_build_sys: + version, download_url = resolved_build_sys[dep_name] + if version in dep.specifier: + logger.info( + "%s dependency %s is already satisfied by " + "build-system dependency %s==%s", + dep_req_type, + dep, + dep_name, + version, + ) + if parent is not None: + self.add_to_graph( + req=dep, + req_type=dep_req_type, + req_version=version, + download_url=download_url, + parent=parent, + ) + continue + else: + logger.warning( + "%s dependency %s conflicts with " + "build-system dependency %s==%s; " + "resolving independently", + dep_req_type, + dep, + dep_name, + version, + ) + unsatisfied.add(dep) + return unsatisfied + + def _handle_phase_error( + self, + item: Phase, + err: Exception, + ) -> list[Phase]: + """Handle errors from phase processing. + + Returns work items to continue processing (e.g. prebuilt fallback), + or empty list to skip this item. Raises in normal mode (fail-fast). + """ + wi = item.work_item + # Resolution failures: recoverable in test mode and multiple versions mode + if isinstance(item, Resolve): + if self.test_mode: + self.record_test_mode_failure(wi.req, None, err, "resolution") + if self.multiple_versions: + self._record_failed_version( + wi.req, + "unresolved", + err, + f"failed during {type(item).phase} phase", + ) + return [] + if self.multiple_versions: + self._record_failed_version( + wi.req, + "unresolved", + err, + f"failed during {type(item).phase} phase", + ) + return [] + raise + + # Test mode: try prebuilt fallback for build-related phases + if self.test_mode: + if ( + isinstance(item, PrepareSource | PrepareBuild | Build) + and not wi.pbi_pre_built + ): + assert wi.resolved_version is not None + fallback = self._handle_test_mode_failure( + req=wi.req, + resolved_version=wi.resolved_version, + req_type=wi.req_type, + build_error=err, + ) + if fallback is not None: + wi.build_result = fallback + return [ProcessInstallDeps(wi)] + self.record_test_mode_failure( + wi.req, str(wi.resolved_version), err, "bootstrap" + ) + return [] + + # Multiple versions mode: record failure, remove from graph, continue + if self.multiple_versions: + assert wi.resolved_version is not None + pkg_name = canonicalize_name(wi.req.name) + self._record_failed_version( + wi.req, + str(wi.resolved_version), + err, + f"failed during {type(item).phase} phase", + ) + self.ctx.dependency_graph.remove_dependency(pkg_name, wi.resolved_version) + self._seen_requirements.discard( + self._resolved_key(wi.req, wi.resolved_version, "sdist") + ) + self._seen_requirements.discard( + self._resolved_key(wi.req, wi.resolved_version, "wheel") + ) + self.ctx.write_to_graph_to_file() + return [] + + # Normal mode: fail-fast + raise + + def _record_failed_version( + self, + req: Requirement, + version: str, + err: Exception, + detail: str, + ) -> None: + """Record a version failure in multiple versions mode.""" + pkg_name = canonicalize_name(req.name) + self._failed_versions[(pkg_name, version)] = err + logger.warning( + "%s==%s: %s: %s: %s", + req.name, + version, + detail, + type(err).__name__, + err, + ) + + def _log_failed_versions_table(self) -> None: + """Log a summary table of all failed versions.""" + logger.warning("%d version(s) failed to bootstrap:", len(self._failed_versions)) + for (name, ver), exc in self._failed_versions.items(): + logger.warning(" %s==%s: %s: %s", name, ver, type(exc).__name__, exc) + + def finalize(self) -> int: + """Finalize bootstrap and return exit code. + + Reports failed versions in multiple versions mode. + In test mode, writes failure report and returns non-zero if there were failures. + + Returns: + 0 if all packages built successfully (or not in test/multiple versions mode) + 1 if any packages failed in test mode + """ + if self._bg_pool is not None: + # cancel_futures=True cancels pending (not-yet-started) futures immediately + # so we don't block waiting for work whose result will never be used. + # Already-running futures still complete naturally. + self._bg_pool.shutdown(wait=True, cancel_futures=True) + self._bg_pool = None + + if self.multiple_versions and self._failed_versions: + self._log_failed_versions_table() + + if not self.test_mode: + return 0 + + if not self.failed_packages: + logger.info("test mode: all packages processed successfully") + return 0 + + # Write JSON failure report with timestamp for uniqueness + timestamp = datetime.datetime.now(tz=datetime.UTC).strftime("%Y%m%d-%H%M%S-%f") + failures_file = self.ctx.work_dir / f"test-mode-failures-{timestamp}.json" + with open(failures_file, "w") as f: + json.dump({"failures": self.failed_packages}, f, indent=2) + logger.info("test mode: wrote failure report to %s", failures_file) + + # Log summary + failed_names = [f["package"] for f in self.failed_packages] + logger.error( + "test mode: %d package(s) failed: %s", + len(self.failed_packages), + ", ".join(failed_names), + ) + return 1 + + def __enter__(self) -> Bootstrapper: + return self + + def __exit__( + self, + exc_type: type | None, + exc_val: BaseException | None, + exc_tb: object, + ) -> None: + if self._bg_pool is not None: + self._bg_pool.shutdown(wait=False, cancel_futures=True) + self._bg_pool = None diff --git a/src/fromager/bootstrapper/_build.py b/src/fromager/bootstrapper/_build.py new file mode 100644 index 00000000..250b99ac --- /dev/null +++ b/src/fromager/bootstrapper/_build.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import logging +import pathlib +import typing + +from .. import finders, server, sources, wheels +from ._phase import Phase +from ._process_install_deps import ProcessInstallDeps +from ._types import BootstrapPhase, SourceBuildResult + +if typing.TYPE_CHECKING: + from .. import context + from ._bootstrapper import Bootstrapper + +logger = logging.getLogger(__name__) + + +class Build(Phase): + """Install remaining build deps and produce a wheel or sdist for the package. + + Three execution paths based on ``WorkItem`` state: + + - **Cached wheel**: a matching wheel was found during source preparation; + no build is performed. + - **sdist-only**: ``sdist_only`` mode is active and this is not a build + requirement; only an sdist is built. + - **Full build**: an sdist is built first, then a wheel is compiled and + the local mirror is updated. + + When ``pbi.exclusive_build`` is set the bootstrap loop drains the + background thread pool before calling ``run()`` so the build has exclusive + access to the environment. + + Next phase: ``ProcessInstallDeps``. + """ + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.BUILD + tracks_why: typing.ClassVar[bool] = True + + @property + def requires_exclusive_run(self) -> bool: + return self.work_item.exclusive_build + + def run(self, bt: Bootstrapper) -> list[Phase]: + """BUILD phase: install remaining deps, build wheel/sdist. + + Returns: + [ProcessInstallDeps]. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.build_env is not None + assert wi.sdist_root_dir is not None + + # Install backend+sdist deps if disjoint from system deps + remaining_deps = wi.build_backend_deps | wi.build_sdist_deps + if remaining_deps.isdisjoint(wi.build_system_deps): + wi.build_env.install(remaining_deps) + + wheel_filename, sdist_filename = self.do_build(bt.ctx, bt.explain) + + source_type = sources.get_source_type(bt.ctx, wi.req) + + wi.build_result = SourceBuildResult( + wheel_filename=wheel_filename, + sdist_filename=sdist_filename, + unpack_dir=wi.sdist_root_dir.parent, + sdist_root_dir=wi.sdist_root_dir, + build_env=wi.build_env, + source_type=source_type, + ) + + return [ProcessInstallDeps(wi)] + + def _build_sdist(self, ctx: context.WorkContext) -> pathlib.Path: + """Build or locate an sdist for this item's package. + + Checks ``ctx.sdists_builds`` for an existing sdist before invoking + ``sources.build_sdist()``. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.sdist_root_dir is not None + assert wi.build_env is not None + find_sdist_result = finders.find_sdist( + ctx, ctx.sdists_builds, wi.req, str(wi.resolved_version) + ) + if not find_sdist_result: + sdist_filename: pathlib.Path = sources.build_sdist( + ctx=ctx, + req=wi.req, + version=wi.resolved_version, + sdist_root_dir=wi.sdist_root_dir, + build_env=wi.build_env, + ) + else: + sdist_filename = find_sdist_result + logger.info( + f"have sdist version {wi.resolved_version}: {find_sdist_result}" + ) + return sdist_filename + + def _build_wheel( + self, + ctx: context.WorkContext, + explain: str = "", + ) -> tuple[pathlib.Path, pathlib.Path]: + """Build a wheel for this item's package. + + Calls :meth:`_build_sdist` first, then invokes ``wheels.build_wheel()`` + and updates the local wheel mirror. Returns ``(wheel_filename, + sdist_filename)`` where *wheel_filename* is the path in + ``ctx.wheels_downloads`` after the mirror update. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.sdist_root_dir is not None + assert wi.build_env is not None + sdist_filename = self._build_sdist(ctx) + + logger.info(f"starting build of {explain} for {ctx.variant}") + built_filename = wheels.build_wheel( + ctx=ctx, + req=wi.req, + sdist_root_dir=wi.sdist_root_dir, + version=wi.resolved_version, + build_env=wi.build_env, + ) + server.update_wheel_mirror(ctx) + # When we update the mirror, the built file moves to the downloads directory. + wheel_filename = ctx.wheels_downloads / built_filename.name + logger.info(f"built wheel for version {wi.resolved_version}: {wheel_filename}") + return wheel_filename, sdist_filename + + def do_build( + self, + ctx: context.WorkContext, + explain: str = "", + ) -> tuple[pathlib.Path | None, pathlib.Path | None]: + """Build a wheel or sdist for this item's package. + + Returns ``(wheel_filename, sdist_filename)``. Either value may be + ``None`` depending on the build path: + + - Cached wheel: ``(cached_wheel_filename, None)`` + - sdist-only: ``(None, sdist_filename)`` + - Full wheel: ``(wheel_filename, sdist_filename)`` + """ + wi = self.work_item + if wi.cached_wheel_filename: + logger.debug( + f"getting install requirements from cached wheel {wi.cached_wheel_filename.name}" + ) + return wi.cached_wheel_filename, None + elif wi.build_sdist_only: + logger.debug( + f"getting install requirements from sdist {wi.req.name}=={wi.resolved_version}" + ) + return None, self._build_sdist(ctx) + else: + logger.debug( + f"building wheel {wi.req.name}=={wi.resolved_version} to get install requirements" + ) + return self._build_wheel(ctx, explain) diff --git a/src/fromager/bootstrapper/_cache.py b/src/fromager/bootstrapper/_cache.py new file mode 100644 index 00000000..d1ca7f31 --- /dev/null +++ b/src/fromager/bootstrapper/_cache.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +import logging +import os +import pathlib +import typing +import zipfile +from urllib.parse import urlparse + +import requests.exceptions +from packaging.requirements import Requirement +from packaging.version import Version +from resolvelib.resolvers import ResolverException + +from .. import dependencies, finders, resolver, server, wheels +from ..requirements_file import RequirementType +from ._types import PreparedSourceData + +if typing.TYPE_CHECKING: + from .. import context + +logger = logging.getLogger(__name__) + + +def _create_unpack_dir( + work_dir: pathlib.Path, + req: Requirement, + resolved_version: Version, +) -> pathlib.Path: + unpack_dir = work_dir / f"{req.name}-{resolved_version}" + unpack_dir.mkdir(parents=True, exist_ok=True) + return unpack_dir + + +def _extract_build_reqs_from_wheel( + work_dir: pathlib.Path, + req: Requirement, + resolved_version: Version, + wheel_filename: pathlib.Path, +) -> pathlib.Path | None: + """Extract fromager build requirement files from a wheel archive. + + Looks for files prefixed with `FROMAGER_BUILD_REQ_PREFIX` inside the + wheel's `.dist-info` directory and extracts them to a local unpack + directory. Returns the unpack directory on success, or None if the + files are not present or extraction fails. + """ + dist_name, dist_version, _, _ = wheels.extract_info_from_wheel_file( + req, wheel_filename + ) + unpack_dir = _create_unpack_dir(work_dir, req, resolved_version) + dist_filename = f"{dist_name}-{dist_version}" + dist_info_path = pathlib.Path(f"{dist_filename}.dist-info") + req_filenames: list[str] = [ + dependencies.BUILD_BACKEND_REQ_FILE_NAME, + dependencies.BUILD_SDIST_REQ_FILE_NAME, + dependencies.BUILD_SYSTEM_REQ_FILE_NAME, + ] + try: + with zipfile.ZipFile(wheel_filename) as archive: + for filename in req_filenames: + zipinfo = archive.getinfo( + str( + dist_info_path + / f"{wheels.FROMAGER_BUILD_REQ_PREFIX}-{filename}" + ) + ) + if os.path.isabs(zipinfo.filename) or ".." in zipinfo.filename: + raise ValueError(f"Unsafe path in wheel: {zipinfo.filename}") + zipinfo.filename = filename + output_file = archive.extract(zipinfo, unpack_dir) + logger.info(f"extracted {output_file}") + logger.info(f"extracted build requirements from wheel into {unpack_dir}") + return unpack_dir + except Exception as e: + logger.info(f"could not extract build requirements from wheel: {e}") + for filename in req_filenames: + unpack_dir.joinpath(filename).unlink(missing_ok=True) + return None + + +def _look_for_existing_wheel( + ctx: context.WorkContext, + req: Requirement, + resolved_version: Version, + search_in: pathlib.Path, +) -> tuple[pathlib.Path | None, pathlib.Path | None]: + pbi = ctx.package_build_info(req) + expected_build_tag = pbi.build_tag(resolved_version) + logger.info( + f"looking for existing wheel for version {resolved_version} with build tag {expected_build_tag} in {search_in}" + ) + wheel_filename = finders.find_wheel( + downloads_dir=search_in, + req=req, + dist_version=str(resolved_version), + build_tag=expected_build_tag, + ) + if not wheel_filename: + return None, None + _, _, build_tag, _ = wheels.extract_info_from_wheel_file(req, wheel_filename) + if expected_build_tag and expected_build_tag != build_tag: + logger.info( + f"found wheel for {resolved_version} in {wheel_filename} but build tag does not match. Got {build_tag} but expected {expected_build_tag}" + ) + return None, None + logger.info(f"found existing wheel {wheel_filename}") + build_reqs_dir = _extract_build_reqs_from_wheel( + ctx.work_dir, req, resolved_version, wheel_filename + ) + return wheel_filename, build_reqs_dir + + +def _download_wheel_from_cache( + ctx: context.WorkContext, + cache_wheel_server_url: str | None, + req: Requirement, + resolved_version: Version, +) -> tuple[pathlib.Path | None, pathlib.Path | None]: + if not cache_wheel_server_url: + return None, None + logger.info(f"checking if wheel was already uploaded to {cache_wheel_server_url}") + try: + pinned_req = Requirement(f"{req.name}=={resolved_version}") + provider = finders.PyPICacheProvider( + cache_server_url=cache_wheel_server_url, + constraints=ctx.constraints, + ) + results = resolver.find_all_matching_from_provider(provider, pinned_req) + wheel_url, _ = results[0] + wheelfile_name = pathlib.Path(urlparse(wheel_url).path) + pbi = ctx.package_build_info(req) + expected_build_tag = pbi.build_tag(resolved_version) + logger.info(f"has expected build tag {expected_build_tag}") + changelogs = pbi.get_changelog(resolved_version) + logger.debug(f"has change logs {changelogs}") + + _, _, build_tag, _ = wheels.extract_info_from_wheel_file(req, wheelfile_name) + if expected_build_tag and expected_build_tag != build_tag: + logger.info( + f"found wheel for {resolved_version} in cache but build tag does not match. Got {build_tag} but expected {expected_build_tag}" + ) + return None, None + + cached_wheel = wheels.download_wheel( + req=req, wheel_url=wheel_url, output_directory=ctx.wheels_downloads + ) + if cache_wheel_server_url != ctx.wheel_server_url: + server.update_wheel_mirror(ctx) + logger.info("found built wheel on cache server") + unpack_dir = _extract_build_reqs_from_wheel( + ctx.work_dir, req, resolved_version, cached_wheel + ) + return cached_wheel, unpack_dir + except ResolverException: + logger.info( + f"did not find wheel for {resolved_version} in {cache_wheel_server_url}" + ) + return None, None + except requests.exceptions.RequestException as err: + logger.warning( + f"network error checking wheel cache for {resolved_version} " + f"at {cache_wheel_server_url}: {err}" + ) + return None, None + except Exception as err: + logger.warning( + f"unexpected error checking wheel cache for {resolved_version} " + f"at {cache_wheel_server_url}: {err}" + ) + return None, None + + +def _find_cached_wheel( + ctx: context.WorkContext, + cache_wheel_server_url: str | None, + req: Requirement, + resolved_version: Version, +) -> tuple[pathlib.Path | None, pathlib.Path | None]: + """Look for cached wheel in 3 locations (thread-safe, no Bootstrapper state). + + Checks for cached wheels in order: + 1. wheels_build directory (previously built) + 2. wheels_downloads directory (previously downloaded) + 3. Cache server (remote cache) + + Returns: + Tuple of (cached_wheel_filename, unpacked_cached_wheel). + Both None if no cache hit. + """ + cached_wheel, unpacked = _look_for_existing_wheel( + ctx, req, resolved_version, ctx.wheels_build + ) + if cached_wheel: + return cached_wheel, unpacked + + cached_wheel, unpacked = _look_for_existing_wheel( + ctx, req, resolved_version, ctx.wheels_downloads + ) + if cached_wheel: + return cached_wheel, unpacked + + cached_wheel, unpacked = _download_wheel_from_cache( + ctx, cache_wheel_server_url, req, resolved_version + ) + if cached_wheel: + return cached_wheel, unpacked + + return None, None + + +def _bg_prepare_prebuilt( + ctx: context.WorkContext, + req: Requirement, + req_type: RequirementType, + resolved_version: Version, + wheel_url: str, +) -> PreparedSourceData: + """Background-safe prebuilt download: no Bootstrapper state accessed.""" + # Thread-safe: paths include {req.name}-{resolved_version} (unique per package), + # mkdir uses exist_ok=True (atomic), and update_wheel_mirror() is already locked. + logger.info(f"using pre-built wheel for {req_type} requirement") + wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt) + unpack_dir = _create_unpack_dir(ctx.work_dir, req, resolved_version) + server.update_wheel_mirror(ctx) + return PreparedSourceData(wheel_filename=wheel_filename, unpack_dir=unpack_dir) diff --git a/src/fromager/bootstrapper/_complete.py b/src/fromager/bootstrapper/_complete.py new file mode 100644 index 00000000..e0a9ed8d --- /dev/null +++ b/src/fromager/bootstrapper/_complete.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import typing + +from ._phase import Phase +from ._types import BootstrapPhase + +if typing.TYPE_CHECKING: + from ._bootstrapper import Bootstrapper + + +class Complete(Phase): + """Clean up build directories after a package has been fully processed. + + Removes the unpacked source tree and build environment created during + ``PrepareSource`` and ``PrepareBuild`` to free disk space. This is the + terminal phase for every package: ``run()`` always returns an empty list. + + Next phase: none (terminal). + """ + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.COMPLETE + tracks_why: typing.ClassVar[bool] = True + + def run(self, bt: Bootstrapper) -> list[Phase]: + """COMPLETE phase: clean up build directories. + + Returns: + Empty list (processing finished for this item). + """ + wi = self.work_item + if wi.build_result is not None: + bt.ctx.clean_build_dirs( + wi.build_result.sdist_root_dir, + wi.build_result.build_env, + ) + return [] diff --git a/src/fromager/bootstrapper/_phase.py b/src/fromager/bootstrapper/_phase.py new file mode 100644 index 00000000..f7f584c2 --- /dev/null +++ b/src/fromager/bootstrapper/_phase.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import abc +import concurrent.futures +import logging +import typing + +from ._types import BootstrapPhase +from ._work_item import WorkItem + +if typing.TYPE_CHECKING: + from ._bootstrapper import Bootstrapper + +logger = logging.getLogger(__name__) + + +class Phase(abc.ABC): + """Abstract base for items pushed onto the bootstrap stack. + + Each subclass encodes one phase of the bootstrap pipeline. + Wraps a ``WorkItem`` (accumulated per-package state) and implements + the processing logic for that phase in ``run()``. + """ + + phase: typing.ClassVar[BootstrapPhase] + tracks_why: typing.ClassVar[bool] = True + + def __init_subclass__(cls, **kwargs: typing.Any) -> None: + super().__init_subclass__(**kwargs) + # Enforce that every concrete subclass defines `phase`. Abstract + # subclasses (those that still have unimplemented abstract methods) + # are allowed to omit it; they will be checked when their own concrete + # subclasses are defined. + # + # Note: ABCMeta sets __abstractmethods__ *after* __init_subclass__ + # runs, so we cannot rely on it here. Instead we inspect the class's + # MRO for any attribute still marked as abstract. + is_abstract = any( + getattr(getattr(cls, attr, None), "__isabstractmethod__", False) + for attr in dir(cls) + ) + if not is_abstract and "phase" not in cls.__dict__: + raise TypeError(f"{cls.__name__} must define the 'phase' class attribute") + + def __init__(self, work_item: WorkItem) -> None: + self.work_item = work_item + self.bg_future: concurrent.futures.Future[typing.Any] | None = None + + @abc.abstractmethod + def run(self, bt: Bootstrapper) -> list[Phase]: ... + + @property + def requires_exclusive_run(self) -> bool: + """Return True if this item must run without concurrent background I/O. + + When True, the bootstrap loop drains the background thread pool before + calling ``run()``. Override in subclasses that require exclusive access. + """ + return False + + def background_work( + self, bt: Bootstrapper + ) -> typing.Callable[[], typing.Any] | None: + """Return a zero-argument callable for background I/O, or None. + + Override in subclasses that need background prefetching. + ``bt`` is provided so subclasses can capture Bootstrapper state + (e.g. resolver, ctx) into the returned closure without storing + a circular reference on the item itself. + """ + return None + + def __str__(self) -> str: + """Human-readable representation: ``"()"``.""" + wi = self.work_item + return f"{type(self).__name__}({wi.req})" + + def as_json(self) -> dict[str, typing.Any]: + """Return a JSON-serialisable dict for stack-state recording.""" + wi = self.work_item + return { + "req": str(wi.req), + "req_type": str(wi.req_type), + "phase": str(self.phase), + "resolved_version": str(wi.resolved_version) + if wi.resolved_version is not None + else None, + "source_url": wi.source_url, + "build_sdist_only": wi.build_sdist_only, + "why": [ + {"req_type": str(rt), "req": str(r), "version": str(v)} + for rt, r, v in wi.why_snapshot + ], + "parent": ( + {"req": str(wi.parent[0]), "version": str(wi.parent[1])} + if wi.parent + else None + ), + "build_system_deps": sorted(str(r) for r in wi.build_system_deps), + "build_backend_deps": sorted(str(r) for r in wi.build_backend_deps), + "build_sdist_deps": sorted(str(r) for r in wi.build_sdist_deps), + } diff --git a/src/fromager/bootstrapper/_prepare_build.py b/src/fromager/bootstrapper/_prepare_build.py new file mode 100644 index 00000000..2426d2b9 --- /dev/null +++ b/src/fromager/bootstrapper/_prepare_build.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import logging +import typing + +from .. import dependencies +from ..requirements_file import RequirementType +from ._build import Build +from ._phase import Phase +from ._types import BootstrapPhase + +if typing.TYPE_CHECKING: + from ._bootstrapper import Bootstrapper + +logger = logging.getLogger(__name__) + + +class PrepareBuild(Phase): + """Install build-system deps and discover build-backend and build-sdist deps. + + Installs the build-system wheels (already available from DFS processing), + then queries the build backend for its own dependencies and the + sdist-generation dependencies. Dependencies already satisfied by the + installed build-system set are filtered out to avoid resolving a + conflicting version (see :issue:`1194`). + + Next phase: ``Build`` + one ``Resolve`` per build-backend dependency + + one ``Resolve`` per build-sdist dependency. + """ + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PREPARE_BUILD + tracks_why: typing.ClassVar[bool] = True + + def run(self, bt: Bootstrapper) -> list[Phase]: + """PREPARE_BUILD phase: install system deps, get backend/sdist deps. + + Build-backend and build-sdist dependencies that are already satisfied + by a resolved build-system dependency reuse that version instead of + resolving independently (see :issue:`1194`). + + Returns: + [Build, *backend_dep_items, *sdist_dep_items]. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.build_env is not None + assert wi.sdist_root_dir is not None + + # Install build system deps (their wheels exist from DFS processing) + wi.build_env.install(wi.build_system_deps) + + # Get build backend dependencies + wi.build_backend_deps = dependencies.get_build_backend_dependencies( + ctx=bt.ctx, + req=wi.req, + version=wi.resolved_version, + sdist_root_dir=wi.sdist_root_dir, + build_env=wi.build_env, + ) + + # Get build sdist dependencies + wi.build_sdist_deps = dependencies.get_build_sdist_dependencies( + ctx=bt.ctx, + req=wi.req, + version=wi.resolved_version, + sdist_root_dir=wi.sdist_root_dir, + build_env=wi.build_env, + ) + + # Filter out deps already satisfied by build-system dependencies + # to avoid resolving to a different (typically newer) version. + resolved_build_sys = bt.get_resolved_build_system_versions(wi) + parent = (wi.req, wi.resolved_version) + wi.build_backend_deps = bt.filter_deps_satisfied_by_build_system( + wi.build_backend_deps, + resolved_build_sys, + RequirementType.BUILD_BACKEND, + parent, + ) + wi.build_sdist_deps = bt.filter_deps_satisfied_by_build_system( + wi.build_sdist_deps, + resolved_build_sys, + RequirementType.BUILD_SDIST, + parent, + ) + + backend_items: list[Phase] = bt.create_unresolved_work_items( + wi.build_backend_deps, + RequirementType.BUILD_BACKEND, + wi.req, + wi.resolved_version, + ) + sdist_items: list[Phase] = bt.create_unresolved_work_items( + wi.build_sdist_deps, + RequirementType.BUILD_SDIST, + wi.req, + wi.resolved_version, + ) + dep_items = backend_items + sdist_items + + return [Build(wi)] + dep_items diff --git a/src/fromager/bootstrapper/_prepare_source.py b/src/fromager/bootstrapper/_prepare_source.py new file mode 100644 index 00000000..454d3ca3 --- /dev/null +++ b/src/fromager/bootstrapper/_prepare_source.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import logging +import typing + +from packaging.requirements import Requirement +from packaging.version import Version + +from .. import build_environment, dependencies, sources +from ..log import req_ctxvar_context +from ..requirements_file import RequirementType, SourceType +from . import _cache +from ._phase import Phase +from ._prepare_build import PrepareBuild +from ._process_install_deps import ProcessInstallDeps +from ._types import BootstrapPhase, PreparedSourceData, SourceBuildResult + +if typing.TYPE_CHECKING: + from .. import context + from ._bootstrapper import Bootstrapper + +logger = logging.getLogger(__name__) + + +def _bg_prepare_source( + ctx: context.WorkContext, + cache_wheel_server_url: str | None, + req: Requirement, + resolved_version: Version, + source_url: str, +) -> PreparedSourceData: + """Background-safe source download+unpack: no Bootstrapper state accessed.""" + # Thread-safe: _seen_requirements in the main thread prevents the same + # package from being submitted to the thread pool more than once. + # Paths from download_source() and prepare_source() include {name}-{version}, + # making them unique across concurrent threads processing different packages. + logger.info("preparing source") + cached_wheel, unpacked = _cache._find_cached_wheel( + ctx, cache_wheel_server_url, req, resolved_version + ) + if unpacked is not None: + return PreparedSourceData( + sdist_root_dir=unpacked / unpacked.stem, + cached_wheel_filename=cached_wheel, + ) + source_filename = sources.download_source( + ctx=ctx, req=req, version=resolved_version, download_url=source_url + ) + sdist_root_dir = sources.prepare_source( + ctx=ctx, req=req, source_filename=source_filename, version=resolved_version + ) + return PreparedSourceData(sdist_root_dir=sdist_root_dir) + + +class PrepareSource(Phase): + """Download the source distribution or prebuilt wheel and set up build-system deps. + + The download runs in a background thread (via ``background_work``). + For prebuilt packages the background task fetches the wheel directly and + the build phases are skipped entirely. For source builds the background + task downloads and unpacks the sdist; ``run()`` then reads build-system + dependencies from the unpacked tree. + + Next phase: + - Prebuilt wheel: ``ProcessInstallDeps``. + - Source build: ``PrepareBuild`` + one ``Resolve`` per build-system dependency. + """ + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PREPARE_SOURCE + tracks_why: typing.ClassVar[bool] = True + + def background_work( + self, bt: Bootstrapper + ) -> typing.Callable[[], typing.Any] | None: + """Return closure for background source download or prebuilt fetch.""" + wi = self.work_item + assert wi.resolved_version is not None + assert wi.source_url is not None + ctx = bt.ctx + cache_wheel_server_url = bt.cache_wheel_server_url + req = wi.req + req_type = wi.req_type + resolved_version = wi.resolved_version + source_url = wi.source_url + + if wi.pbi_pre_built: + + def do_prepare_prebuilt() -> PreparedSourceData: + with req_ctxvar_context(req, resolved_version): + return _cache._bg_prepare_prebuilt( + ctx, req, req_type, resolved_version, source_url + ) + + return do_prepare_prebuilt + + def do_prepare_source() -> PreparedSourceData: + with req_ctxvar_context(req, resolved_version): + return _bg_prepare_source( + ctx, cache_wheel_server_url, req, resolved_version, source_url + ) + + return do_prepare_source + + def run(self, bt: Bootstrapper) -> list[Phase]: + """PREPARE_SOURCE phase: download source or prebuilt, get build system deps. + + Uses background I/O result from ``self.bg_future`` when available, + falling back to inline I/O otherwise. + + Returns: + Prebuilt: [ProcessInstallDeps] (skip build phases). + Source: [PrepareBuild, *build_system_dep_items]. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.source_url is not None + + # bg_future is always set for PREPARE_SOURCE items (see _push_items). + # bg_future.result() blocks until done and re-raises any background exception. + assert self.bg_future is not None + prepared: PreparedSourceData = self.bg_future.result() + + constraint = bt.ctx.constraints.get_constraint(wi.req.name) + if constraint: + logger.info( + f"incoming requirement {wi.req} matches constraint " + f"{constraint}. Will apply both." + ) + + if wi.pbi_pre_built: + # Background task already downloaded the prebuilt wheel + assert prepared.wheel_filename is not None + assert prepared.unpack_dir is not None + wi.build_result = SourceBuildResult( + wheel_filename=prepared.wheel_filename, + sdist_filename=None, + unpack_dir=prepared.unpack_dir, + sdist_root_dir=None, + build_env=None, + source_type=SourceType.PREBUILT, + ) + return [ProcessInstallDeps(wi)] + + # Source build path: background task already downloaded and prepared the source + assert prepared.sdist_root_dir is not None + sdist_root_dir = prepared.sdist_root_dir + wi.cached_wheel_filename = prepared.cached_wheel_filename + + assert sdist_root_dir is not None + + if sdist_root_dir.parent.parent != bt.ctx.work_dir: + raise ValueError(f"'{sdist_root_dir}/../..' should be {bt.ctx.work_dir}") + wi.sdist_root_dir = sdist_root_dir + wi.unpack_dir = sdist_root_dir.parent + + wi.build_env = build_environment.BuildEnvironment( + ctx=bt.ctx, + parent_dir=sdist_root_dir.parent, + ) + + # Get build system dependencies + wi.build_system_deps = dependencies.get_build_system_dependencies( + ctx=bt.ctx, + req=wi.req, + version=wi.resolved_version, + sdist_root_dir=sdist_root_dir, + ) + + dep_items: list[Phase] = bt.create_unresolved_work_items( + wi.build_system_deps, + RequirementType.BUILD_SYSTEM, + wi.req, + wi.resolved_version, + ) + + return [PrepareBuild(wi)] + dep_items diff --git a/src/fromager/bootstrapper/_process_install_deps.py b/src/fromager/bootstrapper/_process_install_deps.py new file mode 100644 index 00000000..a6ed4b58 --- /dev/null +++ b/src/fromager/bootstrapper/_process_install_deps.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import logging +import pathlib +import typing + +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name +from packaging.version import Version + +from .. import build_environment, dependencies, hooks +from ..requirements_file import RequirementType +from ._complete import Complete +from ._phase import Phase +from ._types import BootstrapPhase + +if typing.TYPE_CHECKING: + from .. import context + from ._bootstrapper import Bootstrapper + +logger = logging.getLogger(__name__) + + +def _get_install_dependencies( + ctx: context.WorkContext, + req: Requirement, + resolved_version: Version, + wheel_filename: pathlib.Path | None, + sdist_filename: pathlib.Path | None, + sdist_root_dir: pathlib.Path | None, + build_env: build_environment.BuildEnvironment | None, + unpack_dir: pathlib.Path | None, +) -> list[Requirement]: + """Extract install dependencies from a built wheel or sdist. + + Returns: + List of install requirements. + + Raises: + RuntimeError: If both wheel_filename and sdist_filename are None. + """ + if wheel_filename is not None: + assert unpack_dir is not None + logger.debug( + "get install dependencies of wheel %s", + wheel_filename.name, + ) + return list( + dependencies.get_install_dependencies_of_wheel( + req=req, + wheel_filename=wheel_filename, + requirements_file_dir=unpack_dir, + ) + ) + elif sdist_filename is not None: + assert sdist_root_dir is not None + assert build_env is not None + logger.debug( + "get install dependencies of sdist from directory %s", + sdist_root_dir, + ) + return list( + dependencies.get_install_dependencies_of_sdist( + ctx=ctx, + req=req, + version=resolved_version, + sdist_root_dir=sdist_root_dir, + build_env=build_env, + ) + ) + else: + raise RuntimeError("wheel_filename and sdist_filename are None") + + +class ProcessInstallDeps(Phase): + """Run post-bootstrap hooks, extract install deps, and record the build order. + + Runs any configured post-bootstrap hooks for the completed package (errors + are fatal in normal mode; recorded as warnings in test mode). Extracts + install-time dependencies from the built wheel or sdist. Appends the + package to the persistent build-order file via ``Bootstrapper.add_to_build_order``. + + Next phase: ``Complete`` + one ``Resolve`` per install dependency. + """ + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.PROCESS_INSTALL_DEPS + tracks_why: typing.ClassVar[bool] = True + + def run(self, bt: Bootstrapper) -> list[Phase]: + """PROCESS_INSTALL_DEPS phase: hooks, extract deps, build order. + + Returns: + [Complete, *install_dep_items]. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.source_url is not None + assert wi.build_result is not None + + # Run post-bootstrap hooks (non-fatal in test mode) + try: + hooks.run_post_bootstrap_hooks( + ctx=bt.ctx, + req=wi.req, + dist_name=canonicalize_name(wi.req.name), + dist_version=str(wi.resolved_version), + sdist_filename=wi.build_result.sdist_filename, + wheel_filename=wi.build_result.wheel_filename, + ) + except Exception as hook_error: + if not bt.test_mode: + raise + bt.record_test_mode_failure( + wi.req, + str(wi.resolved_version), + hook_error, + "hook", + "warning", + ) + + # Extract install dependencies (non-fatal in test mode) + try: + install_dependencies = _get_install_dependencies( + ctx=bt.ctx, + req=wi.req, + resolved_version=wi.resolved_version, + wheel_filename=wi.build_result.wheel_filename, + sdist_filename=wi.build_result.sdist_filename, + sdist_root_dir=wi.build_result.sdist_root_dir, + build_env=wi.build_result.build_env, + unpack_dir=wi.build_result.unpack_dir, + ) + except Exception as dep_error: + if not bt.test_mode: + raise + bt.record_test_mode_failure( + wi.req, + str(wi.resolved_version), + dep_error, + "dependency_extraction", + "warning", + ) + install_dependencies = [] + + logger.debug( + "install dependencies: %s", + ", ".join(sorted(str(r) for r in install_dependencies)), + ) + + pbi = bt.ctx.package_build_info(wi.req) + constraint = bt.ctx.constraints.get_constraint(wi.req.name) + bt.add_to_build_order( + req=wi.req, + version=wi.resolved_version, + source_url=wi.source_url, + source_type=wi.build_result.source_type, + prebuilt=pbi.pre_built, + constraint=constraint, + ) + + dep_items: list[Phase] = bt.create_unresolved_work_items( + install_dependencies, + RequirementType.INSTALL, + wi.req, + wi.resolved_version, + ) + + return [Complete(wi)] + dep_items diff --git a/src/fromager/bootstrapper/_resolve.py b/src/fromager/bootstrapper/_resolve.py new file mode 100644 index 00000000..51dc541c --- /dev/null +++ b/src/fromager/bootstrapper/_resolve.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import logging +import typing + +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name +from packaging.version import Version + +from .. import bootstrap_requirement_resolver +from ..log import req_ctxvar_context +from ..requirements_file import RequirementType +from . import _cache +from ._phase import Phase +from ._start import Start +from ._types import BootstrapPhase +from ._work_item import WorkItem + +if typing.TYPE_CHECKING: + from ._bootstrapper import Bootstrapper + +logger = logging.getLogger(__name__) + + +def _bg_resolve( + bg_resolver: bootstrap_requirement_resolver.BootstrapRequirementResolver, + req: Requirement, + req_type: RequirementType, + parent_req: Requirement | None, + return_all_versions: bool, +) -> list[tuple[str, Version]]: + """Background-safe resolution: no Bootstrapper state accessed.""" + logger.info(f"{BootstrapPhase.RESOLVE} for {req_type} requirement") + return bg_resolver.resolve( + req=req, + req_type=req_type, + parent_req=parent_req, + return_all_versions=return_all_versions, + ) + + +class Resolve(Phase): + """Resolve a requirement's available version(s) and fan out to ``Start`` items. + + Resolution runs in a background thread (via ``background_work``); ``run()`` + blocks until the result is ready. In normal mode the highest matching + version is used; in ``multiple_versions`` mode every candidate version is + returned, minus any that previously failed or whose wheels are already + cached. + + Next phase: one ``Start`` item per version that still needs processing. + """ + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.RESOLVE + tracks_why: typing.ClassVar[bool] = False + + def background_work( + self, bt: Bootstrapper + ) -> typing.Callable[[], typing.Any] | None: + """Return closure that calls ``_bg_resolve`` in a thread.""" + bg_resolver = bt.resolver + req = self.work_item.req + req_type = self.work_item.req_type + parent_req = ( + self.work_item.why_snapshot[-1][1] if self.work_item.why_snapshot else None + ) + return_all = bt.multiple_versions + + def do_resolve() -> list[tuple[str, Version]]: + with req_ctxvar_context(req): + return _bg_resolve(bg_resolver, req, req_type, parent_req, return_all) + + return do_resolve + + def run(self, bt: Bootstrapper) -> list[Phase]: + """RESOLVE phase: resolve versions and expand into Start items. + + Centralizes version resolution so all dependencies are expanded + uniformly. In multiple_versions mode, filters out versions that + already failed in this run and versions whose wheels are already + cached to avoid redundant builds and transitive dependency + processing. + + Returns: + One Start item per resolved version that needs building. + Empty list if all versions are already cached. + """ + assert self.bg_future is not None + # bg_future.result() blocks until the background resolution completes, + # then returns the result or re-raises any exception from the background. + resolved_versions = self.bg_future.result() + if not resolved_versions: + raise RuntimeError( + f"Could not resolve any versions for {self.work_item.req}" + ) + + if bt.multiple_versions: + pkg_name = canonicalize_name(self.work_item.req.name) + resolved_versions = [ + (url, ver) + for url, ver in resolved_versions + if not bt.has_failed_version(pkg_name, ver) + ] + if not resolved_versions: + raise RuntimeError( + f"Could not resolve any versions for {self.work_item.req}" + f" (all candidates failed previously)" + ) + + logger.info( + f"resolved {len(resolved_versions)} version(s) for {self.work_item.req}" + ) + filtered: list[tuple[str, Version]] = [] + for source_url, version in resolved_versions: + cached_wheel, _ = _cache._find_cached_wheel( + bt.ctx, bt.cache_wheel_server_url, self.work_item.req, version + ) + if cached_wheel: + logger.info( + f"{self.work_item.req.name}=={version}: wheel already cached " + f"at {cached_wheel.name}, skipping" + ) + else: + filtered.append((source_url, version)) + if not filtered: + # Always process the highest version (first in + # resolved_versions) so new transitive dependencies + # are discovered even when every wheel is cached. + logger.info( + f"all versions of {self.work_item.req.name} already cached, " + f"keeping highest version {resolved_versions[0][1]} " + f"for dependency discovery" + ) + filtered.append(resolved_versions[0]) + resolved_versions = filtered + + # Build list so highest version ends up on top of the stack + # (last element after extend) and is processed first. + items: list[Phase] = [] + for source_url, version in reversed(resolved_versions): + items.append( + Start( + WorkItem( + req=self.work_item.req, + req_type=self.work_item.req_type, + why_snapshot=list(self.work_item.why_snapshot), + parent=self.work_item.parent, + source_url=source_url, + resolved_version=version, + ) + ) + ) + return items diff --git a/src/fromager/bootstrapper/_start.py b/src/fromager/bootstrapper/_start.py new file mode 100644 index 00000000..2db1eac9 --- /dev/null +++ b/src/fromager/bootstrapper/_start.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import logging +import typing + +from ..requirements_file import RequirementType +from ._phase import Phase +from ._prepare_source import PrepareSource +from ._types import BootstrapPhase + +if typing.TYPE_CHECKING: + from ._bootstrapper import Bootstrapper + +logger = logging.getLogger(__name__) + + +class Start(Phase): + """Record a resolved requirement in the dependency graph and deduplicate. + + Adds the ``(parent → req)`` edge to the dependency graph, then checks + whether this ``(req, version)`` pair has already been processed. Duplicate + requirements are silently dropped; new ones proceed to source preparation. + ``tracks_why`` is ``False`` so graph additions happen before the why-stack + is updated. + + Next phase: ``PrepareSource`` (new requirement) or ``[]`` (already seen). + """ + + phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.START + tracks_why: typing.ClassVar[bool] = False + + def run(self, bt: Bootstrapper) -> list[Phase]: + """START phase: add to graph, check if already seen. + + _track_why is a no-op for this phase (tracks_why is False), + matching the original behavior where graph addition and + seen-check happen before pushing onto the why stack. + + Returns: + Empty list if already seen (nothing to do). + [PrepareSource] if this is new work. + """ + wi = self.work_item + assert wi.resolved_version is not None + assert wi.source_url is not None + + # Add to graph (skip TOP_LEVEL, already added in _resolve_and_add_top_level) + if wi.req_type != RequirementType.TOP_LEVEL: + bt.add_to_graph( + wi.req, + wi.req_type, + wi.resolved_version, + wi.source_url, + wi.parent, + ) + + wi.build_sdist_only = bt.sdist_only and not wi.is_build_requirement_context() + + if bt.has_been_seen(wi.req, wi.resolved_version, wi.build_sdist_only): + logger.debug( + f"redundant {wi.req_type} dependency {wi.req} " + f"({wi.resolved_version}, sdist_only={wi.build_sdist_only}) " + f"for {bt.explain}" + ) + return [] + bt.mark_as_seen(wi.req, wi.resolved_version, wi.build_sdist_only) + + logger.info( + f"new {wi.req_type} dependency {wi.req} resolves to {wi.resolved_version}" + ) + + # Must set pbi_pre_built before constructing PrepareSource so that + # PrepareSource.background_work() immediately sees the correct value. + pbi = bt.ctx.package_build_info(wi.req) + wi.pbi_pre_built = pbi.pre_built + wi.exclusive_build = pbi.exclusive_build + return [PrepareSource(wi)] diff --git a/src/fromager/bootstrapper/_types.py b/src/fromager/bootstrapper/_types.py new file mode 100644 index 00000000..cd395006 --- /dev/null +++ b/src/fromager/bootstrapper/_types.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import dataclasses +import logging +import pathlib +import typing +from enum import StrEnum + +from packaging.utils import NormalizedName + +from .. import build_environment, threading_utils +from ..requirements_file import SourceType + +logger = logging.getLogger(__name__) + +# package name, extras, version, sdist/wheel +SeenKey = tuple[NormalizedName, tuple[str, ...], str, typing.Literal["sdist", "wheel"]] + +_DEFAULT_BG_THREADS: int = max(1, threading_utils.get_cpu_count() // 2) + + +@dataclasses.dataclass +class SourceBuildResult: + """Result of building or downloading a package. + + Captures the output artifacts from either a source build or + prebuilt wheel download, used across bootstrap phases. + """ + + wheel_filename: pathlib.Path | None + sdist_filename: pathlib.Path | None + unpack_dir: pathlib.Path + sdist_root_dir: pathlib.Path | None + build_env: build_environment.BuildEnvironment | None + source_type: SourceType + + +@dataclasses.dataclass +class PreparedSourceData: + """Result of background I/O pre-fetching returned to the main thread. + + Fields are set in one of three combinations depending on the result type: + + - Source (no cache hit): only ``sdist_root_dir`` is set. + - Source (cache hit): both ``sdist_root_dir`` and ``cached_wheel_filename`` are set. + - Prebuilt wheel: both ``wheel_filename`` and ``unpack_dir`` are set. + """ + + # Source path: set after download+unpack OR cache hit + sdist_root_dir: pathlib.Path | None = None + # Source path: set when the result came from the wheel cache + cached_wheel_filename: pathlib.Path | None = None + # Prebuilt path: downloaded wheel file + wheel_filename: pathlib.Path | None = None + # Prebuilt path: unpack directory (created by mkdir) + unpack_dir: pathlib.Path | None = None + + +# Valid failure types for test mode error recording +FailureType = typing.Literal["resolution", "bootstrap", "hook", "dependency_extraction"] + + +class FailureRecord(typing.TypedDict): + """Record of a package that failed during bootstrap in test mode. + + Attributes: + package: The package name that failed. + version: The resolved version (None if resolution failed). + exception_type: The exception class name. + exception_message: The exception message string. + failure_type: Category of failure for analysis. + """ + + package: str + version: str | None + exception_type: str + exception_message: str + failure_type: FailureType + + +class BootstrapPhase(StrEnum): + """Processing phases for iterative bootstrap. + + All packages: RESOLVE -> START -> ... + Source packages: ... -> PREPARE_SOURCE -> PREPARE_BUILD -> BUILD + -> PROCESS_INSTALL_DEPS -> COMPLETE. + Prebuilt packages: ... -> PREPARE_SOURCE -> PROCESS_INSTALL_DEPS -> COMPLETE. + """ + + RESOLVE = "resolve" + START = "start" + PREPARE_SOURCE = "prepare-source" + PREPARE_BUILD = "prepare-build" + BUILD = "build" + PROCESS_INSTALL_DEPS = "process-install-deps" + COMPLETE = "complete" diff --git a/src/fromager/bootstrapper/_work_item.py b/src/fromager/bootstrapper/_work_item.py new file mode 100644 index 00000000..30163394 --- /dev/null +++ b/src/fromager/bootstrapper/_work_item.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import dataclasses +import logging +import pathlib + +from packaging.requirements import Requirement +from packaging.version import Version + +from .. import build_environment +from ..requirements_file import RequirementType +from ._types import SourceBuildResult + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class WorkItem: + """A unit of work in the iterative bootstrap loop. + + Carries identity fields set at creation time and accumulated state + populated across phases as processing advances. The current phase is + encoded by the ``Phase`` subclass wrapping this object. + + Items enter at the RESOLVE phase with only req and req_type set. + The RESOLVE phase populates source_url and resolved_version, then + creates new items at the START phase for each resolved version. + """ + + # Identity (set at creation) + req: Requirement + req_type: RequirementType + why_snapshot: list[tuple[RequirementType, Requirement, Version]] + parent: tuple[Requirement, Version] | None = None + + # Populated by RESOLVE phase (None until then) + source_url: str | None = None + resolved_version: Version | None = None + + build_sdist_only: bool = False + + # Accumulated state (populated during phases) + build_env: build_environment.BuildEnvironment | None = None + sdist_root_dir: pathlib.Path | None = None + unpack_dir: pathlib.Path | None = None + cached_wheel_filename: pathlib.Path | None = None + build_result: SourceBuildResult | None = None + pbi_pre_built: bool = False + exclusive_build: bool = False + build_system_deps: set[Requirement] = dataclasses.field(default_factory=set) + build_backend_deps: set[Requirement] = dataclasses.field(default_factory=set) + build_sdist_deps: set[Requirement] = dataclasses.field(default_factory=set) + + def is_build_requirement_context(self) -> bool: + """Return True if this item is being processed as part of a build requirement. + + A package is a build dependency if its own requirement type is + build_system, build_backend, or build_sdist, OR if it is an install + requirement of something that is itself a build dependency (checked + by walking the ``why_snapshot`` chain). + """ + if self.req_type.is_build_requirement: + logger.debug(f"is itself a build requirement: {self.req_type}") + return True + if not self.req_type.is_install_requirement: + logger.debug( + "is not an install requirement, not checking dependency chain for a build requirement" + ) + return False + for req_type, req, resolved_version in reversed(self.why_snapshot): + if req_type.is_build_requirement: + logger.debug( + f"is a build requirement because {req_type} dependency {req} ({resolved_version}) depends on it" + ) + return True + logger.debug("is not a build requirement") + return False diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 3a30ec6f..0a3c9f7c 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -894,3 +894,43 @@ def test_resolve_version_from_git_url_invalid_scheme( bs = bootstrapper.Bootstrapper(tmp_context) with pytest.raises(ValueError, match="unsupported VCS URL scheme"): bs._resolve_version_from_git_url(req) + + +def test_resolve_version_from_git_url_creates_parent_dir_on_first_clone( + tmp_context: context.WorkContext, +) -> None: + """working_src_dir.parent is created before shutil.move on first clone. + + When the URL has no version ref, the version is discovered after cloning, + so working_src_dir is computed inside the clone block. Its parent directory + does not exist yet and must be created before shutil.move is called. + """ + req = Requirement("test-pkg @ git+https://pkg.test/repo.git") + + def fake_download_git_source( + *, + ctx: context.WorkContext, + req: Requirement, + url_to_clone: str, + destination_dir: pathlib.Path, + ref: str, + ) -> None: + destination_dir.mkdir(parents=True, exist_ok=True) + + with ( + patch( + "fromager.sources.download_git_source", + side_effect=fake_download_git_source, + ), + patch( + "fromager.bootstrapper.Bootstrapper._get_version_from_package_metadata", + return_value=Version("1.0.0"), + ), + ): + bs = bootstrapper.Bootstrapper(tmp_context) + src_dir, version = bs._resolve_version_from_git_url(req) + + expected = tmp_context.work_dir / "test-pkg-1.0.0" / "test-pkg-1.0.0" + assert pathlib.Path(src_dir) == expected + assert expected.exists() + assert version == Version("1.0.0") diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index bc55f2f1..797121f5 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -12,17 +12,60 @@ from resolvelib.resolvers import ResolverException from fromager import bootstrapper, log +from fromager.bootstrapper._build import Build +from fromager.bootstrapper._cache import ( + _bg_prepare_prebuilt, + _download_wheel_from_cache, + _find_cached_wheel, +) +from fromager.bootstrapper._phase import Phase +from fromager.bootstrapper._prepare_source import ( + PrepareSource, + _bg_prepare_source, +) +from fromager.bootstrapper._process_install_deps import ( + ProcessInstallDeps, + _get_install_dependencies, +) +from fromager.bootstrapper._resolve import Resolve +from fromager.bootstrapper._start import Start +from fromager.bootstrapper._types import ( + BootstrapPhase, + SourceBuildResult, +) +from fromager.bootstrapper._work_item import WorkItem from fromager.context import WorkContext from fromager.requirements_file import RequirementType, SourceType +def test_phase_subclass_without_phase_attribute_raises() -> None: + """Concrete Phase subclass missing 'phase' raises TypeError at class definition.""" + with pytest.raises(TypeError, match="must define the 'phase' class attribute"): + + class _BadPhase(Phase): + def run(self, bt: typing.Any) -> list[Phase]: # type: ignore[override] + return [] + + +def test_phase_abstract_subclass_without_phase_attribute_is_allowed() -> None: + """Abstract Phase subclass may omit 'phase' without error.""" + import abc + + class _AbstractMiddle(Phase): + @abc.abstractmethod + def helper(self) -> None: ... + + # Defining _AbstractMiddle itself must not raise. + assert _AbstractMiddle.__abstractmethods__ + + def test_seen(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) req = Requirement("testdist") version = Version("1.2") - assert not bt._has_been_seen(req, version) - bt._mark_as_seen(req, version) - assert bt._has_been_seen(req, version) + assert not bt.has_been_seen(req, version) + bt.mark_as_seen(req, version) + assert bt.has_been_seen(req, version) def test_seen_extras(tmp_context: WorkContext) -> None: @@ -30,57 +73,57 @@ def test_seen_extras(tmp_context: WorkContext) -> None: req2 = Requirement("testdist[extra]") version = Version("1.2") bt = bootstrapper.Bootstrapper(tmp_context) - assert not bt._has_been_seen(req1, version) - bt._mark_as_seen(req1, version) - assert bt._has_been_seen(req1, version) - assert not bt._has_been_seen(req2, version) - bt._mark_as_seen(req2, version) - assert bt._has_been_seen(req1, version) - assert bt._has_been_seen(req2, version) + assert not bt.has_been_seen(req1, version) + bt.mark_as_seen(req1, version) + assert bt.has_been_seen(req1, version) + assert not bt.has_been_seen(req2, version) + bt.mark_as_seen(req2, version) + assert bt.has_been_seen(req1, version) + assert bt.has_been_seen(req2, version) def test_seen_name_canonicalization(tmp_context: WorkContext) -> None: req = Requirement("flit_core") version = Version("1.2") bt = bootstrapper.Bootstrapper(tmp_context) - assert not bt._has_been_seen(req, version) - bt._mark_as_seen(req, version) - assert bt._has_been_seen(req, version) + assert not bt.has_been_seen(req, version) + bt.mark_as_seen(req, version) + assert bt.has_been_seen(req, version) def test_seen_requirements_sdist(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) req = Requirement("testdist") version = Version("1.2") - assert not bt._has_been_seen(req, version, sdist_only=False) - assert not bt._has_been_seen(req, version, sdist_only=True) + assert not bt.has_been_seen(req, version, sdist_only=False) + assert not bt.has_been_seen(req, version, sdist_only=True) # sdist only does not affect wheel status - bt._mark_as_seen(req, version, sdist_only=True) - assert bt._has_been_seen(req, version, sdist_only=True) - assert not bt._has_been_seen(req, version, sdist_only=False) + bt.mark_as_seen(req, version, sdist_only=True) + assert bt.has_been_seen(req, version, sdist_only=True) + assert not bt.has_been_seen(req, version, sdist_only=False) - bt._mark_as_seen(req, version, sdist_only=False) - assert bt._has_been_seen(req, version, sdist_only=True) - assert bt._has_been_seen(req, version, sdist_only=False) + bt.mark_as_seen(req, version, sdist_only=False) + assert bt.has_been_seen(req, version, sdist_only=True) + assert bt.has_been_seen(req, version, sdist_only=False) req2 = Requirement("testwheel") - assert not bt._has_been_seen(req2, version, sdist_only=False) - assert not bt._has_been_seen(req2, version, sdist_only=True) + assert not bt.has_been_seen(req2, version, sdist_only=False) + assert not bt.has_been_seen(req2, version, sdist_only=True) # full seen affects both sdist and wheel status - bt._mark_as_seen(req2, version, sdist_only=False) - assert bt._has_been_seen(req2, version, sdist_only=True) - assert bt._has_been_seen(req2, version, sdist_only=False) + bt.mark_as_seen(req2, version, sdist_only=False) + assert bt.has_been_seen(req2, version, sdist_only=True) + assert bt.has_been_seen(req2, version, sdist_only=False) def test_build_order(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) - bt._add_to_build_order( + bt.add_to_build_order( req=Requirement("buildme>1.0"), version=Version("6.0"), source_url="url", source_type=SourceType.SDIST, ) - bt._add_to_build_order( + bt.add_to_build_order( req=Requirement("testdist>1.0"), version=Version("1.2"), source_url="url", @@ -113,19 +156,19 @@ def test_build_order(tmp_context: WorkContext) -> None: def test_build_order_repeats(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) - bt._add_to_build_order( + bt.add_to_build_order( Requirement("buildme>1.0"), Version("6.0"), "url", SourceType.SDIST, ) - bt._add_to_build_order( + bt.add_to_build_order( Requirement("buildme>1.0"), Version("6.0"), "url", SourceType.SDIST, ) - bt._add_to_build_order( + bt.add_to_build_order( Requirement("buildme[extra]>1.0"), Version("6.0"), "url", @@ -149,13 +192,13 @@ def test_build_order_repeats(tmp_context: WorkContext) -> None: def test_build_order_name_canonicalization(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) - bt._add_to_build_order( + bt.add_to_build_order( Requirement("flit-core>1.0"), Version("3.9.0"), "url", SourceType.SDIST, ) - bt._add_to_build_order( + bt.add_to_build_order( Requirement("flit_core>1.0"), Version("3.9.0"), "url", @@ -180,44 +223,72 @@ def test_build_order_name_canonicalization(tmp_context: WorkContext) -> None: def test_explain(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) bt.why = [(RequirementType.TOP_LEVEL, Requirement("foo"), Version("1.0.0"))] - assert bt._explain == f"{RequirementType.TOP_LEVEL} dependency foo (1.0.0)" + assert bt.explain == f"{RequirementType.TOP_LEVEL} dependency foo (1.0.0)" bt.why = [] - assert bt._explain == "" + assert bt.explain == "" bt.why = [ (RequirementType.TOP_LEVEL, Requirement("foo"), Version("1.0.0")), (RequirementType.BUILD_SYSTEM, Requirement("bar==4.0.0"), Version("4.0.0")), ] assert ( - bt._explain + bt.explain == f"{RequirementType.BUILD_SYSTEM} dependency bar==4.0.0 (4.0.0) for {RequirementType.TOP_LEVEL} dependency foo (1.0.0)" ) -def test_is_build_requirement(tmp_context: WorkContext) -> None: - bt = bootstrapper.Bootstrapper(tmp_context) - bt.why = [] - assert not bt._processing_build_requirement(RequirementType.TOP_LEVEL) - assert bt._processing_build_requirement(RequirementType.BUILD_SYSTEM) - assert bt._processing_build_requirement(RequirementType.BUILD_BACKEND) - assert bt._processing_build_requirement(RequirementType.BUILD_SDIST) - assert not bt._processing_build_requirement(RequirementType.INSTALL) +def _make_work_item( + req_type: RequirementType, + why_snapshot: list[tuple[RequirementType, Requirement, Version]] | None = None, +) -> WorkItem: + return WorkItem( + req=Requirement("testpkg"), + req_type=req_type, + why_snapshot=why_snapshot or [], + ) - bt.why = [(RequirementType.TOP_LEVEL, Requirement("foo"), Version("1.0.0"))] - assert not bt._processing_build_requirement(RequirementType.INSTALL) - assert bt._processing_build_requirement(RequirementType.BUILD_SYSTEM) - assert bt._processing_build_requirement(RequirementType.BUILD_BACKEND) - assert bt._processing_build_requirement(RequirementType.BUILD_SDIST) - bt.why = [ +def test_is_build_requirement(tmp_context: WorkContext) -> None: + # No ancestry: req_type alone determines result + assert not _make_work_item(RequirementType.TOP_LEVEL).is_build_requirement_context() + assert _make_work_item(RequirementType.BUILD_SYSTEM).is_build_requirement_context() + assert _make_work_item(RequirementType.BUILD_BACKEND).is_build_requirement_context() + assert _make_work_item(RequirementType.BUILD_SDIST).is_build_requirement_context() + assert not _make_work_item(RequirementType.INSTALL).is_build_requirement_context() + + # TOP_LEVEL ancestry: install is still not a build requirement + top_level_why = [(RequirementType.TOP_LEVEL, Requirement("foo"), Version("1.0.0"))] + assert not _make_work_item( + RequirementType.INSTALL, top_level_why + ).is_build_requirement_context() + assert _make_work_item( + RequirementType.BUILD_SYSTEM, top_level_why + ).is_build_requirement_context() + assert _make_work_item( + RequirementType.BUILD_BACKEND, top_level_why + ).is_build_requirement_context() + assert _make_work_item( + RequirementType.BUILD_SDIST, top_level_why + ).is_build_requirement_context() + + # BUILD_SYSTEM in ancestry: install becomes a build requirement + build_why = [ (RequirementType.TOP_LEVEL, Requirement("foo"), Version("1.0.0")), (RequirementType.BUILD_SYSTEM, Requirement("bar==4.0.0"), Version("4.0.0")), ] - assert bt._processing_build_requirement(RequirementType.INSTALL) - assert bt._processing_build_requirement(RequirementType.BUILD_SYSTEM) - assert bt._processing_build_requirement(RequirementType.BUILD_BACKEND) - assert bt._processing_build_requirement(RequirementType.BUILD_SDIST) + assert _make_work_item( + RequirementType.INSTALL, build_why + ).is_build_requirement_context() + assert _make_work_item( + RequirementType.BUILD_SYSTEM, build_why + ).is_build_requirement_context() + assert _make_work_item( + RequirementType.BUILD_BACKEND, build_why + ).is_build_requirement_context() + assert _make_work_item( + RequirementType.BUILD_SDIST, build_why + ).is_build_requirement_context() def test_find_cached_wheel_returns_tuple(tmp_context: WorkContext) -> None: @@ -225,7 +296,7 @@ def test_find_cached_wheel_returns_tuple(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) # Call method (will return None, None since no wheels exist) - result = bootstrapper._find_cached_wheel( + result = _find_cached_wheel( bt.ctx, bt.cache_wheel_server_url, req=Requirement("test-package"), @@ -242,13 +313,12 @@ def test_get_install_dependencies_returns_list( mock_get_deps: Mock, tmp_context: WorkContext ) -> None: """Verify _get_install_dependencies returns list.""" - bt = bootstrapper.Bootstrapper(tmp_context) - # Create fake wheel file and mock dependencies wheel_file = pathlib.Path("/fake/package-1.0.0-py3-none-any.whl") unpack_dir = tmp_context.work_dir - result = bt._get_install_dependencies( + result = _get_install_dependencies( + ctx=tmp_context, req=Requirement("test-package"), resolved_version=Version("1.0.0"), wheel_filename=wheel_file, @@ -265,19 +335,18 @@ def test_get_install_dependencies_returns_list( def test_phase_build_produces_source_build_result(tmp_context: WorkContext) -> None: - """Verify _phase_build produces a SourceBuildResult with correct values.""" + """Verify Build.run() produces a SourceBuildResult with correct values.""" bt = bootstrapper.Bootstrapper(tmp_context) mock_sdist_root = tmp_context.work_dir / "package-1.0.0" / "package-1.0.0" mock_sdist_root.parent.mkdir(parents=True, exist_ok=True) mock_wheel = tmp_context.work_dir / "package-1.0.0-py3-none-any.whl" - item = bootstrapper.WorkItem( + wi = WorkItem( req=Requirement("test-package"), req_type=RequirementType.TOP_LEVEL, source_url="https://pypi.org/simple/test-package", resolved_version=Version("1.0.0"), - phase=bootstrapper.BootstrapPhase.BUILD, why_snapshot=[], sdist_root_dir=mock_sdist_root, unpack_dir=mock_sdist_root.parent, @@ -286,22 +355,23 @@ def test_phase_build_produces_source_build_result(tmp_context: WorkContext) -> N build_backend_deps=set(), build_sdist_deps=set(), ) + item = Build(wi) # Set up why stack so _track_why works bt.why = [] with ( patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), - patch.object(bt, "_build_wheel", return_value=(mock_wheel, None)), + patch.object(item, "_build_wheel", return_value=(mock_wheel, None)), ): with bt._track_why(item): - result_items = bt._phase_build(item) + result_items = item.run(bt) assert len(result_items) == 1 - assert result_items[0].phase == bootstrapper.BootstrapPhase.PROCESS_INSTALL_DEPS + assert isinstance(result_items[0], ProcessInstallDeps) - result = result_items[0].build_result - assert isinstance(result, bootstrapper.SourceBuildResult) + result = result_items[0].work_item.build_result + assert isinstance(result, SourceBuildResult) assert result.wheel_filename == mock_wheel assert result.sdist_filename is None assert result.unpack_dir == mock_sdist_root.parent @@ -323,26 +393,23 @@ def test_multiple_versions_continues_on_error(tmp_context: WorkContext) -> None: ("https://pypi.org/testpkg-1.0.tar.gz", Version("1.0")), ], ): - # Mock _dispatch_phase to let RESOLVE and START run normally - # but fail for version 1.5 in build phases. - original_dispatch = bt._dispatch_phase + # Let RESOLVE and START run normally, but intercept PrepareSource onwards. + # Fail for version 1.5; count all interceptions. build_phase_count = {"count": 0} - def mock_dispatch(item: bootstrapper.WorkItem) -> list[bootstrapper.WorkItem]: - if item.phase in ( - bootstrapper.BootstrapPhase.RESOLVE, - bootstrapper.BootstrapPhase.START, - ): - return original_dispatch(item) + def prepare_source_run( + self: PrepareSource, + bt_arg: bootstrapper.Bootstrapper, + ) -> list[Phase]: build_phase_count["count"] += 1 - if str(item.resolved_version) == "1.5": + if str(self.work_item.resolved_version) == "1.5": raise ValueError("Simulated failure for version 1.5") return [] req = Requirement("testpkg>=1.0") - with patch.object(bt, "_dispatch_phase", side_effect=mock_dispatch): - with patch.object(bt, "_has_been_seen", return_value=False): + with patch.object(PrepareSource, "run", prepare_source_run): + with patch.object(bt, "has_been_seen", return_value=False): bt._bootstrap_one( req=req, req_type=RequirementType.INSTALL, @@ -389,7 +456,7 @@ def test_download_wheel_from_cache_bypasses_hooks( mock_find_all.side_effect = RuntimeError("no match") with patch("fromager.overrides.find_and_invoke") as mock_override: - result = bootstrapper._download_wheel_from_cache( + result = _download_wheel_from_cache( bt.ctx, bt.cache_wheel_server_url, req=Requirement("test-pkg"), @@ -431,7 +498,7 @@ def test_cache_lookup_resolver_exception_logs_info( side_effect=ResolverException("no matching version"), ), ): - result = bootstrapper._download_wheel_from_cache( + result = _download_wheel_from_cache( bt.ctx, bt.cache_wheel_server_url, req=Requirement("test-package"), @@ -463,7 +530,7 @@ def test_cache_lookup_request_exception_logs_warning( "fromager.resolver.find_all_matching_from_provider", side_effect=exc_class(exc_msg), ): - result = bootstrapper._download_wheel_from_cache( + result = _download_wheel_from_cache( bt.ctx, bt.cache_wheel_server_url, req=Requirement("test-package"), @@ -485,7 +552,7 @@ def test_cache_lookup_unexpected_exception_logs_warning( "fromager.resolver.find_all_matching_from_provider", side_effect=ValueError("unexpected parsing error"), ): - result = bootstrapper._download_wheel_from_cache( + result = _download_wheel_from_cache( bt.ctx, bt.cache_wheel_server_url, req=Requirement("test-package"), @@ -528,15 +595,15 @@ def test_cache_lookup_download_wheel_error_logs_warning( ], ), patch( - "fromager.bootstrapper.wheels.extract_info_from_wheel_file", + "fromager.wheels.extract_info_from_wheel_file", return_value=("test_package", "1.0.0", None, None), ), patch( - "fromager.bootstrapper.wheels.download_wheel", + "fromager.wheels.download_wheel", side_effect=exc_class(exc_msg), ), ): - result = bootstrapper._download_wheel_from_cache( + result = _download_wheel_from_cache( bt.ctx, bt.cache_wheel_server_url, req=Requirement("test-package"), @@ -552,7 +619,7 @@ def test_cache_lookup_no_cache_url_returns_none(tmp_context: WorkContext) -> Non bt = bootstrapper.Bootstrapper(tmp_context) bt.cache_wheel_server_url = "" - result = bootstrapper._download_wheel_from_cache( + result = _download_wheel_from_cache( bt.ctx, bt.cache_wheel_server_url, req=Requirement("test-package"), @@ -567,18 +634,19 @@ def _make_resolve_item( req_type: RequirementType = RequirementType.TOP_LEVEL, why_snapshot: list[tuple[RequirementType, Requirement, Version]] | None = None, parent: tuple[Requirement, Version] | None = None, -) -> bootstrapper.WorkItem: - return bootstrapper.WorkItem( - req=Requirement(req), - req_type=req_type, - phase=bootstrapper.BootstrapPhase.RESOLVE, - why_snapshot=why_snapshot or [], - parent=parent, +) -> Resolve: + return Resolve( + WorkItem( + req=Requirement(req), + req_type=req_type, + why_snapshot=why_snapshot or [], + parent=parent, + ) ) def _record_and_load( - bt: bootstrapper.Bootstrapper, stack: list[bootstrapper.WorkItem] + bt: bootstrapper.Bootstrapper, stack: list[Phase] ) -> list[typing.Any]: bt._record_stack_state(stack) return typing.cast(list[typing.Any], json.loads(bt._stack_filename.read_text())) @@ -592,7 +660,7 @@ def test_record_stack_state_minimal_item(tmp_context: WorkContext) -> None: result = contents[0] assert result["req"] == "testpkg" assert result["req_type"] == str(RequirementType.TOP_LEVEL) - assert result["phase"] == str(bootstrapper.BootstrapPhase.RESOLVE) + assert result["phase"] == str(BootstrapPhase.RESOLVE) assert result["resolved_version"] is None assert result["source_url"] is None assert result["build_sdist_only"] is False @@ -610,18 +678,19 @@ def test_record_stack_state_full_item(tmp_context: WorkContext) -> None: parent_version = Version("2.0") why_snapshot = [(RequirementType.INSTALL, parent_req, parent_version)] - item = bootstrapper.WorkItem( - req=Requirement("child-pkg>=1.0"), - req_type=RequirementType.INSTALL, - phase=bootstrapper.BootstrapPhase.BUILD, - why_snapshot=why_snapshot, - parent=(parent_req, parent_version), - resolved_version=Version("1.5"), - source_url="https://pypi.test/child-pkg-1.5.tar.gz", - build_sdist_only=True, - build_system_deps={Requirement("setuptools")}, - build_backend_deps={Requirement("wheel")}, - build_sdist_deps={Requirement("flit-core")}, + item = Build( + WorkItem( + req=Requirement("child-pkg>=1.0"), + req_type=RequirementType.INSTALL, + why_snapshot=why_snapshot, + parent=(parent_req, parent_version), + resolved_version=Version("1.5"), + source_url="https://pypi.test/child-pkg-1.5.tar.gz", + build_sdist_only=True, + build_system_deps={Requirement("setuptools")}, + build_backend_deps={Requirement("wheel")}, + build_sdist_deps={Requirement("flit-core")}, + ) ) contents = _record_and_load(bt, [item]) @@ -646,12 +715,17 @@ def test_record_stack_state_full_item(tmp_context: WorkContext) -> None: def test_record_stack_state_dep_sets_are_sorted(tmp_context: WorkContext) -> None: """Mixed-order dep sets come out alphabetically sorted.""" bt = bootstrapper.Bootstrapper(tmp_context) - item = bootstrapper.WorkItem( - req=Requirement("mypkg"), - req_type=RequirementType.TOP_LEVEL, - phase=bootstrapper.BootstrapPhase.BUILD, - why_snapshot=[], - build_system_deps={Requirement("zzz"), Requirement("aaa"), Requirement("mmm")}, + item = Build( + WorkItem( + req=Requirement("mypkg"), + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + build_system_deps={ + Requirement("zzz"), + Requirement("aaa"), + Requirement("mmm"), + }, + ) ) contents = _record_and_load(bt, [item]) @@ -661,7 +735,10 @@ def test_record_stack_state_dep_sets_are_sorted(tmp_context: WorkContext) -> Non def test_record_stack_state_writes_file(tmp_context: WorkContext) -> None: """File is created; list length matches stack size.""" bt = bootstrapper.Bootstrapper(tmp_context) - stack = [_make_resolve_item("pkga"), _make_resolve_item("pkgb")] + stack: list[Phase] = [ + _make_resolve_item("pkga"), + _make_resolve_item("pkgb"), + ] bt._record_stack_state(stack) @@ -674,7 +751,7 @@ def test_record_stack_state_writes_file(tmp_context: WorkContext) -> None: def test_record_stack_state_ordering(tmp_context: WorkContext) -> None: """Index 0 = stack[-1] (next to pop); last index = stack[0].""" bt = bootstrapper.Bootstrapper(tmp_context) - stack = [ + stack: list[Phase] = [ _make_resolve_item("pkga"), _make_resolve_item("pkgb"), _make_resolve_item("pkgc"), @@ -709,7 +786,7 @@ def test_bootstrap_calls_record_stack_state(tmp_context: WorkContext) -> None: original = bt._record_stack_state - def counting_record(stack: list[bootstrapper.WorkItem]) -> None: + def counting_record(stack: list[Phase]) -> None: call_count["n"] += 1 original(stack) @@ -722,7 +799,7 @@ def counting_record(stack: list[bootstrapper.WorkItem]) -> None: "resolve", return_value=[("https://pypi.test/testpkg-1.0.tar.gz", Version("1.0"))], ), - patch.object(bt, "_phase_start", return_value=[]), + patch.object(Start, "run", return_value=[]), ): bt._bootstrap_one(req=req, req_type=RequirementType.TOP_LEVEL) @@ -732,15 +809,20 @@ def counting_record(stack: list[bootstrapper.WorkItem]) -> None: def test_bootstrap_with_empty_list(tmp_context: WorkContext) -> None: """bootstrap([]) completes without error and runs no phases.""" bt = bootstrapper.Bootstrapper(tmp_context) - with patch.object(bt, "_dispatch_phase") as mock_dispatch: + with patch.object(Resolve, "run") as mock_run: bt.bootstrap([]) - mock_dispatch.assert_not_called() + mock_run.assert_not_called() def test_bootstrap_with_single_requirement(tmp_context: WorkContext) -> None: """bootstrap([req]) resolves and processes the requirement.""" bt = bootstrapper.Bootstrapper(tmp_context) req = Requirement("testpkg==1.0") + captured: list[Resolve] = [] + + def capture_run(self: Resolve, bt_arg: bootstrapper.Bootstrapper) -> list[Phase]: + captured.append(self) + return [] with ( patch.object( @@ -748,16 +830,15 @@ def test_bootstrap_with_single_requirement(tmp_context: WorkContext) -> None: "_resolve_and_add_top_level", return_value=("http://example.test/testpkg-1.0.tar.gz", Version("1.0")), ), - patch.object(bt, "_dispatch_phase", return_value=[]) as mock_dispatch, + patch.object(Resolve, "run", capture_run), patch.object(bt, "_record_stack_state"), ): bt.bootstrap([req]) - mock_dispatch.assert_called_once() - item = mock_dispatch.call_args[0][0] - assert item.req == req - assert item.req_type == RequirementType.TOP_LEVEL - assert item.phase == bootstrapper.BootstrapPhase.RESOLVE + assert len(captured) == 1 + assert isinstance(captured[0], Resolve) + assert captured[0].work_item.req == req + assert captured[0].work_item.req_type == RequirementType.TOP_LEVEL def test_bootstrap_skips_failed_resolution(tmp_context: WorkContext) -> None: @@ -767,12 +848,12 @@ def test_bootstrap_skips_failed_resolution(tmp_context: WorkContext) -> None: with ( patch.object(bt, "_resolve_and_add_top_level", return_value=None), - patch.object(bt, "_dispatch_phase") as mock_dispatch, + patch.object(Resolve, "run") as mock_run, patch.object(bt, "_record_stack_state"), ): bt.bootstrap([req]) - mock_dispatch.assert_not_called() + mock_run.assert_not_called() def test_bootstrap_two_requirements_both_processed(tmp_context: WorkContext) -> None: @@ -783,8 +864,8 @@ def test_bootstrap_two_requirements_both_processed(tmp_context: WorkContext) -> dispatch_calls: list = [] - def fake_dispatch(item: bootstrapper.WorkItem) -> list: - dispatch_calls.append(item.req.name) + def capture_run(self: Resolve, bt_arg: bootstrapper.Bootstrapper) -> list[Phase]: + dispatch_calls.append(self.work_item.req.name) return [] with ( @@ -793,7 +874,7 @@ def fake_dispatch(item: bootstrapper.WorkItem) -> list: "_resolve_and_add_top_level", return_value=("http://example.test/pkg-1.0.tar.gz", Version("1.0")), ), - patch.object(bt, "_dispatch_phase", side_effect=fake_dispatch), + patch.object(Resolve, "run", capture_run), patch.object(bt, "_record_stack_state"), ): bt.bootstrap([req1, req2]) @@ -816,7 +897,7 @@ def test_bg_prepare_source_log_prefix_includes_version( with ( caplog.at_level(logging.INFO, logger="fromager.bootstrapper"), patch( - "fromager.bootstrapper._find_cached_wheel", + "fromager.bootstrapper._cache._find_cached_wheel", return_value=(None, None), ), patch( @@ -829,7 +910,7 @@ def test_bg_prepare_source_log_prefix_includes_version( ), log.req_ctxvar_context(req, version), ): - bootstrapper._bg_prepare_source( + _bg_prepare_source( ctx=tmp_context, cache_wheel_server_url=None, req=req, @@ -841,7 +922,7 @@ def test_bg_prepare_source_log_prefix_includes_version( messages = [ r.getMessage() for r in caplog.records - if r.name == "fromager.bootstrapper" + if r.name.startswith("fromager.bootstrapper") ] finally: logging.setLogRecordFactory(old_factory) @@ -873,7 +954,7 @@ def test_bg_prepare_prebuilt_log_prefix_includes_version( patch("fromager.server.update_wheel_mirror"), log.req_ctxvar_context(req, version), ): - bootstrapper._bg_prepare_prebuilt( + _bg_prepare_prebuilt( ctx=tmp_context, req=req, req_type=RequirementType.INSTALL, @@ -885,7 +966,7 @@ def test_bg_prepare_prebuilt_log_prefix_includes_version( messages = [ r.getMessage() for r in caplog.records - if r.name == "fromager.bootstrapper" + if r.name.startswith("fromager.bootstrapper") ] finally: logging.setLogRecordFactory(old_factory) @@ -895,3 +976,176 @@ def test_bg_prepare_prebuilt_log_prefix_includes_version( assert msg.startswith("mypkg-1.2.3: "), ( f"Expected 'mypkg-1.2.3: ' prefix, got: {msg!r}" ) + + +def test_build_item_build_sdist_finds_existing(tmp_context: WorkContext) -> None: + """Build._build_sdist returns cached sdist when finders.find_sdist hits.""" + sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" + sdist_root.mkdir(parents=True, exist_ok=True) + wi = WorkItem( + req=Requirement("testpkg"), + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + resolved_version=Version("1.0"), + sdist_root_dir=sdist_root, + build_env=Mock(), + ) + item = Build(wi) + cached = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" + cached.touch() + + with patch("fromager.finders.find_sdist", return_value=cached): + result = item._build_sdist(tmp_context) + + assert result == cached + + +def test_build_item_build_sdist_calls_build_when_not_cached( + tmp_context: WorkContext, +) -> None: + """Build._build_sdist calls sources.build_sdist when no cached sdist found.""" + sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" + sdist_root.mkdir(parents=True, exist_ok=True) + wi = WorkItem( + req=Requirement("testpkg"), + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + resolved_version=Version("1.0"), + sdist_root_dir=sdist_root, + build_env=Mock(), + ) + item = Build(wi) + built = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" + + with ( + patch("fromager.finders.find_sdist", return_value=None), + patch("fromager.sources.build_sdist", return_value=built) as mock_build, + ): + result = item._build_sdist(tmp_context) + + mock_build.assert_called_once_with( + ctx=tmp_context, + req=wi.req, + version=wi.resolved_version, + sdist_root_dir=sdist_root, + build_env=wi.build_env, + ) + assert result == built + + +def test_build_item_build_wheel(tmp_context: WorkContext) -> None: + """Build._build_wheel builds sdist then wheel and updates mirror.""" + sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" + sdist_root.mkdir(parents=True, exist_ok=True) + wi = WorkItem( + req=Requirement("testpkg"), + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + resolved_version=Version("1.0"), + sdist_root_dir=sdist_root, + build_env=Mock(), + ) + item = Build(wi) + built_wheel = tmp_context.wheels_build / "testpkg-1.0-py3-none-any.whl" + built_sdist = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" + + with ( + patch.object( + item, "_build_sdist", return_value=built_sdist + ) as mock_build_sdist, + patch("fromager.wheels.build_wheel", return_value=built_wheel), + patch("fromager.server.update_wheel_mirror") as mock_update_mirror, + ): + wheel_filename, sdist_filename = item._build_wheel(tmp_context) + + mock_build_sdist.assert_called_once_with(tmp_context) + mock_update_mirror.assert_called_once_with(tmp_context) + assert wheel_filename == tmp_context.wheels_downloads / built_wheel.name + assert sdist_filename == built_sdist + + +def test_build_item_do_build_returns_cached_wheel( + tmp_context: WorkContext, +) -> None: + """Build.do_build returns cached wheel immediately without building.""" + cached = tmp_context.wheels_downloads / "testpkg-1.0-py3-none-any.whl" + cached.touch() + wi = WorkItem( + req=Requirement("testpkg"), + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + resolved_version=Version("1.0"), + build_sdist_only=False, + cached_wheel_filename=cached, + sdist_root_dir=tmp_context.work_dir, + build_env=Mock(), + ) + item = Build(wi) + + with patch.object(item, "_build_wheel") as mock_wheel: + wheel, sdist = item.do_build(tmp_context) + + mock_wheel.assert_not_called() + assert wheel == cached + assert sdist is None + + +def test_build_item_do_build_sdist_only(tmp_context: WorkContext) -> None: + """Build.do_build calls _build_sdist and returns (None, sdist) when build_sdist_only=True.""" + built_sdist = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" + sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" + sdist_root.mkdir(parents=True, exist_ok=True) + wi = WorkItem( + req=Requirement("testpkg"), + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + resolved_version=Version("1.0"), + build_sdist_only=True, + cached_wheel_filename=None, + sdist_root_dir=sdist_root, + build_env=Mock(), + ) + item = Build(wi) + + with ( + patch.object(item, "_build_sdist", return_value=built_sdist) as mock_sdist, + patch.object(item, "_build_wheel") as mock_wheel, + ): + wheel, sdist = item.do_build(tmp_context) + + mock_wheel.assert_not_called() + mock_sdist.assert_called_once_with(tmp_context) + assert wheel is None + assert sdist == built_sdist + + +def test_build_item_do_build_builds_wheel(tmp_context: WorkContext) -> None: + """Build.do_build calls _build_wheel when no cache and not sdist_only.""" + built_wheel = tmp_context.wheels_downloads / "testpkg-1.0-py3-none-any.whl" + built_sdist = tmp_context.sdists_builds / "testpkg-1.0.tar.gz" + sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" + sdist_root.mkdir(parents=True, exist_ok=True) + wi = WorkItem( + req=Requirement("testpkg"), + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + resolved_version=Version("1.0"), + build_sdist_only=False, + cached_wheel_filename=None, + sdist_root_dir=sdist_root, + build_env=Mock(), + ) + item = Build(wi) + + with patch.object( + item, "_build_wheel", return_value=(built_wheel, built_sdist) + ) as mock_wheel: + wheel, sdist = item.do_build( + tmp_context, explain="top_level dependency testpkg (1.0)" + ) + + mock_wheel.assert_called_once_with( + tmp_context, "top_level dependency testpkg (1.0)" + ) + assert wheel == built_wheel + assert sdist == built_sdist diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index d4e75c52..6761bcfa 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -1,18 +1,17 @@ """Tests for the iterative bootstrap implementation. Tests cover: -- BootstrapPhase enum and tracks_why property +- Phase class hierarchy: class variables and base class behavior - WorkItem dataclass defaults and state accumulation - _track_why context manager behavior -- _create_unresolved_work_items helper -- _phase_resolve version expansion -- _phase_start graph addition and seen-check -- _phase_prepare_source all branches (prebuilt, source, cached, bad path) -- _phase_prepare_build dep installation and extraction -- _phase_build conditional install and result construction -- _phase_process_install_deps hooks, dep extraction, error modes -- _phase_complete cleanup -- _dispatch_phase routing +- create_unresolved_work_items helper +- Resolve.run() version expansion +- Start.run() graph addition and seen-check +- PrepareSource.run() all branches (prebuilt, source, cached, bad path) +- PrepareBuild.run() dep installation and extraction +- Build.run() conditional install and result construction +- ProcessInstallDeps.run() hooks, dep extraction, error modes +- Complete.run() cleanup - _handle_phase_error for all three error modes - End-to-end iterative loop with LIFO ordering """ @@ -30,7 +29,20 @@ from packaging.version import Version from fromager import bootstrapper, build_environment -from fromager.bootstrapper import BootstrapPhase, SourceBuildResult, WorkItem +from fromager.bootstrapper._build import Build +from fromager.bootstrapper._complete import Complete +from fromager.bootstrapper._phase import Phase +from fromager.bootstrapper._prepare_build import PrepareBuild +from fromager.bootstrapper._prepare_source import PrepareSource +from fromager.bootstrapper._process_install_deps import ProcessInstallDeps +from fromager.bootstrapper._resolve import Resolve +from fromager.bootstrapper._start import Start +from fromager.bootstrapper._types import ( + BootstrapPhase, + PreparedSourceData, + SourceBuildResult, +) +from fromager.bootstrapper._work_item import WorkItem from fromager.context import WorkContext from fromager.requirements_file import RequirementType, SourceType @@ -44,18 +56,59 @@ def _make_resolved_future( return future -def _make_resolve_item( +def _make_work_item( req: str = "testpkg", req_type: RequirementType = RequirementType.INSTALL, why_snapshot: list | None = None, parent: tuple | None = None, + source_url: str | None = None, + version: str | None = None, + build_env: build_environment.BuildEnvironment | None = None, + sdist_root_dir: pathlib.Path | None = None, + unpack_dir: pathlib.Path | None = None, + build_result: SourceBuildResult | None = None, + build_system_deps: set[Requirement] | None = None, + build_backend_deps: set[Requirement] | None = None, + build_sdist_deps: set[Requirement] | None = None, + pbi_pre_built: bool = False, + cached_wheel_filename: pathlib.Path | None = None, + build_sdist_only: bool = False, ) -> WorkItem: return WorkItem( req=Requirement(req), req_type=req_type, - phase=BootstrapPhase.RESOLVE, why_snapshot=why_snapshot or [], parent=parent, + source_url=source_url, + resolved_version=Version(version) if version else None, + build_env=build_env, + sdist_root_dir=sdist_root_dir, + unpack_dir=unpack_dir, + build_result=build_result, + build_system_deps=build_system_deps if build_system_deps is not None else set(), + build_backend_deps=build_backend_deps + if build_backend_deps is not None + else set(), + build_sdist_deps=build_sdist_deps if build_sdist_deps is not None else set(), + pbi_pre_built=pbi_pre_built, + cached_wheel_filename=cached_wheel_filename, + build_sdist_only=build_sdist_only, + ) + + +def _make_resolve_item( + req: str = "testpkg", + req_type: RequirementType = RequirementType.INSTALL, + why_snapshot: list | None = None, + parent: tuple | None = None, +) -> Resolve: + return Resolve( + WorkItem( + req=Requirement(req), + req_type=req_type, + why_snapshot=why_snapshot or [], + parent=parent, + ) ) @@ -66,18 +119,30 @@ def _make_start_item( version: str = "1.0", why_snapshot: list | None = None, parent: tuple | None = None, -) -> WorkItem: - return WorkItem( - req=Requirement(req), - req_type=req_type, - phase=BootstrapPhase.START, - why_snapshot=why_snapshot or [], - parent=parent, - source_url=source_url, - resolved_version=Version(version), +) -> Start: + return Start( + WorkItem( + req=Requirement(req), + req_type=req_type, + why_snapshot=why_snapshot or [], + parent=parent, + source_url=source_url, + resolved_version=Version(version), + ) ) +_PHASE_TO_CLASS: dict[BootstrapPhase, type[Phase]] = { + BootstrapPhase.RESOLVE: Resolve, + BootstrapPhase.START: Start, + BootstrapPhase.PREPARE_SOURCE: PrepareSource, + BootstrapPhase.PREPARE_BUILD: PrepareBuild, + BootstrapPhase.BUILD: Build, + BootstrapPhase.PROCESS_INSTALL_DEPS: ProcessInstallDeps, + BootstrapPhase.COMPLETE: Complete, +} + + def _make_build_item( req: str = "testpkg", version: str = "1.0", @@ -93,11 +158,10 @@ def _make_build_item( pbi_pre_built: bool = False, cached_wheel_filename: pathlib.Path | None = None, build_sdist_only: bool = False, -) -> WorkItem: - return WorkItem( +) -> Phase: + wi = WorkItem( req=Requirement(req), req_type=RequirementType.INSTALL, - phase=phase, why_snapshot=[], source_url=source_url, resolved_version=Version(version), @@ -114,48 +178,75 @@ def _make_build_item( cached_wheel_filename=cached_wheel_filename, build_sdist_only=build_sdist_only, ) + return _PHASE_TO_CLASS[phase](wi) -class TestBootstrapPhase: - def test_tracks_why_false_for_resolve(self) -> None: - assert BootstrapPhase.RESOLVE.tracks_why is False +class TestPhaseClassVariables: + """Verify class-variable declarations on each Phase subclass.""" - def test_tracks_why_false_for_start(self) -> None: - assert BootstrapPhase.START.tracks_why is False + @pytest.mark.parametrize( + "cls, expected_phase, expected_tracks_why", + [ + (Resolve, BootstrapPhase.RESOLVE, False), + (Start, BootstrapPhase.START, False), + (PrepareSource, BootstrapPhase.PREPARE_SOURCE, True), + (PrepareBuild, BootstrapPhase.PREPARE_BUILD, True), + (Build, BootstrapPhase.BUILD, True), + (ProcessInstallDeps, BootstrapPhase.PROCESS_INSTALL_DEPS, True), + (Complete, BootstrapPhase.COMPLETE, True), + ], + ) + def test_class_variables( + self, + tmp_context: WorkContext, + cls: type[Phase], + expected_phase: BootstrapPhase, + expected_tracks_why: bool, + ) -> None: + assert cls.phase == expected_phase + assert cls.tracks_why == expected_tracks_why - def test_tracks_why_true_for_build_phases(self) -> None: - for phase in ( - BootstrapPhase.PREPARE_SOURCE, - BootstrapPhase.PREPARE_BUILD, - BootstrapPhase.BUILD, - BootstrapPhase.PROCESS_INSTALL_DEPS, - BootstrapPhase.COMPLETE, - ): - assert phase.tracks_why is True, f"{phase} should track why" + def test_str_default(self) -> None: + wi = _make_work_item(req="mypkg") + item = Build(wi) + assert str(item) == "Build(mypkg)" + + def test_background_work_returns_none_by_default( + self, tmp_context: WorkContext + ) -> None: + bt = bootstrapper.Bootstrapper(tmp_context) + for cls in (PrepareBuild, Build, ProcessInstallDeps, Complete): + wi = _make_work_item(req="testpkg", version="1.0") + item = cls(wi) + assert item.background_work(bt) is None, ( + f"{cls.__name__}.background_work() should return None" + ) class TestWorkItem: def test_defaults_for_resolve_item(self) -> None: item = _make_resolve_item() - assert item.source_url is None - assert item.resolved_version is None - assert item.build_sdist_only is False - assert item.build_env is None - assert item.build_result is None - assert item.pbi_pre_built is False - assert item.build_system_deps == set() - assert item.build_backend_deps == set() - assert item.build_sdist_deps == set() + wi = item.work_item + assert wi.source_url is None + assert wi.resolved_version is None + assert wi.build_sdist_only is False + assert wi.build_env is None + assert wi.build_result is None + assert wi.pbi_pre_built is False + assert wi.build_system_deps == set() + assert wi.build_backend_deps == set() + assert wi.build_sdist_deps == set() def test_state_accumulation(self) -> None: item = _make_start_item() - item.pbi_pre_built = True - item.build_sdist_only = True + wi = item.work_item + wi.pbi_pre_built = True + wi.build_sdist_only = True mock_env = Mock() - item.build_env = mock_env - assert item.pbi_pre_built is True - assert item.build_sdist_only is True - assert item.build_env is mock_env + wi.build_env = mock_env + assert wi.pbi_pre_built is True + assert wi.build_sdist_only is True + assert wi.build_env is mock_env class TestTrackWhy: @@ -183,11 +274,12 @@ def test_pushes_and_pops_for_build_phase(self, tmp_context: WorkContext) -> None bt = bootstrapper.Bootstrapper(tmp_context) bt.why = [] item = _make_build_item(phase=BootstrapPhase.PREPARE_SOURCE) + wi = item.work_item with bt._track_why(item): assert len(bt.why) == 1 - assert bt.why[0][1] == item.req - assert bt.why[0][2] == item.resolved_version + assert bt.why[0][1] == wi.req + assert bt.why[0][2] == wi.resolved_version assert len(bt.why) == 0 @@ -209,23 +301,24 @@ def test_creates_resolve_phase_items(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) deps = [Requirement("dep-a"), Requirement("dep-b")] - items = bt._create_unresolved_work_items( + items = bt.create_unresolved_work_items( deps, RequirementType.BUILD_SYSTEM, Requirement("parent"), Version("1.0") ) assert len(items) == 2 for item in items: + assert isinstance(item, Resolve) assert item.phase == BootstrapPhase.RESOLVE - assert item.req_type == RequirementType.BUILD_SYSTEM - assert item.parent == (Requirement("parent"), Version("1.0")) - assert item.source_url is None - assert item.resolved_version is None + assert item.work_item.req_type == RequirementType.BUILD_SYSTEM + assert item.work_item.parent == (Requirement("parent"), Version("1.0")) + assert item.work_item.source_url is None + assert item.work_item.resolved_version is None def test_captures_why_snapshot(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) bt.why = [(RequirementType.TOP_LEVEL, Requirement("root"), Version("2.0"))] - items = bt._create_unresolved_work_items( + items = bt.create_unresolved_work_items( [Requirement("dep")], RequirementType.INSTALL, Requirement("parent"), @@ -233,25 +326,25 @@ def test_captures_why_snapshot(self, tmp_context: WorkContext) -> None: ) assert len(items) == 1 - assert items[0].why_snapshot == bt.why + assert items[0].work_item.why_snapshot == bt.why # Verify it's a copy, not a reference bt.why.append((RequirementType.INSTALL, Requirement("other"), Version("3.0"))) - assert len(items[0].why_snapshot) == 1 + assert len(items[0].work_item.why_snapshot) == 1 def test_sorts_by_name(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) deps = [Requirement("zebra"), Requirement("alpha"), Requirement("middle")] - items = bt._create_unresolved_work_items( + items = bt.create_unresolved_work_items( deps, RequirementType.INSTALL, Requirement("p"), Version("1.0") ) - names = [str(item.req.name) for item in items] + names = [str(item.work_item.req.name) for item in items] assert names == ["alpha", "middle", "zebra"] def test_empty_deps(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) - items = bt._create_unresolved_work_items( + items = bt.create_unresolved_work_items( [], RequirementType.INSTALL, Requirement("p"), Version("1.0") ) assert items == [] @@ -261,19 +354,19 @@ class TestPhaseResolve: def test_single_version(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) item = _make_resolve_item() - parent = (Requirement("parent"), Version("2.0")) - item.parent = parent + wi = item.work_item + wi.parent = (Requirement("parent"), Version("2.0")) item.bg_future = _make_resolved_future( [("https://pypi.org/testpkg-1.0.tar.gz", Version("1.0"))] ) - result = bt._phase_resolve(item) + result = item.run(bt) assert len(result) == 1 - assert result[0].phase == BootstrapPhase.START - assert result[0].source_url == "https://pypi.org/testpkg-1.0.tar.gz" - assert result[0].resolved_version == Version("1.0") - assert result[0].parent == parent + assert isinstance(result[0], Start) + assert result[0].work_item.source_url == "https://pypi.org/testpkg-1.0.tar.gz" + assert result[0].work_item.resolved_version == Version("1.0") + assert result[0].work_item.parent == wi.parent def test_multiple_versions(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) @@ -285,12 +378,12 @@ def test_multiple_versions(self, tmp_context: WorkContext) -> None: ] ) - result = bt._phase_resolve(item) + result = item.run(bt) assert len(result) == 2 # Reversed so highest version ends up on top of stack (last element) - assert result[0].resolved_version == Version("1.0") - assert result[1].resolved_version == Version("2.0") + assert result[0].work_item.resolved_version == Version("1.0") + assert result[1].work_item.resolved_version == Version("2.0") def test_empty_resolution_raises(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) @@ -298,7 +391,7 @@ def test_empty_resolution_raises(self, tmp_context: WorkContext) -> None: item.bg_future = _make_resolved_future([]) with pytest.raises(RuntimeError, match="Could not resolve"): - bt._phase_resolve(item) + item.run(bt) def test_preserves_why_snapshot(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) @@ -306,14 +399,14 @@ def test_preserves_why_snapshot(self, tmp_context: WorkContext) -> None: item = _make_resolve_item(why_snapshot=list(snapshot)) item.bg_future = _make_resolved_future([("url", Version("1.0"))]) - result = bt._phase_resolve(item) + result = item.run(bt) - assert result[0].why_snapshot == snapshot + assert result[0].work_item.why_snapshot == snapshot def test_filters_cached_versions_in_multiple_versions_mode( self, tmp_context: WorkContext ) -> None: - """Cached versions are filtered out before creating START items.""" + """Cached versions are filtered out before creating Starts.""" bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) item = _make_resolve_item() item.bg_future = _make_resolved_future( @@ -331,11 +424,13 @@ def mock_cache( return (tmp_context.work_dir / "pkg-2.0-py3-none-any.whl", None) return (None, None) - with patch("fromager.bootstrapper._find_cached_wheel", side_effect=mock_cache): - result = bt._phase_resolve(item) + with patch( + "fromager.bootstrapper._cache._find_cached_wheel", side_effect=mock_cache + ): + result = item.run(bt) assert len(result) == 2 - versions = {str(it.resolved_version) for it in result} + versions = {str(it.work_item.resolved_version) for it in result} assert versions == {"1.0", "3.0"} def test_all_cached_keeps_highest_version(self, tmp_context: WorkContext) -> None: @@ -351,13 +446,13 @@ def test_all_cached_keeps_highest_version(self, tmp_context: WorkContext) -> Non ) with patch( - "fromager.bootstrapper._find_cached_wheel", + "fromager.bootstrapper._cache._find_cached_wheel", return_value=(tmp_context.work_dir / "cached.whl", None), ): - result = bt._phase_resolve(item) + result = item.run(bt) assert len(result) == 1 - assert result[0].resolved_version == Version("3.0") + assert result[0].work_item.resolved_version == Version("3.0") def test_no_filtering_in_single_version_mode( self, tmp_context: WorkContext @@ -367,8 +462,8 @@ def test_no_filtering_in_single_version_mode( item = _make_resolve_item() item.bg_future = _make_resolved_future([("url-1.0", Version("1.0"))]) - with patch("fromager.bootstrapper._find_cached_wheel") as mock_cache: - result = bt._phase_resolve(item) + with patch("fromager.bootstrapper._cache._find_cached_wheel") as mock_cache: + result = item.run(bt) assert len(result) == 1 mock_cache.assert_not_called() @@ -383,12 +478,12 @@ def test_empty_resolution_raises_runtime_error( item.bg_future = _make_resolved_future([]) with pytest.raises(RuntimeError, match="Could not resolve"): - bt._phase_resolve(item) + item.run(bt) def test_filters_failed_versions_in_multiple_versions_mode( self, tmp_context: WorkContext ) -> None: - """Previously failed versions are excluded before creating START items.""" + """Previously failed versions are excluded before creating Starts.""" bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) item = _make_resolve_item() @@ -404,11 +499,11 @@ def test_filters_failed_versions_in_multiple_versions_mode( ) with patch( - "fromager.bootstrapper._find_cached_wheel", return_value=(None, None) + "fromager.bootstrapper._cache._find_cached_wheel", return_value=(None, None) ): - result = bt._phase_resolve(item) + result = item.run(bt) - versions = {str(it.resolved_version) for it in result} + versions = {str(it.work_item.resolved_version) for it in result} assert versions == {"1.0", "3.0"} def test_failed_version_filter_does_not_apply_in_single_version_mode( @@ -423,10 +518,10 @@ def test_failed_version_filter_does_not_apply_in_single_version_mode( ) item.bg_future = _make_resolved_future([("url-1.0", Version("1.0"))]) - result = bt._phase_resolve(item) + result = item.run(bt) assert len(result) == 1 - assert result[0].resolved_version == Version("1.0") + assert result[0].work_item.resolved_version == Version("1.0") def test_all_versions_failed_raises_runtime_error( self, tmp_context: WorkContext @@ -441,10 +536,10 @@ def test_all_versions_failed_raises_runtime_error( item.bg_future = _make_resolved_future([("url-1.0", Version("1.0"))]) with pytest.raises(RuntimeError, match="failed previously"): - bt._phase_resolve(item) + item.run(bt) def test_bg_future_exception_propagates(self, tmp_context: WorkContext) -> None: - """Exceptions from the background resolver thread are surfaced by _phase_resolve.""" + """Exceptions from the background resolver thread are surfaced by run().""" bt = bootstrapper.Bootstrapper(tmp_context) item = _make_resolve_item() future: concurrent.futures.Future[list[tuple[str, Version]]] = ( @@ -454,7 +549,7 @@ def test_bg_future_exception_propagates(self, tmp_context: WorkContext) -> None: item.bg_future = future with pytest.raises(ValueError, match="resolver exploded"): - bt._phase_resolve(item) + item.run(bt) class TestPhaseStart: @@ -465,22 +560,23 @@ def test_new_item_advances_to_prepare_source( bt.why = [] item = _make_start_item() - result = bt._phase_start(item) + result = item.run(bt) assert len(result) == 1 - assert result[0].phase == BootstrapPhase.PREPARE_SOURCE - assert result[0] is item + assert isinstance(result[0], PrepareSource) + assert result[0].work_item is item.work_item def test_already_seen_returns_empty(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) bt.why = [] item = _make_start_item() + wi = item.work_item # Mark as seen first - assert item.resolved_version is not None - bt._mark_as_seen(item.req, item.resolved_version) + assert wi.resolved_version is not None + bt.mark_as_seen(wi.req, wi.resolved_version) - result = bt._phase_start(item) + result = item.run(bt) assert result == [] @@ -489,7 +585,7 @@ def test_adds_to_graph_for_non_toplevel(self, tmp_context: WorkContext) -> None: bt.why = [] item = _make_start_item(req_type=RequirementType.INSTALL) - bt._phase_start(item) + item.run(bt) key = f"{canonicalize_name('testpkg')}==1.0" assert key in tmp_context.dependency_graph.nodes @@ -499,7 +595,7 @@ def test_skips_graph_for_toplevel(self, tmp_context: WorkContext) -> None: bt.why = [] item = _make_start_item(req_type=RequirementType.TOP_LEVEL) - bt._phase_start(item) + item.run(bt) key = f"{canonicalize_name('testpkg')}==1.0" assert key not in tmp_context.dependency_graph.nodes @@ -511,9 +607,9 @@ def test_sdist_only_set_for_non_build_requirement( bt.why = [] item = _make_start_item(req_type=RequirementType.INSTALL) - bt._phase_start(item) + item.run(bt) - assert item.build_sdist_only is True + assert item.work_item.build_sdist_only is True def test_sdist_only_not_set_for_build_requirement( self, tmp_context: WorkContext @@ -522,22 +618,42 @@ def test_sdist_only_not_set_for_build_requirement( bt.why = [] item = _make_start_item(req_type=RequirementType.BUILD_SYSTEM) - bt._phase_start(item) + item.run(bt) - assert item.build_sdist_only is False + assert item.work_item.build_sdist_only is False def test_marks_as_seen(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) bt.why = [] item = _make_start_item() - assert item.resolved_version is not None + wi = item.work_item + assert wi.resolved_version is not None - assert not bt._has_been_seen(item.req, item.resolved_version) - bt._phase_start(item) - assert bt._has_been_seen(item.req, item.resolved_version) + assert not bt.has_been_seen(wi.req, wi.resolved_version) + item.run(bt) + assert bt.has_been_seen(wi.req, wi.resolved_version) + def test_sets_pbi_pre_built_before_prepare_source( + self, tmp_context: WorkContext + ) -> None: + """pbi_pre_built is set on work_item before PrepareSource is constructed.""" + bt = bootstrapper.Bootstrapper(tmp_context) + bt.why = [] + item = _make_start_item() -class TestPhaseComplete: + with patch.object( + tmp_context, + "package_build_info", + return_value=Mock(pre_built=True), + ): + result = item.run(bt) + + assert len(result) == 1 + assert isinstance(result[0], PrepareSource) + assert result[0].work_item.pbi_pre_built is True + + +class TestComplete: def test_calls_clean_build_dirs(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) mock_sdist_root = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" @@ -551,10 +667,10 @@ def test_calls_clean_build_dirs(self, tmp_context: WorkContext) -> None: source_type=SourceType.SDIST, ) item = _make_build_item(phase=BootstrapPhase.COMPLETE) - item.build_result = build_result + item.work_item.build_result = build_result with patch.object(tmp_context, "clean_build_dirs") as mock_clean: - result = bt._phase_complete(item) + result = item.run(bt) assert result == [] mock_clean.assert_called_once_with(mock_sdist_root, mock_env) @@ -562,40 +678,18 @@ def test_calls_clean_build_dirs(self, tmp_context: WorkContext) -> None: def test_no_build_result_skips_cleanup(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) item = _make_build_item(phase=BootstrapPhase.COMPLETE) - item.build_result = None + item.work_item.build_result = None with patch.object(tmp_context, "clean_build_dirs") as mock_clean: - result = bt._phase_complete(item) + result = item.run(bt) assert result == [] mock_clean.assert_not_called() - -class TestDispatchPhase: - @pytest.mark.parametrize( - "phase,method_name", - [ - (BootstrapPhase.RESOLVE, "_phase_resolve"), - (BootstrapPhase.START, "_phase_start"), - (BootstrapPhase.PREPARE_SOURCE, "_phase_prepare_source"), - (BootstrapPhase.PREPARE_BUILD, "_phase_prepare_build"), - (BootstrapPhase.BUILD, "_phase_build"), - (BootstrapPhase.PROCESS_INSTALL_DEPS, "_phase_process_install_deps"), - (BootstrapPhase.COMPLETE, "_phase_complete"), - ], - ) - def test_routes_to_correct_handler( - self, tmp_context: WorkContext, phase: BootstrapPhase, method_name: str - ) -> None: + def test_returns_empty_list(self, tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) - item = _make_build_item(phase=phase) - expected = [item] - - with patch.object(bt, method_name, return_value=expected) as mock_method: - result = bt._dispatch_phase(item) - - assert result is expected - mock_method.assert_called_once_with(item) + item = _make_build_item(phase=BootstrapPhase.COMPLETE) + assert item.run(bt) == [] class TestHandlePhaseError: @@ -654,7 +748,7 @@ def test_build_phase_test_mode_fallback_success( ) -> None: bt = bootstrapper.Bootstrapper(tmp_context, test_mode=True) item = _make_build_item(phase=BootstrapPhase.PREPARE_SOURCE) - item.pbi_pre_built = False + item.work_item.pbi_pre_built = False err = RuntimeError("build failed") mock_fallback = Mock(spec=SourceBuildResult) @@ -662,9 +756,9 @@ def test_build_phase_test_mode_fallback_success( result = bt._handle_phase_error(item, err) assert len(result) == 1 - assert result[0] is item - assert item.build_result is mock_fallback - assert item.phase == BootstrapPhase.PROCESS_INSTALL_DEPS + assert isinstance(result[0], ProcessInstallDeps) + assert result[0].work_item is item.work_item + assert result[0].work_item.build_result is mock_fallback assert len(bt.failed_packages) == 0 def test_build_phase_test_mode_fallback_failure( @@ -672,7 +766,7 @@ def test_build_phase_test_mode_fallback_failure( ) -> None: bt = bootstrapper.Bootstrapper(tmp_context, test_mode=True) item = _make_build_item(phase=BootstrapPhase.BUILD) - item.pbi_pre_built = False + item.work_item.pbi_pre_built = False err = RuntimeError("build failed") with patch.object(bt, "_handle_test_mode_failure", return_value=None): @@ -687,7 +781,7 @@ def test_build_phase_test_mode_prebuilt_skips_fallback( ) -> None: bt = bootstrapper.Bootstrapper(tmp_context, test_mode=True) item = _make_build_item(phase=BootstrapPhase.PREPARE_SOURCE) - item.pbi_pre_built = True + item.work_item.pbi_pre_built = True err = RuntimeError("download failed") result = bt._handle_phase_error(item, err) @@ -716,8 +810,9 @@ def test_multiple_versions_records_and_removes_from_graph( ) -> None: bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) item = _make_build_item(phase=BootstrapPhase.BUILD) - assert item.resolved_version is not None - assert item.source_url is not None + wi = item.work_item + assert wi.resolved_version is not None + assert wi.source_url is not None err = ValueError("build failed") # Add to graph first so remove_dependency has something to remove @@ -725,13 +820,13 @@ def test_multiple_versions_records_and_removes_from_graph( parent_name=None, parent_version=None, req_type=RequirementType.TOP_LEVEL, - req=item.req, - req_version=item.resolved_version, - download_url=item.source_url, + req=wi.req, + req_version=wi.resolved_version, + download_url=wi.source_url, pre_built=False, ) # Mark as seen - bt._mark_as_seen(item.req, item.resolved_version) + bt.mark_as_seen(wi.req, wi.resolved_version) result = bt._handle_phase_error(item, err) @@ -743,8 +838,8 @@ def test_multiple_versions_records_and_removes_from_graph( key = f"{canonicalize_name('testpkg')}==1.0" assert key not in tmp_context.dependency_graph.nodes # Seen markers cleared - assert not bt._has_been_seen(item.req, item.resolved_version) - assert not bt._has_been_seen(item.req, item.resolved_version, sdist_only=True) + assert not bt.has_been_seen(wi.req, wi.resolved_version) + assert not bt.has_been_seen(wi.req, wi.resolved_version, sdist_only=True) def test_multiple_versions_logs_phase( self, tmp_context: WorkContext, caplog: pytest.LogCaptureFixture @@ -777,45 +872,68 @@ def test_full_lifecycle_source_package(self, tmp_context: WorkContext) -> None: """Drive a package through RESOLVE -> START -> ... -> COMPLETE.""" bt = bootstrapper.Bootstrapper(tmp_context) - # Track which phases are visited phases_visited: list[BootstrapPhase] = [] - original_dispatch = bt._dispatch_phase - - def tracking_dispatch(item: WorkItem) -> list[WorkItem]: - phases_visited.append(item.phase) - if item.phase == BootstrapPhase.RESOLVE: - return original_dispatch(item) - if item.phase == BootstrapPhase.START: - return original_dispatch(item) - if item.phase == BootstrapPhase.PREPARE_SOURCE: - # Skip actual source download, simulate source build path - item.phase = BootstrapPhase.PREPARE_BUILD - item.build_env = Mock() - item.sdist_root_dir = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" - return [item] - if item.phase == BootstrapPhase.PREPARE_BUILD: - item.phase = BootstrapPhase.BUILD - return [item] - if item.phase == BootstrapPhase.BUILD: - item.build_result = SourceBuildResult( - wheel_filename=None, - sdist_filename=None, - unpack_dir=tmp_context.work_dir, - sdist_root_dir=None, - build_env=None, - source_type=SourceType.SDIST, - ) - item.phase = BootstrapPhase.PROCESS_INSTALL_DEPS - return [item] - if item.phase == BootstrapPhase.PROCESS_INSTALL_DEPS: - item.phase = BootstrapPhase.COMPLETE - return [item] - if item.phase == BootstrapPhase.COMPLETE: - return [] + original_resolve_run = Resolve.run + original_start_run = Start.run + + def resolve_run( + self: Resolve, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: + phases_visited.append(type(self).phase) + return original_resolve_run(self, bt_arg) + + def start_run(self: Start, bt_arg: bootstrapper.Bootstrapper) -> list[Phase]: + phases_visited.append(type(self).phase) + return original_start_run(self, bt_arg) + + def prepare_source_run( + self: PrepareSource, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: + phases_visited.append(type(self).phase) + wi = self.work_item + wi.build_env = Mock() + wi.sdist_root_dir = tmp_context.work_dir / "pkg-1.0" / "pkg-1.0" + return [PrepareBuild(wi)] + + def prepare_build_run( + self: PrepareBuild, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: + phases_visited.append(type(self).phase) + return [Build(self.work_item)] + + def build_run(self: Build, bt_arg: bootstrapper.Bootstrapper) -> list[Phase]: + phases_visited.append(type(self).phase) + wi = self.work_item + wi.build_result = SourceBuildResult( + wheel_filename=None, + sdist_filename=None, + unpack_dir=tmp_context.work_dir, + sdist_root_dir=None, + build_env=None, + source_type=SourceType.SDIST, + ) + return [ProcessInstallDeps(wi)] + + def process_install_deps_run( + self: ProcessInstallDeps, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: + phases_visited.append(type(self).phase) + return [Complete(self.work_item)] + + def complete_run( + self: Complete, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: + phases_visited.append(type(self).phase) return [] with ( - patch.object(bt, "_dispatch_phase", side_effect=tracking_dispatch), + patch.object(Resolve, "run", resolve_run), + patch.object(Start, "run", start_run), + patch.object(PrepareSource, "run", prepare_source_run), + patch.object(PrepareBuild, "run", prepare_build_run), + patch.object(Build, "run", build_run), + patch.object(ProcessInstallDeps, "run", process_install_deps_run), + patch.object(Complete, "run", complete_run), patch.object( bt._resolver, "resolve", @@ -841,31 +959,49 @@ def test_lifo_ordering_deps_before_continuation( bt = bootstrapper.Bootstrapper(tmp_context) processing_order: list[tuple[str, str]] = [] - original_dispatch = bt._dispatch_phase + original_resolve_run = Resolve.run + original_start_run = Start.run + + def resolve_run( + self: Resolve, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: + processing_order.append( + (str(self.work_item.req.name), str(type(self).phase)) + ) + return original_resolve_run(self, bt_arg) - def tracking_dispatch(item: WorkItem) -> list[WorkItem]: - phase_name = str(item.phase) - req_name = str(item.req.name) - processing_order.append((req_name, phase_name)) + def start_run(self: Start, bt_arg: bootstrapper.Bootstrapper) -> list[Phase]: + processing_order.append( + (str(self.work_item.req.name), str(type(self).phase)) + ) + return original_start_run(self, bt_arg) - if item.phase == BootstrapPhase.RESOLVE: - return original_dispatch(item) - if item.phase == BootstrapPhase.START: - return original_dispatch(item) - - # Parent discovers a dep at PREPARE_SOURCE - if req_name == "parent" and item.phase == BootstrapPhase.PREPARE_SOURCE: - assert item.resolved_version is not None - dep_item = WorkItem( - req=Requirement("child"), - req_type=RequirementType.BUILD_SYSTEM, - phase=BootstrapPhase.RESOLVE, - why_snapshot=[], - parent=(item.req, item.resolved_version), + def prepare_source_run( + self: PrepareSource, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: + phase_name = str(type(self).phase) + req_name = str(self.work_item.req.name) + processing_order.append((req_name, phase_name)) + wi = self.work_item + if req_name == "parent": + assert wi.resolved_version is not None + dep_item = Resolve( + WorkItem( + req=Requirement("child"), + req_type=RequirementType.BUILD_SYSTEM, + why_snapshot=[], + parent=(wi.req, wi.resolved_version), + ) ) - item.phase = BootstrapPhase.COMPLETE - return [item, dep_item] + return [Complete(wi), dep_item] + return [] + def complete_run( + self: Complete, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: + processing_order.append( + (str(self.work_item.req.name), str(type(self).phase)) + ) return [] # Pre-add parent to graph so child can reference it as parent @@ -880,7 +1016,10 @@ def tracking_dispatch(item: WorkItem) -> list[WorkItem]: ) with ( - patch.object(bt, "_dispatch_phase", side_effect=tracking_dispatch), + patch.object(Resolve, "run", resolve_run), + patch.object(Start, "run", start_run), + patch.object(PrepareSource, "run", prepare_source_run), + patch.object(Complete, "run", complete_run), patch.object( bt._resolver, "resolve", @@ -911,18 +1050,15 @@ def test_multiple_versions_error_isolation(self, tmp_context: WorkContext) -> No """Each version fails independently without crashing the loop.""" bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) - original_dispatch = bt._dispatch_phase - - def mock_dispatch(item: WorkItem) -> list[WorkItem]: - if item.phase in (BootstrapPhase.RESOLVE, BootstrapPhase.START): - return original_dispatch(item) - # Fail version 1.5, succeed for others - if str(item.resolved_version) == "1.5": + def prepare_source_run( + self: PrepareSource, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: + if str(self.work_item.resolved_version) == "1.5": raise ValueError("1.5 broken") return [] with ( - patch.object(bt, "_dispatch_phase", side_effect=mock_dispatch), + patch.object(PrepareSource, "run", prepare_source_run), patch.object( bt._resolver, "resolve", @@ -932,7 +1068,7 @@ def mock_dispatch(item: WorkItem) -> list[WorkItem]: ("url-1.0", Version("1.0")), ], ), - patch.object(bt, "_has_been_seen", return_value=False), + patch.object(bt, "has_been_seen", return_value=False), ): bt._bootstrap_one(Requirement("pkg"), RequirementType.INSTALL) @@ -948,27 +1084,31 @@ def test_multiple_versions_resolve_failure_continues( """RESOLVE failure for one dependency does not crash the loop.""" bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=True) - original_dispatch = bt._dispatch_phase + original_resolve_run = Resolve.run completed: list[str] = [] - def mock_dispatch(item: WorkItem) -> list[WorkItem]: - if item.phase == BootstrapPhase.START: - return original_dispatch(item) - if item.phase == BootstrapPhase.RESOLVE: - if str(item.req.name) == "bad-dep": - raise RuntimeError("Could not resolve any versions for bad-dep") - return original_dispatch(item) - completed.append(str(item.req.name)) + def resolve_run( + self: Resolve, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: + if str(self.work_item.req.name) == "bad-dep": + raise RuntimeError("Could not resolve any versions for bad-dep") + return original_resolve_run(self, bt_arg) + + def prepare_source_run( + self: PrepareSource, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: + completed.append(str(self.work_item.req.name)) return [] with ( - patch.object(bt, "_dispatch_phase", side_effect=mock_dispatch), + patch.object(Resolve, "run", resolve_run), + patch.object(PrepareSource, "run", prepare_source_run), patch.object( bt._resolver, "resolve", return_value=[("url-1.0", Version("1.0"))], ), - patch.object(bt, "_has_been_seen", return_value=False), + patch.object(bt, "has_been_seen", return_value=False), ): bt._bootstrap_one(Requirement("good-pkg"), RequirementType.INSTALL) bt._bootstrap_one(Requirement("bad-dep"), RequirementType.INSTALL) @@ -983,19 +1123,18 @@ def test_test_mode_continues_after_failure(self, tmp_context: WorkContext) -> No """In test mode, failed items are recorded and processing continues.""" bt = bootstrapper.Bootstrapper(tmp_context, test_mode=True) - original_dispatch = bt._dispatch_phase items_completed: list[str] = [] - def mock_dispatch(item: WorkItem) -> list[WorkItem]: - if item.phase in (BootstrapPhase.RESOLVE, BootstrapPhase.START): - return original_dispatch(item) - if str(item.req.name) == "fail-pkg": + def prepare_source_run( + self: PrepareSource, bt_arg: bootstrapper.Bootstrapper + ) -> list[Phase]: + if str(self.work_item.req.name) == "fail-pkg": raise RuntimeError("build error") - items_completed.append(str(item.req.name)) + items_completed.append(str(self.work_item.req.name)) return [] with ( - patch.object(bt, "_dispatch_phase", side_effect=mock_dispatch), + patch.object(PrepareSource, "run", prepare_source_run), patch.object( bt._resolver, "resolve", @@ -1013,7 +1152,7 @@ def mock_dispatch(item: WorkItem) -> list[WorkItem]: class TestPhasePrepareSource: - """Tests for _phase_prepare_source: prebuilt, source, cache, and error paths.""" + """Tests for PrepareSource.run(): prebuilt, source, cache, and error paths.""" def test_prebuilt_uses_background_result(self, tmp_context: WorkContext) -> None: """Prebuilt package uses bg_future result and advances to PROCESS_INSTALL_DEPS.""" @@ -1027,22 +1166,21 @@ def test_prebuilt_uses_background_result(self, tmp_context: WorkContext) -> None mock_wheel = tmp_context.work_dir / "testpkg-1.0-py3-none-any.whl" mock_unpack = tmp_context.work_dir / "testpkg-1.0" item.bg_future = _make_resolved_future( - bootstrapper.PreparedSourceData( - wheel_filename=mock_wheel, unpack_dir=mock_unpack - ) + PreparedSourceData(wheel_filename=mock_wheel, unpack_dir=mock_unpack) ) with patch.object(tmp_context.constraints, "get_constraint", return_value=None): - result = bt._phase_prepare_source(item) + result = item.run(bt) assert len(result) == 1 - assert result[0] is item - assert item.phase == BootstrapPhase.PROCESS_INSTALL_DEPS - assert item.build_result is not None - assert item.build_result.source_type == SourceType.PREBUILT - assert item.build_result.wheel_filename == mock_wheel - assert item.build_result.sdist_filename is None - assert item.build_result.build_env is None + assert isinstance(result[0], ProcessInstallDeps) + assert result[0].work_item is item.work_item + wi = result[0].work_item + assert wi.build_result is not None + assert wi.build_result.source_type == SourceType.PREBUILT + assert wi.build_result.wheel_filename == mock_wheel + assert wi.build_result.sdist_filename is None + assert wi.build_result.build_env is None def test_source_no_cache_uses_background_result( self, tmp_context: WorkContext @@ -1055,37 +1193,42 @@ def test_source_no_cache_uses_background_result( mock_env = Mock() mock_dep_item = _make_resolve_item(req="setuptools") item.bg_future = _make_resolved_future( - bootstrapper.PreparedSourceData(sdist_root_dir=sdist_root) + PreparedSourceData(sdist_root_dir=sdist_root) ) with ( patch.object(tmp_context.constraints, "get_constraint", return_value=None), - patch.object( - bt, "_create_build_env", return_value=mock_env - ) as mock_create_env, + patch( + "fromager.build_environment.BuildEnvironment", + return_value=mock_env, + ) as mock_build_env_cls, patch( "fromager.dependencies.get_build_system_dependencies", return_value={Requirement("setuptools")}, ), patch.object( - bt, "_create_unresolved_work_items", return_value=[mock_dep_item] + bt, "create_unresolved_work_items", return_value=[mock_dep_item] ) as mock_create_items, ): - result = bt._phase_prepare_source(item) - - assert item.phase == BootstrapPhase.PREPARE_BUILD - assert item.build_env is mock_env - assert item.sdist_root_dir == sdist_root - assert item.unpack_dir == sdist_root.parent - assert item.cached_wheel_filename is None - assert result[0] is item + result = item.run(bt) + + wi = item.work_item + assert isinstance(result[0], PrepareBuild) + assert result[0].work_item is wi + assert wi.build_env is mock_env + assert wi.sdist_root_dir == sdist_root + assert wi.unpack_dir == sdist_root.parent + assert wi.cached_wheel_filename is None assert result[1] is mock_dep_item - mock_create_env.assert_called_once() + mock_build_env_cls.assert_called_once_with( + ctx=bt.ctx, + parent_dir=sdist_root.parent, + ) mock_create_items.assert_called_once_with( - item.build_system_deps, + wi.build_system_deps, RequirementType.BUILD_SYSTEM, - item.req, - item.resolved_version, + wi.req, + wi.resolved_version, ) def test_source_cached_wheel_uses_background_result( @@ -1100,7 +1243,7 @@ def test_source_cached_wheel_uses_background_result( cached_wheel = tmp_context.work_dir / "testpkg-1.0-py3-none-any.whl" mock_env = Mock() item.bg_future = _make_resolved_future( - bootstrapper.PreparedSourceData( + PreparedSourceData( sdist_root_dir=unpacked / unpacked.stem, cached_wheel_filename=cached_wheel, ) @@ -1108,18 +1251,22 @@ def test_source_cached_wheel_uses_background_result( with ( patch.object(tmp_context.constraints, "get_constraint", return_value=None), - patch.object(bt, "_create_build_env", return_value=mock_env), + patch( + "fromager.build_environment.BuildEnvironment", + return_value=mock_env, + ), patch( "fromager.dependencies.get_build_system_dependencies", return_value=set(), ), - patch.object(bt, "_create_unresolved_work_items", return_value=[]), + patch.object(bt, "create_unresolved_work_items", return_value=[]), ): - result = bt._phase_prepare_source(item) + result = item.run(bt) - assert item.cached_wheel_filename == cached_wheel - assert item.sdist_root_dir == unpacked / unpacked.stem - assert item.phase == BootstrapPhase.PREPARE_BUILD + wi = item.work_item + assert wi.cached_wheel_filename == cached_wheel + assert wi.sdist_root_dir == unpacked / unpacked.stem + assert isinstance(result[0], PrepareBuild) assert len(result) == 1 def test_bad_sdist_root_raises_valueerror(self, tmp_context: WorkContext) -> None: @@ -1129,12 +1276,12 @@ def test_bad_sdist_root_raises_valueerror(self, tmp_context: WorkContext) -> Non bad_root = tmp_context.work_dir / "a" / "b" / "c" item.bg_future = _make_resolved_future( - bootstrapper.PreparedSourceData(sdist_root_dir=bad_root) + PreparedSourceData(sdist_root_dir=bad_root) ) with patch.object(tmp_context.constraints, "get_constraint", return_value=None): with pytest.raises(ValueError, match="should be"): - bt._phase_prepare_source(item) + item.run(bt) def test_constraint_logged_when_present( self, tmp_context: WorkContext, caplog: pytest.LogCaptureFixture @@ -1146,7 +1293,7 @@ def test_constraint_logged_when_present( sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" mock_env = Mock() item.bg_future = _make_resolved_future( - bootstrapper.PreparedSourceData(sdist_root_dir=sdist_root) + PreparedSourceData(sdist_root_dir=sdist_root) ) with ( @@ -1155,21 +1302,24 @@ def test_constraint_logged_when_present( "get_constraint", return_value=Requirement("testpkg>=1.0"), ), - patch.object(bt, "_create_build_env", return_value=mock_env), + patch( + "fromager.build_environment.BuildEnvironment", + return_value=mock_env, + ), patch( "fromager.dependencies.get_build_system_dependencies", return_value=set(), ), - patch.object(bt, "_create_unresolved_work_items", return_value=[]), + patch.object(bt, "create_unresolved_work_items", return_value=[]), ): - bt._phase_prepare_source(item) + result = item.run(bt) - assert item.phase == BootstrapPhase.PREPARE_BUILD + assert isinstance(result[0], PrepareBuild) assert "matches constraint" in caplog.text class TestPhasePrepareBuild: - """Tests for _phase_prepare_build: dep installation and extraction.""" + """Tests for PrepareBuild.run(): dep installation and extraction.""" def test_installs_system_deps_and_returns_backend_sdist_items( self, tmp_context: WorkContext @@ -1185,6 +1335,7 @@ def test_installs_system_deps_and_returns_backend_sdist_items( sdist_root_dir=sdist_root, build_system_deps=system_deps, ) + wi = item.work_item backend_item = _make_resolve_item(req="wheel") sdist_item = _make_resolve_item(req="flit-core") @@ -1200,17 +1351,18 @@ def test_installs_system_deps_and_returns_backend_sdist_items( ), patch.object( bt, - "_create_unresolved_work_items", + "create_unresolved_work_items", side_effect=[[backend_item], [sdist_item]], ), ): - result = bt._phase_prepare_build(item) + result = item.run(bt) - assert item.phase == BootstrapPhase.BUILD - assert item.build_backend_deps == {Requirement("wheel")} - assert item.build_sdist_deps == {Requirement("flit-core")} + assert isinstance(result[0], Build) + assert result[0].work_item is wi + assert wi.build_backend_deps == {Requirement("wheel")} + assert wi.build_sdist_deps == {Requirement("flit-core")} mock_env.install.assert_called_once_with(system_deps) - assert result == [item, backend_item, sdist_item] + assert result == [result[0], backend_item, sdist_item] def test_no_extra_deps_returns_item_only(self, tmp_context: WorkContext) -> None: """When backend and sdist deps are empty, returns only the item.""" @@ -1236,14 +1388,14 @@ def test_no_extra_deps_returns_item_only(self, tmp_context: WorkContext) -> None ), patch.object( bt, - "_create_unresolved_work_items", + "create_unresolved_work_items", side_effect=[[], []], ), ): - result = bt._phase_prepare_build(item) + result = item.run(bt) - assert result == [item] - assert item.phase == BootstrapPhase.BUILD + assert len(result) == 1 + assert isinstance(result[0], Build) mock_env.install.assert_called_once_with(system_deps) def test_install_called_once_with_system_deps( @@ -1272,11 +1424,11 @@ def test_install_called_once_with_system_deps( ), patch.object( bt, - "_create_unresolved_work_items", + "create_unresolved_work_items", side_effect=[[], []], ), ): - bt._phase_prepare_build(item) + item.run(bt) mock_env.install.assert_called_once_with(system_deps) @@ -1293,6 +1445,7 @@ def test_creates_items_with_correct_requirement_types( sdist_root_dir=sdist_root, build_system_deps={Requirement("setuptools")}, ) + wi = item.work_item with ( patch( @@ -1305,35 +1458,35 @@ def test_creates_items_with_correct_requirement_types( ), patch.object( bt, - "_create_unresolved_work_items", + "create_unresolved_work_items", side_effect=[[], []], ) as mock_create_items, ): - bt._phase_prepare_build(item) + item.run(bt) calls = mock_create_items.call_args_list assert calls[0] == call( {Requirement("wheel")}, RequirementType.BUILD_BACKEND, - item.req, - item.resolved_version, + wi.req, + wi.resolved_version, ) assert calls[1] == call( {Requirement("flit-core")}, RequirementType.BUILD_SDIST, - item.req, - item.resolved_version, + wi.req, + wi.resolved_version, ) class TestPhaseBuild: - """Tests for _phase_build: conditional dep install and result construction.""" + """Tests for Build.run(): conditional dep install and result construction.""" - def _make_build_phase_item(self, tmp_context: WorkContext) -> WorkItem: + def _make_build_phase_item(self, tmp_context: WorkContext) -> Build: mock_env = Mock() sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" sdist_root.parent.mkdir(parents=True, exist_ok=True) - return _make_build_item( + item = _make_build_item( phase=BootstrapPhase.BUILD, build_env=mock_env, sdist_root_dir=sdist_root, @@ -1341,6 +1494,8 @@ def _make_build_phase_item(self, tmp_context: WorkContext) -> WorkItem: build_backend_deps={Requirement("wheel")}, build_sdist_deps={Requirement("flit-core")}, ) + assert isinstance(item, Build) + return item def test_disjoint_deps_installs_remaining(self, tmp_context: WorkContext) -> None: """Disjoint backend/sdist deps from system deps triggers install.""" @@ -1359,16 +1514,16 @@ def test_disjoint_deps_installs_remaining(self, tmp_context: WorkContext) -> Non mock_wheel = tmp_context.work_dir / "testpkg-1.0-py3-none-any.whl" with ( - patch.object(bt, "_do_build", return_value=(mock_wheel, None)), + patch.object(item, "do_build", return_value=(mock_wheel, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): - result = bt._phase_build(item) + result = item.run(bt) mock_env.install.assert_called_once_with( {Requirement("wheel"), Requirement("flit-core")} ) - assert item.phase == BootstrapPhase.PROCESS_INSTALL_DEPS assert len(result) == 1 + assert isinstance(result[0], ProcessInstallDeps) def test_overlapping_deps_skips_install(self, tmp_context: WorkContext) -> None: """Overlapping backend/sdist deps with system deps skips install.""" @@ -1386,17 +1541,15 @@ def test_overlapping_deps_skips_install(self, tmp_context: WorkContext) -> None: ) with ( - patch.object(bt, "_do_build", return_value=(None, None)), + patch.object(item, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): - bt._phase_build(item) + item.run(bt) mock_env.install.assert_not_called() def test_partial_overlap_deps_skips_install(self, tmp_context: WorkContext) -> None: - """isdisjoint is False on partial overlap, so install is skipped entirely - even for deps not in build_system_deps (here: cython). - """ + """isdisjoint is False on partial overlap, so install is skipped entirely.""" bt = bootstrapper.Bootstrapper(tmp_context) mock_env = Mock() sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" @@ -1411,15 +1564,15 @@ def test_partial_overlap_deps_skips_install(self, tmp_context: WorkContext) -> N ) with ( - patch.object(bt, "_do_build", return_value=(None, None)), + patch.object(item, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): - bt._phase_build(item) + item.run(bt) mock_env.install.assert_not_called() def test_do_build_receives_item_fields(self, tmp_context: WorkContext) -> None: - """build_sdist_only and cached_wheel_filename are forwarded to _do_build.""" + """build_sdist_only and cached_wheel_filename are read directly by do_build, not forwarded as parameters.""" bt = bootstrapper.Bootstrapper(tmp_context) mock_env = Mock() sdist_root = tmp_context.work_dir / "testpkg-1.0" / "testpkg-1.0" @@ -1434,35 +1587,29 @@ def test_do_build_receives_item_fields(self, tmp_context: WorkContext) -> None: ) with ( - patch.object(bt, "_do_build", return_value=(None, None)) as mock_do_build, + patch.object(item, "do_build", return_value=(None, None)) as mock_do_build, patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): - bt._phase_build(item) + item.run(bt) - mock_do_build.assert_called_once_with( - req=item.req, - resolved_version=item.resolved_version, - sdist_root_dir=sdist_root, - build_env=mock_env, - build_sdist_only=True, - cached_wheel_filename=cached_wheel, - ) + mock_do_build.assert_called_once_with(bt.ctx, bt.explain) def test_build_result_references_item_build_env( self, tmp_context: WorkContext ) -> None: - """build_result.build_env is the same object as item.build_env.""" + """build_result.build_env is the same object as item.work_item.build_env.""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_build_phase_item(tmp_context) + wi = item.work_item with ( - patch.object(bt, "_do_build", return_value=(None, None)), + patch.object(item, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): - result = bt._phase_build(item) + item.run(bt) - assert result[0].build_result is not None - assert result[0].build_result.build_env is item.build_env + assert wi.build_result is not None + assert wi.build_result.build_env is wi.build_env def test_build_result_uses_source_type_from_sources( self, tmp_context: WorkContext @@ -1470,38 +1617,55 @@ def test_build_result_uses_source_type_from_sources( """source_type comes from sources.get_source_type, not hardcoded.""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_build_phase_item(tmp_context) + wi = item.work_item with ( - patch.object(bt, "_do_build", return_value=(None, None)), + patch.object(item, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.PREBUILT), ): - bt._phase_build(item) + item.run(bt) - assert item.build_result is not None - assert item.build_result.source_type == SourceType.PREBUILT + assert wi.build_result is not None + assert wi.build_result.source_type == SourceType.PREBUILT - def test_returns_single_item_at_process_install_deps( + def test_returns_single_process_install_deps_item( self, tmp_context: WorkContext ) -> None: - """_phase_build returns exactly [item] with phase PROCESS_INSTALL_DEPS.""" + """Build.run() returns exactly [ProcessInstallDeps].""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_build_phase_item(tmp_context) with ( - patch.object(bt, "_do_build", return_value=(None, None)), + patch.object(item, "do_build", return_value=(None, None)), patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), ): - result = bt._phase_build(item) + result = item.run(bt) assert len(result) == 1 - assert result[0] is item - assert item.phase == BootstrapPhase.PROCESS_INSTALL_DEPS + assert isinstance(result[0], ProcessInstallDeps) + assert result[0].work_item is item.work_item + + def test_phase_advancement_preserves_work_item_identity( + self, tmp_context: WorkContext + ) -> None: + """When Build.run() returns ProcessInstallDeps, work_item is same obj.""" + bt = bootstrapper.Bootstrapper(tmp_context) + item = self._make_build_phase_item(tmp_context) + wi = item.work_item + + with ( + patch.object(item, "do_build", return_value=(None, None)), + patch("fromager.sources.get_source_type", return_value=SourceType.SDIST), + ): + result = item.run(bt) + + assert result[0].work_item is wi class TestPhaseProcessInstallDeps: - """Tests for _phase_process_install_deps: hooks, dep extraction, error modes.""" + """Tests for ProcessInstallDeps.run(): hooks, dep extraction, error modes.""" - def _make_process_item(self, tmp_context: WorkContext) -> WorkItem: + def _make_process_item(self, tmp_context: WorkContext) -> ProcessInstallDeps: build_result = SourceBuildResult( wheel_filename=tmp_context.work_dir / "testpkg-1.0-py3-none-any.whl", sdist_filename=tmp_context.work_dir / "testpkg-1.0.tar.gz", @@ -1510,25 +1674,27 @@ def _make_process_item(self, tmp_context: WorkContext) -> WorkItem: build_env=Mock(), source_type=SourceType.SDIST, ) - return _make_build_item( + item = _make_build_item( phase=BootstrapPhase.PROCESS_INSTALL_DEPS, build_result=build_result, source_url="https://pkg.test/testpkg-1.0.tar.gz", ) + assert isinstance(item, ProcessInstallDeps) + return item def test_normal_path_returns_item_and_dep_items( self, tmp_context: WorkContext ) -> None: - """Normal path: hooks, deps, build order, returns [item, *dep_items].""" + """Normal path: hooks, deps, build order, returns [Complete, *dep_items].""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_process_item(tmp_context) + wi = item.work_item dep_item = _make_resolve_item(req="dep-a") with ( patch("fromager.hooks.run_post_bootstrap_hooks"), - patch.object( - bt, - "_get_install_dependencies", + patch( + "fromager.bootstrapper._process_install_deps._get_install_dependencies", return_value=[Requirement("dep-a")], ), patch.object( @@ -1537,21 +1703,22 @@ def test_normal_path_returns_item_and_dep_items( return_value=Mock(pre_built=False), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), - patch.object(bt, "_add_to_build_order") as mock_build_order, + patch.object(bt, "add_to_build_order") as mock_build_order, patch.object( - bt, "_create_unresolved_work_items", return_value=[dep_item] + bt, "create_unresolved_work_items", return_value=[dep_item] ) as mock_create_items, ): - result = bt._phase_process_install_deps(item) + result = item.run(bt) - assert item.phase == BootstrapPhase.COMPLETE - assert result == [item, dep_item] + assert isinstance(result[0], Complete) + assert result[0].work_item is wi + assert result[1] is dep_item mock_build_order.assert_called_once() mock_create_items.assert_called_once_with( [Requirement("dep-a")], RequirementType.INSTALL, - item.req, - item.resolved_version, + wi.req, + wi.resolved_version, ) def test_hook_error_test_mode_records_and_continues( @@ -1566,8 +1733,9 @@ def test_hook_error_test_mode_records_and_continues( "fromager.hooks.run_post_bootstrap_hooks", side_effect=RuntimeError("hook failed"), ), - patch.object( - bt, "_get_install_dependencies", return_value=[] + patch( + "fromager.bootstrapper._process_install_deps._get_install_dependencies", + return_value=[], ) as mock_get_deps, patch.object( tmp_context, @@ -1575,13 +1743,12 @@ def test_hook_error_test_mode_records_and_continues( return_value=Mock(pre_built=False), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), - patch.object(bt, "_add_to_build_order") as mock_build_order, - patch.object(bt, "_create_unresolved_work_items", return_value=[]), + patch.object(bt, "add_to_build_order") as mock_build_order, + patch.object(bt, "create_unresolved_work_items", return_value=[]), ): - result = bt._phase_process_install_deps(item) + result = item.run(bt) - assert item.phase == BootstrapPhase.COMPLETE - assert result == [item] + assert isinstance(result[0], Complete) assert len(bt.failed_packages) == 1 assert bt.failed_packages[0]["failure_type"] == "hook" mock_get_deps.assert_called_once() @@ -1599,7 +1766,7 @@ def test_hook_error_normal_mode_raises(self, tmp_context: WorkContext) -> None: ), ): with pytest.raises(RuntimeError, match="hook failed"): - bt._phase_process_install_deps(item) + item.run(bt) def test_dep_extraction_error_test_mode_uses_empty_deps( self, tmp_context: WorkContext @@ -1607,12 +1774,12 @@ def test_dep_extraction_error_test_mode_uses_empty_deps( """Dep extraction error in test mode uses empty dep list, still writes build order.""" bt = bootstrapper.Bootstrapper(tmp_context, test_mode=True) item = self._make_process_item(tmp_context) + wi = item.work_item with ( patch("fromager.hooks.run_post_bootstrap_hooks"), - patch.object( - bt, - "_get_install_dependencies", + patch( + "fromager.bootstrapper._process_install_deps._get_install_dependencies", side_effect=RuntimeError("dep failed"), ), patch.object( @@ -1621,23 +1788,22 @@ def test_dep_extraction_error_test_mode_uses_empty_deps( return_value=Mock(pre_built=False), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), - patch.object(bt, "_add_to_build_order") as mock_build_order, + patch.object(bt, "add_to_build_order") as mock_build_order, patch.object( - bt, "_create_unresolved_work_items", return_value=[] + bt, "create_unresolved_work_items", return_value=[] ) as mock_create_items, ): - result = bt._phase_process_install_deps(item) + result = item.run(bt) - assert item.phase == BootstrapPhase.COMPLETE - assert result == [item] + assert isinstance(result[0], Complete) assert len(bt.failed_packages) == 1 assert bt.failed_packages[0]["failure_type"] == "dependency_extraction" mock_build_order.assert_called_once() mock_create_items.assert_called_once_with( [], RequirementType.INSTALL, - item.req, - item.resolved_version, + wi.req, + wi.resolved_version, ) def test_dep_extraction_error_normal_mode_raises( @@ -1649,48 +1815,54 @@ def test_dep_extraction_error_normal_mode_raises( with ( patch("fromager.hooks.run_post_bootstrap_hooks"), - patch.object( - bt, - "_get_install_dependencies", + patch( + "fromager.bootstrapper._process_install_deps._get_install_dependencies", side_effect=RuntimeError("dep failed"), ), ): with pytest.raises(RuntimeError, match="dep failed"): - bt._phase_process_install_deps(item) + item.run(bt) def test_no_install_deps_returns_item_only(self, tmp_context: WorkContext) -> None: - """When no install deps, returns [item] at COMPLETE phase.""" + """When no install deps, returns [Complete].""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_process_item(tmp_context) with ( patch("fromager.hooks.run_post_bootstrap_hooks"), - patch.object(bt, "_get_install_dependencies", return_value=[]), + patch( + "fromager.bootstrapper._process_install_deps._get_install_dependencies", + return_value=[], + ), patch.object( tmp_context, "package_build_info", return_value=Mock(pre_built=False), ), patch.object(tmp_context.constraints, "get_constraint", return_value=None), - patch.object(bt, "_add_to_build_order"), - patch.object(bt, "_create_unresolved_work_items", return_value=[]), + patch.object(bt, "add_to_build_order"), + patch.object(bt, "create_unresolved_work_items", return_value=[]), ): - result = bt._phase_process_install_deps(item) + result = item.run(bt) - assert result == [item] - assert item.phase == BootstrapPhase.COMPLETE + assert len(result) == 1 + assert isinstance(result[0], Complete) def test_build_order_called_with_correct_args( self, tmp_context: WorkContext ) -> None: - """_add_to_build_order receives correct source_type, prebuilt, constraint.""" + """add_to_build_order receives correct source_type, prebuilt, constraint.""" bt = bootstrapper.Bootstrapper(tmp_context) item = self._make_process_item(tmp_context) + wi = item.work_item constraint = Requirement("testpkg>=1.0") with ( patch("fromager.hooks.run_post_bootstrap_hooks"), - patch.object(bt, "_get_install_dependencies", return_value=[]), + patch( + "fromager.bootstrapper._process_install_deps._get_install_dependencies", + return_value=[], + ), patch.object( tmp_context, "package_build_info", @@ -1701,15 +1873,15 @@ def test_build_order_called_with_correct_args( "get_constraint", return_value=constraint, ), - patch.object(bt, "_add_to_build_order") as mock_build_order, - patch.object(bt, "_create_unresolved_work_items", return_value=[]), + patch.object(bt, "add_to_build_order") as mock_build_order, + patch.object(bt, "create_unresolved_work_items", return_value=[]), ): - bt._phase_process_install_deps(item) + item.run(bt) mock_build_order.assert_called_once_with( - req=item.req, - version=item.resolved_version, - source_url=item.source_url, + req=wi.req, + version=wi.resolved_version, + source_url=wi.source_url, source_type=SourceType.SDIST, prebuilt=True, constraint=constraint, @@ -1762,8 +1934,8 @@ def test_satisfied_dep_reuses_build_system_version( } parent = (Requirement("biotite==1.6.0"), Version("1.6.0")) - with patch.object(bt, "_add_to_graph") as mock_add: - result = bt._filter_deps_satisfied_by_build_system( + with patch.object(bt, "add_to_graph") as mock_add: + result = bt.filter_deps_satisfied_by_build_system( {Requirement("hatch-cython")}, resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -1789,8 +1961,8 @@ def test_satisfied_dep_with_compatible_specifier( } parent = (Requirement("testpkg==1.0"), Version("1.0")) - with patch.object(bt, "_add_to_graph") as mock_add: - result = bt._filter_deps_satisfied_by_build_system( + with patch.object(bt, "add_to_graph") as mock_add: + result = bt.filter_deps_satisfied_by_build_system( {Requirement("foo>=1.0")}, resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -1809,8 +1981,8 @@ def test_unsatisfied_dep_passes_through(self, tmp_context: WorkContext) -> None: parent = (Requirement("testpkg==1.0"), Version("1.0")) wheel_req = Requirement("wheel") - with patch.object(bt, "_add_to_graph") as mock_add: - result = bt._filter_deps_satisfied_by_build_system( + with patch.object(bt, "add_to_graph") as mock_add: + result = bt.filter_deps_satisfied_by_build_system( {wheel_req}, resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -1831,8 +2003,8 @@ def test_incompatible_specifier_passes_through( parent = (Requirement("testpkg==1.0"), Version("1.0")) foo_req = Requirement("foo>=2.0") - with patch.object(bt, "_add_to_graph") as mock_add: - result = bt._filter_deps_satisfied_by_build_system( + with patch.object(bt, "add_to_graph") as mock_add: + result = bt.filter_deps_satisfied_by_build_system( {foo_req}, resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -1852,8 +2024,8 @@ def test_incompatible_specifier_logs_warning( } parent = (Requirement("testpkg==1.0"), Version("1.0")) - with patch.object(bt, "_add_to_graph"): - bt._filter_deps_satisfied_by_build_system( + with patch.object(bt, "add_to_graph"): + bt.filter_deps_satisfied_by_build_system( {Requirement("foo>=2.0")}, resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -1877,8 +2049,8 @@ def test_mixed_satisfied_and_unsatisfied(self, tmp_context: WorkContext) -> None cython_req = Requirement("hatch-cython") wheel_req = Requirement("wheel") - with patch.object(bt, "_add_to_graph") as mock_add: - result = bt._filter_deps_satisfied_by_build_system( + with patch.object(bt, "add_to_graph") as mock_add: + result = bt.filter_deps_satisfied_by_build_system( {cython_req, wheel_req}, resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -1896,7 +2068,7 @@ def test_empty_deps_returns_empty(self, tmp_context: WorkContext) -> None: } parent = (Requirement("testpkg==1.0"), Version("1.0")) - result = bt._filter_deps_satisfied_by_build_system( + result = bt.filter_deps_satisfied_by_build_system( set(), resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -1913,7 +2085,7 @@ def test_empty_build_system_returns_all_deps( wheel_req = Requirement("wheel") parent = (Requirement("testpkg==1.0"), Version("1.0")) - result = bt._filter_deps_satisfied_by_build_system( + result = bt.filter_deps_satisfied_by_build_system( {wheel_req}, {}, RequirementType.BUILD_BACKEND, @@ -1931,8 +2103,8 @@ def test_dep_with_extras_passes_through(self, tmp_context: WorkContext) -> None: parent = (Requirement("testpkg==1.0"), Version("1.0")) extras_req = Requirement("foo[bar]>=1.0") - with patch.object(bt, "_add_to_graph") as mock_add: - result = bt._filter_deps_satisfied_by_build_system( + with patch.object(bt, "add_to_graph") as mock_add: + result = bt.filter_deps_satisfied_by_build_system( {extras_req}, resolved_build_sys, RequirementType.BUILD_BACKEND, @@ -1944,7 +2116,7 @@ def test_dep_with_extras_passes_through(self, tmp_context: WorkContext) -> None: class TestGetResolvedBuildSystemVersions: - """Tests for _get_resolved_build_system_versions.""" + """Tests for get_resolved_build_system_versions.""" def test_returns_build_system_edges(self, tmp_context: WorkContext) -> None: """Returns resolved versions from BUILD_SYSTEM edges.""" @@ -1974,9 +2146,9 @@ def test_returns_build_system_edges(self, tmp_context: WorkContext) -> None: ) bt = bootstrapper.Bootstrapper(tmp_context) - item = _make_build_item(req="biotite", version="1.6.0") + wi = _make_work_item(req="biotite", version="1.6.0") - result = bt._get_resolved_build_system_versions(item) + result = bt.get_resolved_build_system_versions(wi) assert canonicalize_name("hatch-cython") in result assert result[canonicalize_name("hatch-cython")] == ( @@ -1990,15 +2162,15 @@ def test_returns_empty_when_parent_not_in_graph( ) -> None: """Returns empty dict when parent node is not in the graph.""" bt = bootstrapper.Bootstrapper(tmp_context) - item = _make_build_item(req="nonexistent", version="1.0") + wi = _make_work_item(req="nonexistent", version="1.0") - result = bt._get_resolved_build_system_versions(item) + result = bt.get_resolved_build_system_versions(wi) assert result == {} class TestPhasePrepareBuildFiltering: - """Tests for _phase_prepare_build with build-system satisfaction filtering.""" + """Tests for PrepareBuild.run() with build-system satisfaction filtering.""" def test_satisfied_backend_dep_skips_resolve( self, tmp_context: WorkContext @@ -2032,6 +2204,7 @@ def test_satisfied_backend_dep_skips_resolve( sdist_root_dir=sdist_root, build_system_deps={Requirement("hatch-cython==0.5")}, ) + wi = item.work_item with ( patch( @@ -2044,18 +2217,18 @@ def test_satisfied_backend_dep_skips_resolve( ), patch.object( bt, - "_create_unresolved_work_items", + "create_unresolved_work_items", return_value=[], ) as mock_create, ): - result = bt._phase_prepare_build(item) + result = item.run(bt) - assert item.phase == BootstrapPhase.BUILD + assert isinstance(result[0], Build) calls = mock_create.call_args_list assert calls[0][0][0] == set() assert calls[1][0][0] == set() - assert item.build_backend_deps == set() - assert result == [item] + assert wi.build_backend_deps == set() + assert len(result) == 1 def test_extras_dep_not_filtered(self, tmp_context: WorkContext) -> None: """Backend dep with extras passes through even if name matches.""" @@ -2100,16 +2273,16 @@ def test_extras_dep_not_filtered(self, tmp_context: WorkContext) -> None: ), patch.object( bt, - "_create_unresolved_work_items", + "create_unresolved_work_items", side_effect=[[resolve_item], []], ) as mock_create, ): - result = bt._phase_prepare_build(item) + result = item.run(bt) calls = mock_create.call_args_list assert calls[0][0][0] == {extras_req} - assert item.build_backend_deps == {extras_req} - assert result == [item, resolve_item] + assert item.work_item.build_backend_deps == {extras_req} + assert result[1] is resolve_item def test_unsatisfied_backend_dep_creates_resolve_item( self, tmp_context: WorkContext @@ -2147,34 +2320,104 @@ def test_unsatisfied_backend_dep_creates_resolve_item( ), patch.object( bt, - "_create_unresolved_work_items", + "create_unresolved_work_items", side_effect=[[resolve_item], []], ) as mock_create, ): - result = bt._phase_prepare_build(item) + result = item.run(bt) calls = mock_create.call_args_list assert calls[0][0][0] == {Requirement("wheel")} - assert result == [item, resolve_item] + assert result[1] is resolve_item + + +class TestBackgroundWork: + """Tests for background_work() dispatch on Phase subclasses.""" + + def test_resolve_item_background_work_returns_callable( + self, tmp_context: WorkContext + ) -> None: + """Resolve.background_work(bt) returns a non-None callable.""" + bt = bootstrapper.Bootstrapper(tmp_context) + item = _make_resolve_item() + bg = item.background_work(bt) + assert bg is not None + assert callable(bg) + + def test_prepare_source_item_background_work_source( + self, tmp_context: WorkContext + ) -> None: + """When pbi_pre_built=False, background_work() returns a source callable.""" + bt = bootstrapper.Bootstrapper(tmp_context) + wi = _make_work_item( + req="testpkg", + version="1.0", + source_url="https://pypi.test/testpkg-1.0.tar.gz", + ) + wi.pbi_pre_built = False + item = PrepareSource(wi) + bg = item.background_work(bt) + assert bg is not None + assert callable(bg) + + def test_prepare_source_item_background_work_prebuilt( + self, tmp_context: WorkContext + ) -> None: + """When pbi_pre_built=True, background_work() returns a prebuilt callable.""" + bt = bootstrapper.Bootstrapper(tmp_context) + wi = _make_work_item( + req="testpkg", + version="1.0", + source_url="https://pypi.test/testpkg-1.0-py3-none-any.whl", + ) + wi.pbi_pre_built = True + item = PrepareSource(wi) + bg = item.background_work(bt) + assert bg is not None + assert callable(bg) + + +class TestAsJson: + """Tests for Phase.as_json() serialization.""" + + @pytest.mark.parametrize( + "cls, expected_phase", + [ + (Resolve, BootstrapPhase.RESOLVE), + (Start, BootstrapPhase.START), + (PrepareSource, BootstrapPhase.PREPARE_SOURCE), + (PrepareBuild, BootstrapPhase.PREPARE_BUILD), + (Build, BootstrapPhase.BUILD), + (ProcessInstallDeps, BootstrapPhase.PROCESS_INSTALL_DEPS), + (Complete, BootstrapPhase.COMPLETE), + ], + ) + def test_phase_field_per_subclass( + self, cls: type[Phase], expected_phase: BootstrapPhase + ) -> None: + wi = _make_work_item(req="testpkg") + item = cls(wi) + data = item.as_json() + assert data["phase"] == str(expected_phase) class TestThreadPoolSubmission: """Tests that exercise the real ThreadPoolExecutor submission path.""" - def test_push_items_submits_resolve_to_real_thread_pool( + def test_push_items_sets_bg_future_when_pool_exists( self, tmp_context: WorkContext ) -> None: - """_push_items submits RESOLVE work to a real thread pool and result() returns correctly.""" + """_push_items submits RESOLVE work to a real thread pool.""" expected = [("https://pkg.test/testpkg-1.0.tar.gz", Version("1.0"))] with bootstrapper.Bootstrapper(tmp_context, num_bg_threads=1) as bt: with patch.object(bt._resolver, "resolve", return_value=expected): item = _make_resolve_item(req="testpkg") - stack: list[WorkItem] = [] + stack: list[Phase] = [] bt._push_items(stack, [item]) - # bg_future must be a real Future, not pre-set + # bg_future must be set on the Phase, not None assert item.bg_future is not None assert isinstance(item.bg_future, concurrent.futures.Future) @@ -2182,9 +2425,37 @@ def test_push_items_submits_resolve_to_real_thread_pool( result = item.bg_future.result(timeout=10) assert result == expected - # _phase_resolve processes the future correctly - new_items = bt._phase_resolve(item) + # Resolve.run() processes the future correctly + new_items = item.run(bt) assert len(new_items) == 1 - assert new_items[0].phase == BootstrapPhase.START - assert new_items[0].resolved_version == Version("1.0") + assert isinstance(new_items[0], Start) + assert new_items[0].work_item.resolved_version == Version("1.0") + + def test_push_items_no_bg_future_when_pool_is_none( + self, tmp_context: WorkContext + ) -> None: + """When _bg_pool is None, calling _push_items leaves item.bg_future as None.""" + bt = bootstrapper.Bootstrapper(tmp_context) + bt._bg_pool = None + item = _make_resolve_item(req="testpkg") + stack: list[Phase] = [] + + bt._push_items(stack, [item]) + + assert item.bg_future is None + + +class TestCreateUnresolvedWorkItemsReturnType: + """create_unresolved_work_items returns Resolve instances.""" + + def test_returns_resolve_items(self, tmp_context: WorkContext) -> None: + bt = bootstrapper.Bootstrapper(tmp_context) + deps = [Requirement("dep-a")] + items = bt.create_unresolved_work_items( + deps, RequirementType.INSTALL, Requirement("parent"), Version("1.0") + ) + assert len(items) == 1 + assert isinstance(items[0], Resolve) + assert items[0].work_item.req == Requirement("dep-a") + assert items[0].work_item.req_type == RequirementType.INSTALL