refactor(bootstrapper): decompose monolithic bootstrapper into a subpackage

src/fromager/bootstrapper/__init__.py

@@ -0,0 +1,4 @@
+from ._bootstrapper import Bootstrapper
+from ._types import _DEFAULT_BG_THREADS
+
+__all__ = ["_DEFAULT_BG_THREADS", "Bootstrapper"]

src/fromager/bootstrapper/_build.py

@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+import logging
+import pathlib
+import typing
+
+from .. import finders, server, sources, wheels
+from ._phase import Phase
+from ._process_install_deps import ProcessInstallDeps
+from ._types import BootstrapPhase, SourceBuildResult
+
+if typing.TYPE_CHECKING:
+    from .. import context
+    from ._bootstrapper import Bootstrapper
+
+logger = logging.getLogger(__name__)
+
+
+class Build(Phase):
+    """Install remaining build deps and produce a wheel or sdist for the package.
+
+    Three execution paths based on ``WorkItem`` state:
+
+    - **Cached wheel**: a matching wheel was found during source preparation;
+      no build is performed.
+    - **sdist-only**: ``sdist_only`` mode is active and this is not a build
+      requirement; only an sdist is built.
+    - **Full build**: an sdist is built first, then a wheel is compiled and
+      the local mirror is updated.
+
+    When ``pbi.exclusive_build`` is set the bootstrap loop drains the
+    background thread pool before calling ``run()`` so the build has exclusive
+    access to the environment.
+
+    Next phase: ``ProcessInstallDeps``.
+    """
+
+    phase: typing.ClassVar[BootstrapPhase] = BootstrapPhase.BUILD
+    tracks_why: typing.ClassVar[bool] = True
+
+    @property
+    def requires_exclusive_run(self) -> bool:
+        return self.work_item.exclusive_build
+
+    def run(self, bt: Bootstrapper) -> list[Phase]:
+        """BUILD phase: install remaining deps, build wheel/sdist.
+
+        Returns:
+            [ProcessInstallDeps].
+        """
+        wi = self.work_item
+        assert wi.resolved_version is not None
+        assert wi.build_env is not None
+        assert wi.sdist_root_dir is not None
+
+        # Install backend+sdist deps if disjoint from system deps
+        remaining_deps = wi.build_backend_deps | wi.build_sdist_deps
+        if remaining_deps.isdisjoint(wi.build_system_deps):
+            wi.build_env.install(remaining_deps)
+
+        wheel_filename, sdist_filename = self.do_build(bt.ctx, bt.explain)
+
+        source_type = sources.get_source_type(bt.ctx, wi.req)
+
+        wi.build_result = SourceBuildResult(
+            wheel_filename=wheel_filename,
+            sdist_filename=sdist_filename,
+            unpack_dir=wi.sdist_root_dir.parent,
+            sdist_root_dir=wi.sdist_root_dir,
+            build_env=wi.build_env,
+            source_type=source_type,
+        )
+
+        return [ProcessInstallDeps(wi)]
+
+    def _build_sdist(self, ctx: context.WorkContext) -> pathlib.Path:
+        """Build or locate an sdist for this item's package.
+
+        Checks ``ctx.sdists_builds`` for an existing sdist before invoking
+        ``sources.build_sdist()``.
+        """
+        wi = self.work_item
+        assert wi.resolved_version is not None
+        assert wi.sdist_root_dir is not None
+        assert wi.build_env is not None
+        find_sdist_result = finders.find_sdist(
+            ctx, ctx.sdists_builds, wi.req, str(wi.resolved_version)
+        )
+        if not find_sdist_result:
+            sdist_filename: pathlib.Path = sources.build_sdist(
+                ctx=ctx,
+                req=wi.req,
+                version=wi.resolved_version,
+                sdist_root_dir=wi.sdist_root_dir,
+                build_env=wi.build_env,
+            )
+        else:
+            sdist_filename = find_sdist_result
+            logger.info(
+                f"have sdist version {wi.resolved_version}: {find_sdist_result}"
+            )
+        return sdist_filename
+
+    def _build_wheel(
+        self,
+        ctx: context.WorkContext,
+        explain: str = "",
+    ) -> tuple[pathlib.Path, pathlib.Path]:
+        """Build a wheel for this item's package.
+
+        Calls :meth:`_build_sdist` first, then invokes ``wheels.build_wheel()``
+        and updates the local wheel mirror. Returns ``(wheel_filename,
+        sdist_filename)`` where *wheel_filename* is the path in
+        ``ctx.wheels_downloads`` after the mirror update.
+        """
+        wi = self.work_item
+        assert wi.resolved_version is not None
+        assert wi.sdist_root_dir is not None
+        assert wi.build_env is not None
+        sdist_filename = self._build_sdist(ctx)
+
+        logger.info(f"starting build of {explain} for {ctx.variant}")
+        built_filename = wheels.build_wheel(
+            ctx=ctx,
+            req=wi.req,
+            sdist_root_dir=wi.sdist_root_dir,
+            version=wi.resolved_version,
+            build_env=wi.build_env,
+        )
+        server.update_wheel_mirror(ctx)
+        # When we update the mirror, the built file moves to the downloads directory.
+        wheel_filename = ctx.wheels_downloads / built_filename.name
+        logger.info(f"built wheel for version {wi.resolved_version}: {wheel_filename}")
+        return wheel_filename, sdist_filename
+
+    def do_build(
+        self,
+        ctx: context.WorkContext,
+        explain: str = "",
+    ) -> tuple[pathlib.Path | None, pathlib.Path | None]:
+        """Build a wheel or sdist for this item's package.
+
+        Returns ``(wheel_filename, sdist_filename)``.  Either value may be
+        ``None`` depending on the build path:
+
+        - Cached wheel:  ``(cached_wheel_filename, None)``
+        - sdist-only:    ``(None, sdist_filename)``
+        - Full wheel:    ``(wheel_filename, sdist_filename)``
+        """
+        wi = self.work_item
+        if wi.cached_wheel_filename:
+            logger.debug(
+                f"getting install requirements from cached wheel {wi.cached_wheel_filename.name}"
+            )
+            return wi.cached_wheel_filename, None
+        elif wi.build_sdist_only:
+            logger.debug(
+                f"getting install requirements from sdist {wi.req.name}=={wi.resolved_version}"
+            )
+            return None, self._build_sdist(ctx)
+        else:
+            logger.debug(
+                f"building wheel {wi.req.name}=={wi.resolved_version} to get install requirements"
+            )
+            return self._build_wheel(ctx, explain)

src/fromager/bootstrapper/_cache.py

@@ -0,0 +1,226 @@
+from __future__ import annotations
+
+import logging
+import os
+import pathlib
+import typing
+import zipfile
+from urllib.parse import urlparse
+
+import requests.exceptions
+from packaging.requirements import Requirement
+from packaging.version import Version
+from resolvelib.resolvers import ResolverException
+
+from .. import dependencies, finders, resolver, server, wheels
+from ..requirements_file import RequirementType
+from ._types import PreparedSourceData
+
+if typing.TYPE_CHECKING:
+    from .. import context
+
+logger = logging.getLogger(__name__)
+
+
+def _create_unpack_dir(
+    work_dir: pathlib.Path,
+    req: Requirement,
+    resolved_version: Version,
+) -> pathlib.Path:
+    unpack_dir = work_dir / f"{req.name}-{resolved_version}"
+    unpack_dir.mkdir(parents=True, exist_ok=True)
+    return unpack_dir
+
+
+def _extract_build_reqs_from_wheel(
+    work_dir: pathlib.Path,
+    req: Requirement,
+    resolved_version: Version,
+    wheel_filename: pathlib.Path,
+) -> pathlib.Path | None:
+    """Extract fromager build requirement files from a wheel archive.
+
+    Looks for files prefixed with `FROMAGER_BUILD_REQ_PREFIX` inside the
+    wheel's `.dist-info` directory and extracts them to a local unpack
+    directory. Returns the unpack directory on success, or None if the
+    files are not present or extraction fails.
+    """
+    dist_name, dist_version, _, _ = wheels.extract_info_from_wheel_file(
+        req, wheel_filename
+    )
+    unpack_dir = _create_unpack_dir(work_dir, req, resolved_version)
+    dist_filename = f"{dist_name}-{dist_version}"
+    dist_info_path = pathlib.Path(f"{dist_filename}.dist-info")
+    req_filenames: list[str] = [
+        dependencies.BUILD_BACKEND_REQ_FILE_NAME,
+        dependencies.BUILD_SDIST_REQ_FILE_NAME,
+        dependencies.BUILD_SYSTEM_REQ_FILE_NAME,
+    ]
+    try:
+        with zipfile.ZipFile(wheel_filename) as archive:
+            for filename in req_filenames:
+                zipinfo = archive.getinfo(
+                    str(
+                        dist_info_path
+                        / f"{wheels.FROMAGER_BUILD_REQ_PREFIX}-{filename}"
+                    )
+                )
+                if os.path.isabs(zipinfo.filename) or ".." in zipinfo.filename:
+                    raise ValueError(f"Unsafe path in wheel: {zipinfo.filename}")
+                zipinfo.filename = filename
+                output_file = archive.extract(zipinfo, unpack_dir)
+                logger.info(f"extracted {output_file}")
+        logger.info(f"extracted build requirements from wheel into {unpack_dir}")
+        return unpack_dir
+    except Exception as e:
+        logger.info(f"could not extract build requirements from wheel: {e}")
+        for filename in req_filenames:
+            unpack_dir.joinpath(filename).unlink(missing_ok=True)
+        return None
+
+
+def _look_for_existing_wheel(
+    ctx: context.WorkContext,
+    req: Requirement,
+    resolved_version: Version,
+    search_in: pathlib.Path,
+) -> tuple[pathlib.Path | None, pathlib.Path | None]:
+    pbi = ctx.package_build_info(req)
+    expected_build_tag = pbi.build_tag(resolved_version)
+    logger.info(
+        f"looking for existing wheel for version {resolved_version} with build tag {expected_build_tag} in {search_in}"
+    )
+    wheel_filename = finders.find_wheel(
+        downloads_dir=search_in,
+        req=req,
+        dist_version=str(resolved_version),
+        build_tag=expected_build_tag,
+    )
+    if not wheel_filename:
+        return None, None
+    _, _, build_tag, _ = wheels.extract_info_from_wheel_file(req, wheel_filename)
+    if expected_build_tag and expected_build_tag != build_tag:
+        logger.info(
+            f"found wheel for {resolved_version} in {wheel_filename} but build tag does not match. Got {build_tag} but expected {expected_build_tag}"
+        )
+        return None, None
+    logger.info(f"found existing wheel {wheel_filename}")
+    build_reqs_dir = _extract_build_reqs_from_wheel(
+        ctx.work_dir, req, resolved_version, wheel_filename
+    )
+    return wheel_filename, build_reqs_dir
+
+
+def _download_wheel_from_cache(
+    ctx: context.WorkContext,
+    cache_wheel_server_url: str | None,
+    req: Requirement,
+    resolved_version: Version,
+) -> tuple[pathlib.Path | None, pathlib.Path | None]:
+    if not cache_wheel_server_url:
+        return None, None
+    logger.info(f"checking if wheel was already uploaded to {cache_wheel_server_url}")
+    try:
+        pinned_req = Requirement(f"{req.name}=={resolved_version}")
+        provider = finders.PyPICacheProvider(
+            cache_server_url=cache_wheel_server_url,
+            constraints=ctx.constraints,
+        )
+        results = resolver.find_all_matching_from_provider(provider, pinned_req)
+        wheel_url, _ = results[0]
+        wheelfile_name = pathlib.Path(urlparse(wheel_url).path)
+        pbi = ctx.package_build_info(req)
+        expected_build_tag = pbi.build_tag(resolved_version)
+        logger.info(f"has expected build tag {expected_build_tag}")
+        changelogs = pbi.get_changelog(resolved_version)
+        logger.debug(f"has change logs {changelogs}")
+
+        _, _, build_tag, _ = wheels.extract_info_from_wheel_file(req, wheelfile_name)
+        if expected_build_tag and expected_build_tag != build_tag:
+            logger.info(
+                f"found wheel for {resolved_version} in cache but build tag does not match. Got {build_tag} but expected {expected_build_tag}"
+            )
+            return None, None
+
+        cached_wheel = wheels.download_wheel(
+            req=req, wheel_url=wheel_url, output_directory=ctx.wheels_downloads
+        )
+        if cache_wheel_server_url != ctx.wheel_server_url:
+            server.update_wheel_mirror(ctx)
+        logger.info("found built wheel on cache server")
+        unpack_dir = _extract_build_reqs_from_wheel(
+            ctx.work_dir, req, resolved_version, cached_wheel
+        )
+        return cached_wheel, unpack_dir
+    except ResolverException:
+        logger.info(
+            f"did not find wheel for {resolved_version} in {cache_wheel_server_url}"
+        )
+        return None, None
+    except requests.exceptions.RequestException as err:
+        logger.warning(
+            f"network error checking wheel cache for {resolved_version} "
+            f"at {cache_wheel_server_url}: {err}"
+        )
+        return None, None
+    except Exception as err:
+        logger.warning(
+            f"unexpected error checking wheel cache for {resolved_version} "
+            f"at {cache_wheel_server_url}: {err}"
+        )
+        return None, None
+
+
+def _find_cached_wheel(
+    ctx: context.WorkContext,
+    cache_wheel_server_url: str | None,
+    req: Requirement,
+    resolved_version: Version,
+) -> tuple[pathlib.Path | None, pathlib.Path | None]:
+    """Look for cached wheel in 3 locations (thread-safe, no Bootstrapper state).
+
+    Checks for cached wheels in order:
+    1. wheels_build directory (previously built)
+    2. wheels_downloads directory (previously downloaded)
+    3. Cache server (remote cache)
+
+    Returns:
+        Tuple of (cached_wheel_filename, unpacked_cached_wheel).
+        Both None if no cache hit.
+    """
+    cached_wheel, unpacked = _look_for_existing_wheel(
+        ctx, req, resolved_version, ctx.wheels_build
+    )
+    if cached_wheel:
+        return cached_wheel, unpacked
+
+    cached_wheel, unpacked = _look_for_existing_wheel(
+        ctx, req, resolved_version, ctx.wheels_downloads
+    )
+    if cached_wheel:
+        return cached_wheel, unpacked
+
+    cached_wheel, unpacked = _download_wheel_from_cache(
+        ctx, cache_wheel_server_url, req, resolved_version
+    )
+    if cached_wheel:
+        return cached_wheel, unpacked
+
+    return None, None
+
+
+def _bg_prepare_prebuilt(
+    ctx: context.WorkContext,
+    req: Requirement,
+    req_type: RequirementType,
+    resolved_version: Version,
+    wheel_url: str,
+) -> PreparedSourceData:
+    """Background-safe prebuilt download: no Bootstrapper state accessed."""
+    # Thread-safe: paths include {req.name}-{resolved_version} (unique per package),
+    # mkdir uses exist_ok=True (atomic), and update_wheel_mirror() is already locked.
+    logger.info(f"using pre-built wheel for {req_type} requirement")
+    wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt)
+    unpack_dir = _create_unpack_dir(ctx.work_dir, req, resolved_version)
+    server.update_wheel_mirror(ctx)
+    return PreparedSourceData(wheel_filename=wheel_filename, unpack_dir=unpack_dir)