diff --git a/docker/containerized/README.md b/docker/containerized/README.md index cb5d19189..7f35c9f99 100644 --- a/docker/containerized/README.md +++ b/docker/containerized/README.md @@ -277,6 +277,24 @@ If a request targets a tag with no corresponding Git branch — for example a ti This lets IIB build and validate a one-off tag without requiring per-tag branch/Component provisioning. +### Chaining IIB Build Outputs + +Containerized `add`, `rm`, and `fbc-operations` requests may use the completed +`index_image` from an earlier request as `from_index`. IIB resolves request +ancestry to locate the original onboarded Git repository and its Konflux +scaffolding, but reconstructs the immediate parent's state from: + +- FBC configs embedded in the parent's resolved output image; and +- `index-db:idb--` in Quay. + +Chained requests are always throw-away. Supplying `overwrite_from_index` or +`overwrite_from_index_token` for an IIB output image fails the request because +users cannot overwrite IIB's output registry. The Konflux MR is always closed +and never merged. + +Chaining is not supported for `merge-index-image`, `create-empty-index`, +`regenerate-bundle`, or legacy worker requests. + ## Index DB Artifact and ImageStream Tag Naming Cached `index.db` artifact tags (ORAS) and ImageStream tags are keyed on the index image's content (manifest) digest — `idb-` — resolved via `skopeo inspect`, not on its pullspec. Because the key is the content itself, it is both namespace-safe and promotion-safe: diff --git a/iib/workers/tasks/build.py b/iib/workers/tasks/build.py index 7025d19ac..aca3e0ac9 100644 --- a/iib/workers/tasks/build.py +++ b/iib/workers/tasks/build.py @@ -260,7 +260,7 @@ def _update_index_image_pull_spec( is_image_fbc: bool = False, index_repo_map: Optional[Dict[str, str]] = None, rm_operators: Optional[List[str]] = None, -) -> None: +) -> Optional[str]: """ Update the request with the modified index image. @@ -283,6 +283,8 @@ def _update_index_image_pull_spec( required if ``is_image_fbc`` is ``True``. :param list(str) rm_operators: List of operator package names to remove from the Git catalog during overwrite. Used by RM requests and ADD requests with deprecations. + :return: The recorded index_image_resolved when add_or_rm is True, otherwise None. + :rtype: Optional[str] :raises IIBError: if the manifest list couldn't be created and pushed """ conf = get_worker_config() @@ -322,6 +324,7 @@ def _update_index_image_pull_spec( payload['internal_index_image_copy_resolved'] = get_resolved_image(output_pull_spec) update_request(request_id, payload, exc_msg='Failed setting the index image on the request') + return payload.get('index_image_resolved') def _get_external_arch_pull_spec( diff --git a/iib/workers/tasks/build_containerized_add.py b/iib/workers/tasks/build_containerized_add.py index 6075c55be..0bdf75334 100644 --- a/iib/workers/tasks/build_containerized_add.py +++ b/iib/workers/tasks/build_containerized_add.py @@ -180,6 +180,7 @@ def handle_containerized_add_request( ocp_version=prebuild_info['ocp_version'], index_to_gitlab_push_map=index_to_gitlab_push_map, overwrite_from_index=overwrite_from_index, + overwrite_from_index_token=overwrite_from_index_token, ) index_git_repo = sources.index_git_repo local_git_repo_path = sources.local_git_repo_path @@ -335,7 +336,7 @@ def handle_containerized_add_request( "This should not happen if the pipeline completed successfully." ) - _update_index_image_pull_spec( + index_image_resolved = _update_index_image_pull_spec( output_pull_spec=output_pull_spec, request_id=request_id, arches=arches, @@ -351,6 +352,8 @@ def handle_containerized_add_request( # a Konflux pipelinerun. So the old workflow isn't needed. index_repo_map={}, ) + if index_image_resolved is None: + raise IIBError(f'request {request_id} has no resolved index image.') # Push updated index.db before merging the MR so that on failure both # git and the index.db artifact remain consistent (MR stays open, @@ -360,7 +363,7 @@ def handle_containerized_add_request( from_index=str(from_index), index_db_path=artifact_index_db_file, operators=operators, - output_image=image_url, + output_image=index_image_resolved, overwrite_from_index=overwrite_from_index, request_type='add', ) @@ -369,17 +372,14 @@ def handle_containerized_add_request( # (replication, metadata, index.db push) have succeeded before git # is advanced. This prevents git/index.db divergence on partial failure. # - # The `not sources.is_divergent` half is defense-in-depth: prepare_build_sources - # already rejects overwrite_from_index on the divergent path, so this cannot be - # reached with both set. It stays because a divergent build reuses the base OCP - # branch's Konflux Component -- merging its MR would publish one tag's content - # onto the shared branch. - if overwrite_from_index and not sources.is_divergent: + # Only standard sources may publish their MR. Divergent and chained builds + # reuse shared Git/Konflux scaffolding and must remain throw-away. + if overwrite_from_index and sources.merge_allowed: merge_mr_after_build(mr_details, index_git_repo) # Prevent cleanup_on_failure from trying to close an already-merged MR mr_details = None else: - cleanup_merge_request_if_exists(mr_details, index_git_repo) + cleanup_merge_request_if_exists(mr_details, index_git_repo, raise_on_error=True) set_request_state( request_id, diff --git a/iib/workers/tasks/build_containerized_fbc_operations.py b/iib/workers/tasks/build_containerized_fbc_operations.py index 0979923dd..f7bc047d2 100644 --- a/iib/workers/tasks/build_containerized_fbc_operations.py +++ b/iib/workers/tasks/build_containerized_fbc_operations.py @@ -148,6 +148,7 @@ def handle_containerized_fbc_operation_request( ocp_version=prebuild_info['ocp_version'], index_to_gitlab_push_map=index_to_gitlab_push_map, overwrite_from_index=overwrite_from_index, + overwrite_from_index_token=overwrite_from_index_token, ) index_git_repo = sources.index_git_repo local_git_repo_path = sources.local_git_repo_path @@ -228,7 +229,7 @@ def handle_containerized_fbc_operation_request( "This should not happen if the pipeline completed successfully." ) - _update_index_image_pull_spec( + index_image_resolved = _update_index_image_pull_spec( output_pull_spec=output_pull_spec, request_id=request_id, arches=arches, @@ -244,6 +245,8 @@ def handle_containerized_fbc_operation_request( # a Konflux pipelinerun. So the old workflow isn't needed. index_repo_map={}, ) + if index_image_resolved is None: + raise IIBError(f'request {request_id} has no resolved index image.') # Push updated index.db before merging the MR so that on failure both # git and the index.db artifact remain consistent (MR stays open, @@ -253,7 +256,7 @@ def handle_containerized_fbc_operation_request( from_index=from_index, index_db_path=index_db_path, operators=operators_in_db, - output_image=image_url, + output_image=index_image_resolved, overwrite_from_index=overwrite_from_index, request_type='fbc_operations', ) @@ -262,17 +265,14 @@ def handle_containerized_fbc_operation_request( # (replication, metadata, index.db push) have succeeded before git # is advanced. This prevents git/index.db divergence on partial failure. # - # The `not sources.is_divergent` half is defense-in-depth: prepare_build_sources - # already rejects overwrite_from_index on the divergent path, so this cannot be - # reached with both set. It stays because a divergent build reuses the base OCP - # branch's Konflux Component -- merging its MR would publish one tag's content - # onto the shared branch. - if overwrite_from_index and not sources.is_divergent: + # Only standard sources may publish their MR. Divergent and chained builds + # reuse shared Git/Konflux scaffolding and must remain throw-away. + if overwrite_from_index and sources.merge_allowed: merge_mr_after_build(mr_details, index_git_repo) # Prevent cleanup_on_failure from trying to close an already-merged MR mr_details = None else: - cleanup_merge_request_if_exists(mr_details, index_git_repo) + cleanup_merge_request_if_exists(mr_details, index_git_repo, raise_on_error=True) # Summarize every action the request took. The per-step messages (extracting # fragments, removing operators from index.db, adding packages) are reported diff --git a/iib/workers/tasks/build_containerized_rm.py b/iib/workers/tasks/build_containerized_rm.py index 0acd2d526..01f754c66 100644 --- a/iib/workers/tasks/build_containerized_rm.py +++ b/iib/workers/tasks/build_containerized_rm.py @@ -144,6 +144,7 @@ def handle_containerized_rm_request( ocp_version=ocp_version, index_to_gitlab_push_map=index_to_gitlab_push_map or {}, overwrite_from_index=overwrite_from_index, + overwrite_from_index_token=overwrite_from_index_token, ) index_git_repo = sources.index_git_repo local_git_repo_path = sources.local_git_repo_path @@ -272,7 +273,7 @@ def handle_containerized_rm_request( # Send an empty index_repo_map because the Git repository is already # updated with the changes - _update_index_image_pull_spec( + index_image_resolved = _update_index_image_pull_spec( output_pull_spec=output_pull_spec, request_id=request_id, arches=arches, @@ -289,6 +290,8 @@ def handle_containerized_rm_request( index_repo_map={}, rm_operators=operators, ) + if index_image_resolved is None: + raise IIBError(f'request {request_id} has no resolved index image.') # Push updated index.db before merging the MR so that on failure both # git and the index.db artifact remain consistent (MR stays open, @@ -298,7 +301,7 @@ def handle_containerized_rm_request( from_index=from_index, index_db_path=index_db_path, operators=operators, - output_image=image_url, + output_image=index_image_resolved, overwrite_from_index=overwrite_from_index, request_type='rm', ) @@ -307,17 +310,14 @@ def handle_containerized_rm_request( # (replication, metadata, index.db push) have succeeded before git # is advanced. This prevents git/index.db divergence on partial failure. # - # The `not sources.is_divergent` half is defense-in-depth: prepare_build_sources - # already rejects overwrite_from_index on the divergent path, so this cannot be - # reached with both set. It stays because a divergent build reuses the base OCP - # branch's Konflux Component -- merging its MR would publish one tag's content - # onto the shared branch. - if overwrite_from_index and not sources.is_divergent: + # Only standard sources may publish their MR. Divergent and chained builds + # reuse shared Git/Konflux scaffolding and must remain throw-away. + if overwrite_from_index and sources.merge_allowed: merge_mr_after_build(mr_details, index_git_repo) # Prevent cleanup_on_failure from trying to close an already-merged MR mr_details = None else: - cleanup_merge_request_if_exists(mr_details, index_git_repo) + cleanup_merge_request_if_exists(mr_details, index_git_repo, raise_on_error=True) operators_str = ', '.join(operators) set_request_state( diff --git a/iib/workers/tasks/containerized_utils.py b/iib/workers/tasks/containerized_utils.py index 24ce97b58..94a591e02 100644 --- a/iib/workers/tasks/containerized_utils.py +++ b/iib/workers/tasks/containerized_utils.py @@ -5,15 +5,18 @@ import os import posixpath import queue +import re import shutil import tempfile import threading +from contextlib import nullcontext from dataclasses import dataclass +from enum import Enum from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Set, Tuple, Union from iib.exceptions import ArtifactNotFoundError, IIBError, FileNotFoundInImageError -from iib.workers.api_utils import set_request_state +from iib.workers.api_utils import get_request, set_request_state from iib.workers.config import get_worker_config from iib.workers.tasks.iib_static_types import BundleImage from iib.workers.tasks.build import _skopeo_copy @@ -34,11 +37,11 @@ wait_for_pipeline_completion, ) from iib.workers.tasks.oras_utils import ( - _get_index_digest, get_index_tag, get_indexdb_artifact_pullspec, get_imagestream_artifact_pullspec, get_oras_artifact, + get_request_indexdb_artifact_pullspec, push_oras_artifact, refresh_indexdb_cache_for_image, verify_indexdb_cache_for_image, @@ -48,6 +51,106 @@ log = logging.getLogger(__name__) +_REQUEST_ID_SENTINEL = '__IIB_REQUEST_ID__' +_CHAINED_REQUEST_TYPES = frozenset({'add', 'rm', 'fbc-operations'}) + + +@dataclass(frozen=True) +class ChainedBuildSource: + """Immutable ancestry and immediate-parent data for a chained build.""" + + parent_request_id: int + parent_index_image: str + parent_index_image_resolved: str + original_from_index: str + ancestry: Tuple[int, ...] + + +def get_iib_output_request_id(image: str) -> Optional[int]: + """Return the request ID when an image exactly matches an IIB output template.""" + conf = get_worker_config() + registries = {conf['iib_registry']} + if conf.get('iib_index_image_output_registry'): + registries.add(conf['iib_index_image_output_registry']) + + for registry in registries: + rendered = conf['iib_image_push_template'].format( + registry=registry, + request_id=_REQUEST_ID_SENTINEL, + ) + pattern = re.escape(rendered).replace( + re.escape(_REQUEST_ID_SENTINEL), + r'(?P[0-9]+)', + ) + match = re.fullmatch(pattern, image) + if match: + return int(match.group('request_id')) + return None + + +def resolve_chained_build_source( + from_index: str, + overwrite_from_index: bool, + overwrite_from_index_token: Optional[str], +) -> Optional[ChainedBuildSource]: + """Resolve immutable ancestry when ``from_index`` is an IIB build output.""" + parent_id = get_iib_output_request_id(from_index) + if parent_id is None: + return None + if overwrite_from_index or overwrite_from_index_token: + raise IIBError( + f'Chained from_index {from_index} cannot be overwritten; ' + 'IIB output images are read-only build results.' + ) + + current_image = from_index + current_id: Optional[int] = parent_id + seen: Set[int] = set() + ancestry: List[int] = [] + immediate_parent: Optional[Dict[str, Any]] = None + + while current_id is not None: + if current_id in seen: + raise IIBError(f'Build chain contains a cycle at request {current_id}: {ancestry}') + seen.add(current_id) + ancestry.append(current_id) + request_data = get_request(current_id) + + if request_data.get('state') != 'complete': + raise IIBError(f'Parent request {current_id} is not complete.') + if request_data.get('request_type') not in _CHAINED_REQUEST_TYPES: + raise IIBError( + f'Parent request {current_id} has unsupported request type ' + f"{request_data.get('request_type')!r}." + ) + if request_data.get('index_image') != current_image: + raise IIBError( + f'Parent request {current_id} index_image ' + f"{request_data.get('index_image')!r} does not match " + f'{current_image!r}.' + ) + if not request_data.get('index_image_resolved'): + raise IIBError(f'Parent request {current_id} has no resolved index image.') + if immediate_parent is None: + immediate_parent = request_data + + next_image = request_data.get('from_index') + if not next_image: + raise IIBError(f'Parent request {current_id} has no from_index.') + current_image = next_image + current_id = get_iib_output_request_id(current_image) + + if immediate_parent is None: + raise IIBError(f'Failed to resolve parent request {parent_id}.') + return ChainedBuildSource( + parent_request_id=parent_id, + parent_index_image=immediate_parent['index_image'], + parent_index_image_resolved=immediate_parent['index_image_resolved'], + original_from_index=current_image, + ancestry=tuple(ancestry), + ) + + def extract_files_from_image_non_privileged(image: str, src_path: str, dest_path: str) -> None: """ Extract a file or directory from a container image, unprivileged. @@ -214,37 +317,19 @@ def _reject_escaping_symlinks(root: str, image: str, description: str) -> None: ) -def extract_catalog_and_db_from_image(from_index_resolved: str, temp_dir: str) -> Tuple[str, str]: - """ - Extract FBC configs and index.db from an index image, unprivileged. - - Used on the divergent-tag path where no git branch / ORAS artifact exists yet. - The image is the source of truth for its own content. +def extract_catalog_from_image(from_index_resolved: str, temp_dir: str) -> str: + """Extract FBC configs from an index image, unprivileged. The caller must pass the digest-resolved pullspec (``from_index_resolved``), not the mutable tag, so the extracted content matches the image the request - already inspected during prebuild (OPM version, build metadata) and so the - repeated image reads here cannot disagree with each other. - - Only the FBC configs and the hidden index.db are extracted. The hidden db is - the sole source of truth for the SQLite index; there is no fallback to the - labeled database path and no synthesised empty db. An image that carries no - hidden index.db has not been onboarded to the containerized build flow and - the request is failed so the image can be onboarded first. - - The configs label is the signal that the image declares an FBC root, so a - declared-but-empty configs directory (an empty index, whose ``/configs`` holds - no files) is treated as an empty catalog rather than a missing path — the - divergent add/rm operation then populates it. + already inspected during prebuild. :param str from_index_resolved: The digest-resolved from_index image pullspec. :param str temp_dir: Base temp directory for extraction. - :return: Tuple of (configs_dir_path, index_db_path). - :rtype: Tuple[str, str] - :raises IIBError: If the image has no FBC configs label, if its configs tree - contains a symlink pointing outside the extracted content, if it carries - no hidden index.db, or if the hidden-db extraction fails for any other - reason (e.g. a registry, OCI parsing, layer, or tar error). + :return: Path to the extracted configs directory. + :rtype: str + :raises IIBError: If the image has no FBC configs label or if its configs tree + contains a symlink pointing outside the extracted content. """ configs_label = get_image_label( from_index_resolved, 'operators.operatorframework.io.index.configs.v1' @@ -277,6 +362,42 @@ def extract_catalog_and_db_from_image(from_index_resolved: str, temp_dir: str) - # The configs tree is committed to git verbatim, so refuse one that would # drag content in from outside the extraction root when it is copied. _reject_escaping_symlinks(configs_dir, from_index_resolved, 'FBC configs directory') + return configs_dir + + +def extract_catalog_and_db_from_image(from_index_resolved: str, temp_dir: str) -> Tuple[str, str]: + """ + Extract FBC configs and index.db from an index image, unprivileged. + + Used on the divergent-tag path where no git branch / ORAS artifact exists yet. + The image is the source of truth for its own content. + + The caller must pass the digest-resolved pullspec (``from_index_resolved``), + not the mutable tag, so the extracted content matches the image the request + already inspected during prebuild (OPM version, build metadata) and so the + repeated image reads here cannot disagree with each other. + + Only the FBC configs and the hidden index.db are extracted. The hidden db is + the sole source of truth for the SQLite index; there is no fallback to the + labeled database path and no synthesised empty db. An image that carries no + hidden index.db has not been onboarded to the containerized build flow and + the request is failed so the image can be onboarded first. + + The configs label is the signal that the image declares an FBC root, so a + declared-but-empty configs directory (an empty index, whose ``/configs`` holds + no files) is treated as an empty catalog rather than a missing path — the + divergent add/rm operation then populates it. + + :param str from_index_resolved: The digest-resolved from_index image pullspec. + :param str temp_dir: Base temp directory for extraction. + :return: Tuple of (configs_dir_path, index_db_path). + :rtype: Tuple[str, str] + :raises IIBError: If the image has no FBC configs label, if its configs tree + contains a symlink pointing outside the extracted content, if it carries + no hidden index.db, or if the hidden-db extraction fails for any other + reason (e.g. a registry, OCI parsing, layer, or tar error). + """ + configs_dir = extract_catalog_from_image(from_index_resolved, temp_dir) index_db_path = str(Path(temp_dir) / 'extracted_index.db') conf = get_worker_config() @@ -449,6 +570,42 @@ def pull_index_db_artifact(from_index: str, temp_dir: str) -> str: ) from e +def fetch_and_verify_request_index_db_artifact( + index_image_resolved: str, request_id: int, temp_dir: str +) -> str: + """ + Fetch a request-specific index.db artifact and verify its payload. + + :param str index_image_resolved: The producing request's resolved output image. + :param int request_id: The producing IIB request ID. + :param str temp_dir: Directory where the artifact should be downloaded. + :return: Path to the canonical or legacy-compatible index database payload. + :rtype: str + :raises IIBError: If the artifact cannot be fetched or has no supported payload. + """ + artifact_ref = get_request_indexdb_artifact_pullspec(index_image_resolved, request_id) + try: + artifact_dir = get_oras_artifact(artifact_ref, temp_dir) + except ArtifactNotFoundError as e: + raise IIBError( + f'Request-specific index.db artifact {artifact_ref} not found for ' + f'parent request {request_id}.' + ) from e + except IIBError as e: + raise IIBError( + f'Failed to pull request-specific index.db artifact {artifact_ref}: {e}' + ) from e + + index_db = Path(artifact_dir) / 'index.db' + if not index_db.is_file(): + # Divergent builds published this basename before payload normalization. + legacy_index_db = Path(artifact_dir) / 'extracted_index.db' + if not legacy_index_db.is_file(): + raise IIBError(f'Index.db file not found at {index_db}.') + return str(legacy_index_db) + return str(index_db) + + def write_build_metadata( local_repo_path: str, opm_version: str, @@ -548,29 +705,12 @@ def push_index_db_artifact( index_db_file = Path(index_db_path) index_db_dir = str(index_db_file.parent) - index_db_filename = index_db_file.name - log.info('Pushing from directory: %s, filename: %s', index_db_dir, index_db_filename) set_request_state(request_id, 'in_progress', 'Pushing updated index database') - conf = get_worker_config() - # Derive the tag through the same config template the read path uses - # (oras_utils._get_content_addressed_artifact_tag), so an operator override - # cannot leave pushed tags and looked-up tags disagreeing. - output_tag = conf['iib_index_db_artifact_tag_template'].format( - digest=_get_index_digest(output_image) - ) - - request_artifact_ref = conf['iib_index_db_artifact_template'].format( - registry=conf['iib_index_db_artifact_registry'], - tag=f'{output_tag}-{request_id}', - ) + request_artifact_ref = get_request_indexdb_artifact_pullspec(output_image, request_id) artifact_refs = [request_artifact_ref] if overwrite_from_index: - current_artifact_ref = conf['iib_index_db_artifact_template'].format( - registry=conf['iib_index_db_artifact_registry'], - tag=output_tag, - ) - artifact_refs.append(current_artifact_ref) + artifact_refs.append(get_indexdb_artifact_pullspec(output_image)) annotations = { 'request_id': str(request_id), @@ -581,14 +721,24 @@ def push_index_db_artifact( if operators: annotations['operators'] = ','.join(operators) - for artifact_ref in artifact_refs: - push_oras_artifact( - artifact_ref=artifact_ref, - local_path=index_db_filename, - cwd=index_db_dir, - annotations=annotations.copy(), - ) - log.info('Pushed %s to registry', artifact_ref) + # ORAS preserves local filenames in the payload. Stage noncanonical inputs + # without renaming or overwriting the source database used by the build. + with ( + nullcontext(index_db_dir) + if index_db_file.name == 'index.db' + else tempfile.TemporaryDirectory(prefix='iib-index-db-') + ) as artifact_dir: + if index_db_file.name != 'index.db': + shutil.copyfile(index_db_file, Path(artifact_dir) / 'index.db') + log.info('Pushing index.db from directory: %s', artifact_dir) + for artifact_ref in artifact_refs: + push_oras_artifact( + artifact_ref=artifact_ref, + local_path='index.db', + cwd=artifact_dir, + annotations=annotations.copy(), + ) + log.info('Pushed %s to registry', artifact_ref) def cleanup_on_failure( @@ -644,20 +794,29 @@ def cleanup_on_failure( log.error("Neither MR nor commit to revert. No cleanup needed for %s", reason) +class BuildSourceKind(str, Enum): + """Describe how build content and Git/Konflux scaffolding are sourced.""" + + STANDARD = 'standard' + DIVERGENT = 'divergent' + CHAINED = 'chained' + + @dataclass(frozen=True) class BuildSources: - """Resolved inputs for a containerized build. - - Frozen so the ``is_divergent`` guard -- which decides whether an MR may be - merged -- cannot be flipped after ``prepare_build_sources`` has resolved it. - """ + """Resolved inputs and merge policy for a containerized build.""" index_git_repo: str local_git_repo_path: str localized_git_catalog_path: str index_db_path: Optional[str] # None => pull from ORAS; set => use extracted db target_branch: str - is_divergent: bool + source_kind: BuildSourceKind + + @property + def merge_allowed(self) -> bool: + """Return whether a build may merge its generated merge request.""" + return self.source_kind is BuildSourceKind.STANDARD def prepare_build_sources( @@ -668,17 +827,20 @@ def prepare_build_sources( ocp_version: str, index_to_gitlab_push_map: Dict[str, str], overwrite_from_index: bool, + overwrite_from_index_token: Optional[str] = None, ) -> BuildSources: """ - Resolve git repo + branch and decide the normal vs divergent build path. + Resolve Git/Konflux scaffolding and content for a containerized build. - Normal path: a branch named after the image tag exists -> build against it + Standard path: a branch named after the image tag exists -> build against it (overwrite allowed). Divergent path: no branch for the tag -> reject overwrite, reuse the base OCP branch's Konflux Component, and seed content - by extracting configs+index.db from the image. + by extracting configs+index.db from the image. Chained path: select Git + scaffolding from the original ancestor and seed configs+index.db from the + immediate parent's immutable outputs. - Branch selection keys on the mutable tag (``from_index``), but divergent - content is extracted from the digest-resolved pullspec + Standard/divergent branch selection keys on the mutable tag + (``from_index``), but divergent content is extracted from the digest-resolved pullspec (``from_index_resolved``) so it matches the image the request already inspected during prebuild and cannot drift if the tag moves mid-request. @@ -689,20 +851,72 @@ def prepare_build_sources( :param str ocp_version: Base OCP version branch, e.g. "v4.19" :param Dict[str, str] index_to_gitlab_push_map: Mapping of index images to Git repositories :param bool overwrite_from_index: Whether the request wants to overwrite from_index + :param Optional[str] overwrite_from_index_token: Registry credentials supplied for overwrite :return: The resolved build sources :rtype: BuildSources :raises IIBError: if the git mapping is missing, overwrite is requested on - the divergent path, or the base OCP branch is not onboarded. + the divergent or chained path, or required Git scaffolding is not onboarded. """ - index_git_repo = resolve_git_url(from_index=from_index, index_repo_map=index_to_gitlab_push_map) + chained_source = resolve_chained_build_source( + from_index, overwrite_from_index, overwrite_from_index_token + ) + mapping_index = chained_source.original_from_index if chained_source else from_index + index_git_repo = resolve_git_url( + from_index=mapping_index, + index_repo_map=index_to_gitlab_push_map, + ) if not index_git_repo: raise IIBError( - f"Git repository mapping not found for from_index: {from_index}. " + f"Git repository mapping not found for from_index: {mapping_index}. " "index_to_gitlab_push_map is required." ) token_name, git_token = get_git_token(index_git_repo) - tag = get_index_tag(from_index) + tag = get_index_tag(mapping_index) + + if chained_source: + target_branch = tag + if not remote_branch_exists(index_git_repo, target_branch, token_name, git_token): + target_branch = ocp_version + if not remote_branch_exists(index_git_repo, target_branch, token_name, git_token): + raise IIBError( + f"Base OCP branch '{target_branch}' is not onboarded for " + f'{index_git_repo}; chained build scaffolding is unavailable.' + ) + + set_request_state(request_id, 'in_progress', 'Cloning Git repository') + local_git_repo_path = Path(temp_dir) / 'git' / target_branch + local_git_repo_path.mkdir(parents=True, exist_ok=True) + clone_git_repo( + index_git_repo, + target_branch, + token_name, + git_token, + str(local_git_repo_path), + ) + catalog_path = local_git_repo_path / 'configs' + if not catalog_path.exists(): + raise IIBError(f'Catalogs directory not found in {local_git_repo_path}') + + parent_catalog = extract_catalog_from_image( + chained_source.parent_index_image_resolved, temp_dir + ) + shutil.rmtree(catalog_path) + shutil.copytree(parent_catalog, catalog_path, symlinks=True) + index_db_path = fetch_and_verify_request_index_db_artifact( + chained_source.parent_index_image_resolved, + chained_source.parent_request_id, + temp_dir, + ) + return BuildSources( + index_git_repo=index_git_repo, + local_git_repo_path=str(local_git_repo_path), + localized_git_catalog_path=str(catalog_path), + index_db_path=index_db_path, + target_branch=target_branch, + source_kind=BuildSourceKind.CHAINED, + ) + set_request_state(request_id, 'in_progress', 'Cloning Git repository') if remote_branch_exists(index_git_repo, tag, token_name, git_token): @@ -722,7 +936,7 @@ def prepare_build_sources( localized_git_catalog_path=str(catalog_path), index_db_path=None, target_branch=target_branch, - is_divergent=False, + source_kind=BuildSourceKind.STANDARD, ) # Divergent path. @@ -762,7 +976,7 @@ def prepare_build_sources( localized_git_catalog_path=str(catalog_path), index_db_path=extracted_db, target_branch=target_branch, - is_divergent=True, + source_kind=BuildSourceKind.DIVERGENT, ) @@ -792,8 +1006,16 @@ def prepare_git_repository_for_build( :param Dict[str, str] index_to_gitlab_push_map: Mapping of index images to Git repositories :return: Tuple of (index_git_repo, local_git_repo_path, localized_git_catalog_path) :rtype: Tuple[str, str, str] - :raises IIBError: If Git repository cannot be resolved or configs directory not found + :raises IIBError: If chaining is unsupported, the Git repository cannot be resolved, + or the configs directory is not found. """ + parent_request_id = get_iib_output_request_id(from_index) + if parent_request_id is not None: + raise IIBError( + 'Chaining is only supported for add, rm, and fbc-operations requests; ' + f'{from_index} is the output of request {parent_request_id}.' + ) + # Get Git repository information index_git_repo = resolve_git_url(from_index=from_index, index_repo_map=index_to_gitlab_push_map) if not index_git_repo: @@ -956,15 +1178,18 @@ def replicate_image_to_tagged_destinations( def cleanup_merge_request_if_exists( mr_details: Optional[Dict[str, str]], index_git_repo: Optional[str], + raise_on_error: bool = False, ) -> None: """ Close merge request if it was created. - This function attempts to close a merge request and logs a warning - if the operation fails. + This function attempts to close a merge request. By default it logs a warning + if closure fails; callers may require the error to propagate for a success-path cleanup. :param Optional[Dict[str, str]] mr_details: Details of the merge request :param Optional[str] index_git_repo: URL of the Git repository + :param bool raise_on_error: Re-raise closure errors when cleanup is a required success step. + :raises IIBError: If closure fails and ``raise_on_error`` is true. """ if mr_details and index_git_repo: try: @@ -972,6 +1197,8 @@ def cleanup_merge_request_if_exists( log.info("Closed merge request: %s", mr_details.get('mr_url')) except IIBError as e: log.warning("Failed to close merge request: %s", e) + if raise_on_error: + raise def merge_mr_after_build( diff --git a/iib/workers/tasks/oras_utils.py b/iib/workers/tasks/oras_utils.py index 65caedff9..cb2e07e56 100644 --- a/iib/workers/tasks/oras_utils.py +++ b/iib/workers/tasks/oras_utils.py @@ -116,6 +116,22 @@ def get_indexdb_artifact_pullspec(from_index: str) -> str: ) +def get_request_indexdb_artifact_pullspec(index_image: str, request_id: int) -> str: + """ + Construct the per-request index.db artifact pullspec. + + :param str index_image: The resolved output index image. + :param int request_id: The IIB request that produced the artifact. + :return: The request-specific index.db artifact pullspec. + :rtype: str + """ + conf = get_worker_config() + tag = f'{_get_content_addressed_artifact_tag(index_image)}-{request_id}' + return conf['iib_index_db_artifact_template'].format( + registry=conf['iib_index_db_artifact_registry'], tag=tag + ) + + @instrument_tracing(span_name="workers.tasks.oras_utils.get_oras_artifact") def get_oras_artifact( artifact_ref: str, diff --git a/tests/test_workers/test_tasks/test_build.py b/tests/test_workers/test_tasks/test_build.py index b7ff2bd32..58bcea63c 100644 --- a/tests/test_workers/test_tasks/test_build.py +++ b/tests/test_workers/test_tasks/test_build.py @@ -259,6 +259,16 @@ def test_create_and_push_manifest_list_failure_to_rm_manifest_list(mock_run_cmd, False, True, ), + (None, None, False, '{default}', None, True, True), + ( + 'registry-proxy.domain.local', + None, + False, + 'registry-proxy.domain.local/{default_no_registry}', + None, + True, + True, + ), ( None, 'quay.io/ns/iib:v4.5', @@ -305,14 +315,16 @@ def test_update_index_image_pull_spec( arches = {'amd64'} overwrite_token = 'username:password' - mock_get_rslv_img.return_value = "quay.io/ns/iib@sha256:abcdef1234" + final_resolved = 'quay.io/ns/iib@sha256:abcdef1234' + internal_resolved = 'quay.io/namespace/some-image@sha256:9876543210' + mock_get_rslv_img.side_effect = [final_resolved, internal_resolved] mock_gwc.return_value = { 'iib_index_image_output_registry': iib_index_image_output_registry, 'iib_registry': 'quay.io', } if add_or_rm: - build._update_index_image_pull_spec( + result = build._update_index_image_pull_spec( default, request_id, arches, @@ -324,7 +336,7 @@ def test_update_index_image_pull_spec( is_image_fbc=is_image_fbc, ) else: - build._update_index_image_pull_spec( + result = build._update_index_image_pull_spec( default, request_id, arches, @@ -338,6 +350,13 @@ def test_update_index_image_pull_spec( mock_ur.assert_called_once() update_request_payload = mock_ur.call_args[0][1] if add_or_rm: + assert result == final_resolved + assert result == update_request_payload['index_image_resolved'] + assert update_request_payload['internal_index_image_copy_resolved'] == internal_resolved + assert mock_get_rslv_img.call_args_list == [ + mock.call(expected_pull_spec), + mock.call(default), + ] assert update_request_payload.keys() == { 'arches', 'index_image', @@ -346,6 +365,8 @@ def test_update_index_image_pull_spec( 'internal_index_image_copy_resolved', } else: + assert result is None + mock_get_rslv_img.assert_not_called() assert update_request_payload.keys() == {'arches', 'index_image'} assert update_request_payload['index_image'] == expected_pull_spec if overwrite: diff --git a/tests/test_workers/test_tasks/test_build_containerized_add.py b/tests/test_workers/test_tasks/test_build_containerized_add.py index 9fecb16af..8ee1be7c8 100644 --- a/tests/test_workers/test_tasks/test_build_containerized_add.py +++ b/tests/test_workers/test_tasks/test_build_containerized_add.py @@ -107,6 +107,7 @@ def test_handle_containerized_add_request( index_db_path = '/tmp/index.db' temp_dir_path = '/tmp/iib-123-temp' from_index = 'index:latest' + overwrite_from_index_token = "user:pass" mock_get_resolved.return_value = resolved_bundles mock_td.return_value.__enter__.return_value = temp_dir_path @@ -134,7 +135,7 @@ def test_handle_containerized_add_request( localized_git_catalog_path=localized_git_catalog_path, index_db_path=None, target_branch='v4.12', - is_divergent=False, + source_kind=containerized_utils.BuildSourceKind.STANDARD, ) mock_fetch_index_db.return_value = index_db_path @@ -160,7 +161,8 @@ def test_handle_containerized_add_request( mock_opm_migrate.return_value = (catalog_from_db, None) # Mock commit and push - mock_git_commit.return_value = ({'mr_id': 1}, 'commit_sha_123') + mr_details = {'mr_id': 1} + mock_git_commit.return_value = (mr_details, 'commit_sha_123') # Mock pipeline monitoring image_url = 'registry.example.com/output-image:tag' @@ -169,6 +171,7 @@ def test_handle_containerized_add_request( # Mock replication output_pull_specs = ['registry.example.com/final-image:123'] mock_replicate.return_value = output_pull_specs + mock_update_pull_spec.return_value = 'registry.example.com/final-image@sha256:destination' # Mock final artifact push mock_push_index_db.return_value = None @@ -183,7 +186,7 @@ def test_handle_containerized_add_request( from_index=from_index, check_related_images=check_related_images, deprecation_list=deprecation_list, - overwrite_from_index_token="user:pass", + overwrite_from_index_token=overwrite_from_index_token, ) else: build_containerized_add.handle_containerized_add_request( @@ -193,7 +196,7 @@ def test_handle_containerized_add_request( from_index=from_index, check_related_images=check_related_images, deprecation_list=deprecation_list, - overwrite_from_index_token="user:pass", + overwrite_from_index_token=overwrite_from_index_token, ) # Verifications @@ -218,6 +221,10 @@ def test_handle_containerized_add_request( ocp_version='v4.12', index_to_gitlab_push_map={}, overwrite_from_index=False, + overwrite_from_index_token=overwrite_from_index_token, + ) + assert mock_prepare_sources.call_args.kwargs['overwrite_from_index_token'] == ( + overwrite_from_index_token ) # Verify bundle checks @@ -284,8 +291,10 @@ def test_handle_containerized_add_request( ) mock_push_index_db.assert_called_once() - assert mock_push_index_db.call_args.kwargs['output_image'] == image_url - mock_cleanup_mr.assert_called_once() + assert mock_push_index_db.call_args.kwargs['output_image'] == ( + 'registry.example.com/final-image@sha256:destination' + ) + mock_cleanup_mr.assert_called_once_with(mr_details, index_git_repo, raise_on_error=True) mock_cleanup_failure.assert_not_called() @@ -383,7 +392,7 @@ def test_handle_containerized_add_request_failure( localized_git_catalog_path='/tmp/repo/catalog', index_db_path=None, target_branch='v4.12', - is_divergent=False, + source_kind=containerized_utils.BuildSourceKind.STANDARD, ) # Mock TD @@ -521,7 +530,7 @@ def test_handle_containerized_add_request_overwrite( localized_git_catalog_path=localized_git_catalog_path, index_db_path=None, target_branch='v4.12', - is_divergent=False, + source_kind=containerized_utils.BuildSourceKind.STANDARD, ) mock_fetch_index_db.return_value = index_db_path @@ -542,6 +551,7 @@ def test_handle_containerized_add_request_overwrite( output_pull_specs = ['registry.example.com/final-image:456'] mock_replicate.return_value = output_pull_specs + mock_update_pull_spec.return_value = 'index@sha256:destination' mock_push_index_db.return_value = None @@ -562,7 +572,7 @@ def test_handle_containerized_add_request_overwrite( # Verify the handler completed successfully mock_push_index_db.assert_called_once() - assert mock_push_index_db.call_args.kwargs['output_image'] == image_url + assert mock_push_index_db.call_args.kwargs['output_image'] == 'index@sha256:destination' mock_cleanup_failure.assert_not_called() @@ -601,7 +611,14 @@ def test_handle_containerized_add_request_overwrite( @mock.patch('iib.workers.tasks.build_containerized_add.set_registry_token') @mock.patch('iib.workers.tasks.build_containerized_add.reset_docker_config') @mock.patch('iib.workers.tasks.build_containerized_add.merge_mr_after_build') -def test_add_divergent_never_merges( +@pytest.mark.parametrize( + 'source_kind', + ( + containerized_utils.BuildSourceKind.DIVERGENT, + containerized_utils.BuildSourceKind.CHAINED, + ), +) +def test_add_nonmergeable_source_never_merges( mock_merge_mr, mock_reset_docker, mock_set_token, @@ -635,9 +652,10 @@ def test_add_divergent_never_merges( mock_cleanup_failure, mock_makedirs, mock_copytree, + source_kind, tmpdir, ): - """Divergent path must never fall back to ORAS and must never merge the MR.""" + """Nonmergeable sources use their extracted index.db and never merge the MR.""" bundles = ['some-bundle:latest'] request_id = 789 binary_image = 'binary-image:latest' @@ -670,7 +688,7 @@ def test_add_divergent_never_merges( localized_git_catalog_path=localized_git_catalog_path, index_db_path=index_db_path, target_branch='v4.14', - is_divergent=True, + source_kind=source_kind, ) mock_path_isdir.return_value = True @@ -684,13 +702,19 @@ def test_add_divergent_never_merges( mr_details = {'mr_id': 1, 'mr_url': 'https://gitlab.com/mr/1', 'source_branch': 'iib-789-v4.14'} mock_git_commit.return_value = (mr_details, 'commit_sha_789') - image_url = 'registry.example.com/output-image:tag' + image_url = 'registry.example.com/output-image@sha256:' + 'a' * 64 mock_monitor.return_value = image_url output_pull_specs = ['registry.example.com/final-image:789'] mock_replicate.return_value = output_pull_specs + resolved_output = 'registry.example.com/final-image@sha256:' + 'b' * 64 + mock_update_pull_spec.return_value = resolved_output mock_push_index_db.return_value = None + publication = mock.Mock() + publication.attach_mock(mock_update_pull_spec, 'metadata') + publication.attach_mock(mock_push_index_db, 'artifact') + publication.attach_mock(mock_cleanup_mr, 'close') build_containerized_add.handle_containerized_add_request( bundles=bundles, @@ -703,14 +727,15 @@ def test_add_divergent_never_merges( # Divergent path uses the extracted index.db, never ORAS. mock_fetch_index_db.assert_not_called() - assert mock_push_index_db.call_args.kwargs['output_image'] == image_url + assert mock_push_index_db.call_args.kwargs['output_image'] == resolved_output + assert [call[0] for call in publication.mock_calls] == ['metadata', 'artifact', 'close'] # overwrite_from_index=True here: the divergent BuildSources bypasses Task 4's # entry-point overwrite rejection (mocked directly), so the ONLY thing that can - # prevent a merge is the handler-level `and not sources.is_divergent` guard. If + # prevent a merge is the handler-level `sources.merge_allowed` guard. If # that guard were removed, this MR would be merged and this assertion would fail. mock_merge_mr.assert_not_called() - mock_cleanup_mr.assert_called_once() + mock_cleanup_mr.assert_called_once_with(mr_details, index_git_repo, raise_on_error=True) mock_opm_add.assert_called_once_with( base_dir=temp_dir_path, @@ -830,7 +855,7 @@ def test_handle_containerized_add_request_scopes_overwrite_token_to_bundles( localized_git_catalog_path=Path(local_git_repo_path) / 'configs', index_db_path=None, target_branch='v4.15', - is_divergent=False, + source_kind=containerized_utils.BuildSourceKind.STANDARD, ) mock_fetch_index_db.return_value = '/tmp/index.db' mock_path_isdir.return_value = False diff --git a/tests/test_workers/test_tasks/test_build_containerized_create_empty_index.py b/tests/test_workers/test_tasks/test_build_containerized_create_empty_index.py index 42d7573bb..d85754d00 100644 --- a/tests/test_workers/test_tasks/test_build_containerized_create_empty_index.py +++ b/tests/test_workers/test_tasks/test_build_containerized_create_empty_index.py @@ -18,7 +18,7 @@ @mock.patch('iib.workers.tasks.containerized_utils.set_request_state') @mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') @mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') -@mock.patch('iib.workers.tasks.containerized_utils._get_index_digest') +@mock.patch('iib.workers.tasks.oras_utils._get_index_digest') @mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') @mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') @mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') @@ -232,7 +232,7 @@ def test_handle_containerized_create_empty_index_primary_path( @mock.patch('iib.workers.tasks.containerized_utils.set_request_state') @mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') @mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') -@mock.patch('iib.workers.tasks.containerized_utils._get_index_digest') +@mock.patch('iib.workers.tasks.oras_utils._get_index_digest') @mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') @mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') @mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') @@ -592,7 +592,7 @@ def test_handle_containerized_create_empty_index_missing_git_mapping( @mock.patch('iib.workers.tasks.containerized_utils.set_request_state') @mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') @mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') -@mock.patch('iib.workers.tasks.containerized_utils._get_index_digest') +@mock.patch('iib.workers.tasks.oras_utils._get_index_digest') @mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') @mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') @mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') diff --git a/tests/test_workers/test_tasks/test_build_containerized_fbc_operations.py b/tests/test_workers/test_tasks/test_build_containerized_fbc_operations.py index 0302ed9ef..698f2c8d7 100644 --- a/tests/test_workers/test_tasks/test_build_containerized_fbc_operations.py +++ b/tests/test_workers/test_tasks/test_build_containerized_fbc_operations.py @@ -7,6 +7,12 @@ from iib.workers.tasks.utils import RequestConfigFBCOperation +@pytest.fixture(autouse=True) +def _mock_registry_token(): + with mock.patch('iib.workers.tasks.build_containerized_fbc_operations.set_registry_token'): + yield + + @mock.patch('iib.workers.tasks.containerized_utils.remote_branch_exists') @mock.patch('iib.workers.tasks.build_containerized_fbc_operations._update_index_image_pull_spec') @mock.patch('iib.workers.tasks.build_containerized_fbc_operations.cleanup_on_failure') @@ -89,12 +95,14 @@ def test_handle_containerized_fbc_operation_request( mock_ggt.return_value = ('token_name', 'token_value') # Mock os.path.exists for index.db check and catalogs dir check - with mock.patch('iib.workers.tasks.containerized_utils.Path.exists', return_value=True): + with mock.patch( + 'iib.workers.tasks.containerized_utils.Path.exists', return_value=True + ), mock.patch('iib.workers.tasks.containerized_utils.close_mr') as mock_close_mr: # Mock opm operation result mock_oraff.return_value = ('/tmp/updated_catalog_path', '/tmp/index.db', [], ['op1']) # Mock Konflux pipeline flow - mock_cmr.return_value = {'mr_url': 'http://mr.url'} + mock_cmr.return_value = {'mr_id': '1', 'mr_url': 'http://mr.url'} mock_glcs.return_value = 'sha123' mock_fp.return_value = [{'metadata': {'name': 'pipeline-run-1'}}] mock_wfpc.return_value = {'status': 'Succeeded'} @@ -109,6 +117,8 @@ def test_handle_containerized_fbc_operation_request( binary_image_config=binary_image_config, ) + mock_close_mr.assert_called_once_with(mock_cmr.return_value, index_git_repo) + # Assertions mock_prfb.assert_called_once_with( request_id, @@ -262,9 +272,11 @@ def test_handle_containerized_fbc_operation_request_multiple_fragments( mock_rgu.return_value = index_git_repo mock_ggt.return_value = ('token_name', 'token_value') - with mock.patch('iib.workers.tasks.containerized_utils.Path.exists', return_value=True): + with mock.patch( + 'iib.workers.tasks.containerized_utils.Path.exists', return_value=True + ), mock.patch('iib.workers.tasks.containerized_utils.close_mr') as mock_close_mr: mock_oraff.return_value = ('/tmp/updated', '/tmp/db', [], ['op1', 'op2']) - mock_cmr.return_value = {'mr_url': 'http://mr.url'} + mock_cmr.return_value = {'mr_id': '1', 'mr_url': 'http://mr.url'} mock_glcs.return_value = 'sha123' mock_fp.return_value = [{'metadata': {'name': 'pipeline-run-1'}}] mock_wfpc.return_value = {'status': 'Succeeded'} @@ -279,6 +291,8 @@ def test_handle_containerized_fbc_operation_request_multiple_fragments( binary_image_config=binary_image_config, ) + mock_close_mr.assert_called_once_with(mock_cmr.return_value, index_git_repo) + # Verify OPM operation was called with list of resolved fragments mock_oraff.assert_called_once_with( request_id=request_id, @@ -327,7 +341,12 @@ def test_handle_containerized_fbc_operation_request_multiple_fragments( @mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') @mock.patch('iib.workers.tasks.containerized_utils.set_request_state') @mock.patch('iib.workers.tasks.build_containerized_fbc_operations.merge_mr_after_build') +@mock.patch( + 'iib.workers.tasks.build_containerized_fbc_operations.prepare_build_sources', + wraps=containerized_utils.prepare_build_sources, +) def test_handle_containerized_fbc_operation_request_with_overwrite( + mock_prepare_sources, mock_merge_mr, mock_srs_utils, mock_makedirs, @@ -379,6 +398,7 @@ def test_handle_containerized_fbc_operation_request_with_overwrite( 'mr_url': 'https://gitlab.com/mr/1', 'source_branch': 'iib-request-10-v4.6', } + mock_uiips.return_value = 'quay.io/iib/from-index@sha256:destination' mock_docker_config = json.dumps({'auths': {}}) with mock.patch('iib.workers.tasks.containerized_utils.Path.exists', return_value=True): @@ -401,6 +421,7 @@ def test_handle_containerized_fbc_operation_request_with_overwrite( # Verify MR creation and merge for overwrite flow mock_cmr.assert_called_once() mock_merge_mr.assert_called_once() + assert mock_prepare_sources.call_args.kwargs['overwrite_from_index_token'] == (overwrite_token) # Verify DB artifacts pushed mock_pida_push.assert_called_once_with( @@ -408,7 +429,7 @@ def test_handle_containerized_fbc_operation_request_with_overwrite( from_index='quay.io/iib/from-index:latest', index_db_path='/tmp/d', operators=['op1'], - output_image='reg/img', + output_image='quay.io/iib/from-index@sha256:destination', overwrite_from_index=True, request_type='fbc_operations', ) @@ -554,7 +575,14 @@ def test_handle_containerized_fbc_operation_request_failure( @mock.patch('iib.workers.tasks.build_containerized_fbc_operations.get_resolved_image') @mock.patch('iib.workers.tasks.build_containerized_fbc_operations.set_request_state') @mock.patch('iib.workers.tasks.build_containerized_fbc_operations.reset_docker_config') -def test_fbc_operations_divergent_never_merges( +@pytest.mark.parametrize( + 'source_kind', + ( + containerized_utils.BuildSourceKind.DIVERGENT, + containerized_utils.BuildSourceKind.CHAINED, + ), +) +def test_fbc_operations_nonmergeable_source_never_merges( mock_rdc, mock_srs, mock_gri, @@ -573,9 +601,10 @@ def test_fbc_operations_divergent_never_merges( mock_cleanup_mr, mock_cof, mock_uiips, + source_kind, tmp_path, ): - """Divergent path must never fall back to ORAS and must never merge the MR.""" + """Nonmergeable sources use their extracted index.db and never merge the MR.""" request_id = 999 from_index = 'quay.io/iib/from-index:v4.99' binary_image = 'binary-image:latest' @@ -600,7 +629,7 @@ def test_fbc_operations_divergent_never_merges( localized_git_catalog_path=localized_git_catalog_path, index_db_path=index_db_path, target_branch='v4.14', - is_divergent=True, + source_kind=source_kind, ) mock_oraff.return_value = ('/tmp/updated_catalog_path', index_db_path, ['op1'], ['op1']) @@ -611,9 +640,15 @@ def test_fbc_operations_divergent_never_merges( 'source_branch': 'iib-request-999-v4.14', } mock_git_commit.return_value = (mr_details, 'commit_sha_999') - mock_monitor.return_value = 'registry/output-image:sha256-12345' + mock_monitor.return_value = 'registry/output-image@sha256:' + 'a' * 64 mock_replicate.return_value = ['registry.example.com/final-image:999'] + resolved_output = 'registry.example.com/final-image@sha256:' + 'b' * 64 + mock_uiips.return_value = resolved_output mock_push_index_db.return_value = None + publication = mock.Mock() + publication.attach_mock(mock_uiips, 'metadata') + publication.attach_mock(mock_push_index_db, 'artifact') + publication.attach_mock(mock_cleanup_mr, 'close') overwrite_token = 'user:token' build_containerized_fbc_operations.handle_containerized_fbc_operation_request( @@ -631,10 +666,12 @@ def test_fbc_operations_divergent_never_merges( # overwrite_from_index=True here: the divergent BuildSources bypasses Task 4's # entry-point overwrite rejection (mocked directly), so the ONLY thing that can - # prevent a merge is the handler-level `and not sources.is_divergent` guard. If + # prevent a merge is the handler-level `sources.merge_allowed` guard. If # that guard were removed, this MR would be merged and this assertion would fail. mock_merge_mr.assert_not_called() - mock_cleanup_mr.assert_called_once() + mock_cleanup_mr.assert_called_once_with(mr_details, index_git_repo, raise_on_error=True) + assert mock_push_index_db.call_args.kwargs['output_image'] == resolved_output + assert [call[0] for call in publication.mock_calls] == ['metadata', 'artifact', 'close'] mock_oraff.assert_called_once_with( request_id=request_id, @@ -732,7 +769,7 @@ def test_fbc_operation_scopes_overwrite_token_to_same_namespace_fragments( localized_git_catalog_path=str(tmp_path / 'git_repo' / 'configs'), index_db_path=None, target_branch='v4.15', - is_divergent=False, + source_kind=containerized_utils.BuildSourceKind.STANDARD, ) mock_fetch_index_db.return_value = index_db_path mock_oraff.return_value = ('/tmp/updated_catalog_path', index_db_path, ['op1'], ['op1']) @@ -847,7 +884,7 @@ def test_fbc_operations_complete_state_reason_summarizes_all_actions( localized_git_catalog_path=str(tmp_path / 'git_repo' / 'configs'), index_db_path=index_db_path, target_branch='v4.14', - is_divergent=False, + source_kind=containerized_utils.BuildSourceKind.STANDARD, ) mock_oraff.return_value = ( '/tmp/updated_catalog_path', diff --git a/tests/test_workers/test_tasks/test_build_containerized_rm.py b/tests/test_workers/test_tasks/test_build_containerized_rm.py index 6951491df..4b42ef678 100644 --- a/tests/test_workers/test_tasks/test_build_containerized_rm.py +++ b/tests/test_workers/test_tasks/test_build_containerized_rm.py @@ -8,6 +8,12 @@ from iib.workers.tasks.utils import RequestConfigAddRm +@pytest.fixture(autouse=True) +def _mock_registry_token(): + with mock.patch('iib.workers.tasks.build_containerized_rm.set_registry_token'): + yield + + @mock.patch('iib.workers.tasks.containerized_utils.remote_branch_exists') @mock.patch('iib.workers.tasks.build_containerized_rm.reset_docker_config') @mock.patch('iib.workers.tasks.build_containerized_rm.cleanup_on_failure') @@ -16,7 +22,7 @@ @mock.patch('iib.workers.tasks.containerized_utils.set_request_state') @mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') @mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') -@mock.patch('iib.workers.tasks.containerized_utils._get_index_digest') +@mock.patch('iib.workers.tasks.oras_utils._get_index_digest') @mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') @mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') @mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') @@ -45,7 +51,12 @@ @mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') @mock.patch('iib.workers.tasks.build_containerized_rm.merge_mr_after_build') @mock.patch('iib.workers.tasks.build_containerized_rm.set_registry_token') +@mock.patch( + 'iib.workers.tasks.build_containerized_rm.prepare_build_sources', + wraps=containerized_utils.prepare_build_sources, +) def test_handle_containerized_rm_request_success_with_overwrite( + mock_prepare_sources, mock_srt, mock_merge_mr, mock_makedirs, @@ -140,6 +151,7 @@ def test_handle_containerized_rm_request_success_with_overwrite( mock_gpiu.return_value = 'quay.io/konflux/built-image@sha256:xyz789' # Mock ORAS push related functions + mock_uiips.return_value = 'quay.io/namespace/index-image@sha256:destination' mock_giap.return_value = 'registry.io/index-db:v4.14' mock_gid.return_value = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abc0' @@ -175,6 +187,9 @@ def test_handle_containerized_rm_request_success_with_overwrite( binary_image_config=None, ), ) + assert mock_prepare_sources.call_args.kwargs['overwrite_from_index_token'] == ( + overwrite_from_index_token + ) # Verify OPM version was set. It reads a label off from_index, so the overwrite token # must be stamped for the resolved index -- otherwise a private from_index fails here. @@ -215,8 +230,8 @@ def test_handle_containerized_rm_request_success_with_overwrite( # Verify index.db was pushed (2 times: request_id tag + current-artifact tag) assert mock_poa.call_count == 2 - # Verify the content key was resolved from the built output image - mock_gid.assert_called_once_with('quay.io/konflux/built-image@sha256:xyz789') + # The final destination may have a different digest after replication. + mock_gid.assert_called_once_with('quay.io/namespace/index-image@sha256:destination') # Verify final state final_call = mock_srs.call_args_list[-1] @@ -236,7 +251,7 @@ def test_handle_containerized_rm_request_success_with_overwrite( @mock.patch('iib.workers.tasks.containerized_utils.set_request_state') @mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') @mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') -@mock.patch('iib.workers.tasks.containerized_utils._get_index_digest') +@mock.patch('iib.workers.tasks.oras_utils._get_index_digest') @mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') @mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') @mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') @@ -399,7 +414,7 @@ def test_handle_containerized_rm_request_with_mr( @mock.patch('iib.workers.tasks.containerized_utils.set_request_state') @mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') @mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') -@mock.patch('iib.workers.tasks.containerized_utils._get_index_digest') +@mock.patch('iib.workers.tasks.oras_utils._get_index_digest') @mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') @mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') @mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') @@ -847,7 +862,7 @@ def test_handle_containerized_rm_pipeline_failure( @mock.patch('iib.workers.tasks.containerized_utils.set_request_state') @mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') @mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') -@mock.patch('iib.workers.tasks.containerized_utils._get_index_digest') +@mock.patch('iib.workers.tasks.oras_utils._get_index_digest') @mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') @mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') @mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') @@ -1005,7 +1020,7 @@ def test_handle_containerized_rm_with_index_db_push( @mock.patch('iib.workers.tasks.containerized_utils._skopeo_copy') @mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') @mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') -@mock.patch('iib.workers.tasks.containerized_utils._get_index_digest') +@mock.patch('iib.workers.tasks.oras_utils._get_index_digest') @mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') @mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') @mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') @@ -1144,7 +1159,7 @@ def test_handle_containerized_rm_with_build_tags( @mock.patch('iib.workers.tasks.containerized_utils.set_request_state') @mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') @mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') -@mock.patch('iib.workers.tasks.containerized_utils._get_index_digest') +@mock.patch('iib.workers.tasks.oras_utils._get_index_digest') @mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') @mock.patch('iib.workers.tasks.containerized_utils.get_pipelinerun_image_url') @mock.patch('iib.workers.tasks.containerized_utils.wait_for_pipeline_completion') @@ -1171,7 +1186,7 @@ def test_handle_containerized_rm_with_build_tags( @mock.patch('iib.workers.tasks.build_containerized_rm.shutil.copytree') @mock.patch('iib.workers.tasks.containerized_utils.Path.exists') @mock.patch('iib.workers.tasks.containerized_utils.Path.mkdir') -def test_handle_containerized_rm_close_mr_failure_logged( +def test_handle_containerized_rm_close_mr_failure_fails_request( mock_makedirs, mock_exists, mock_copytree, @@ -1208,7 +1223,7 @@ def test_handle_containerized_rm_close_mr_failure_logged( mock_rdc, mock_rbe, ): - """Test that MR close failure is logged but doesn't fail the request.""" + """A successful build is not completed while its throw-away MR remains open.""" request_id = 10 operators = ['test-operator'] from_index = 'quay.io/namespace/index-image:v4.14' @@ -1257,20 +1272,17 @@ def test_handle_containerized_rm_close_mr_failure_logged( # Mock close_mr to raise error mock_close_mr.side_effect = IIBError('Failed to close MR') - # Test - should complete successfully despite MR close failure - build_containerized_rm.handle_containerized_rm_request( - operators=operators, - request_id=request_id, - from_index=from_index, - index_to_gitlab_push_map={'quay.io/namespace/index-image': 'https://gitlab.com/repo'}, - ) + with pytest.raises(IIBError, match='Failed to remove operators: Failed to close MR'): + build_containerized_rm.handle_containerized_rm_request( + operators=operators, + request_id=request_id, + from_index=from_index, + index_to_gitlab_push_map={'quay.io/namespace/index-image': 'https://gitlab.com/repo'}, + ) - # Verify MR was attempted to be closed mock_close_mr.assert_called_once() - - # Verify request still completed successfully - final_call = mock_srs.call_args_list[-1] - assert final_call[0][1] == 'complete' + mock_cof.assert_called_once() + assert all(call.args[1] != 'complete' for call in mock_srs.call_args_list) @mock.patch('iib.workers.tasks.containerized_utils.remote_branch_exists') @@ -1528,7 +1540,14 @@ def test_handle_containerized_rm_missing_output_pull_spec( @mock.patch('iib.workers.tasks.build_containerized_rm.prepare_request_for_build') @mock.patch('iib.workers.tasks.build_containerized_rm.set_request_state') @mock.patch('iib.workers.tasks.build_containerized_rm.set_registry_token') -def test_rm_divergent_never_merges( +@pytest.mark.parametrize( + 'source_kind', + ( + containerized_utils.BuildSourceKind.DIVERGENT, + containerized_utils.BuildSourceKind.CHAINED, + ), +) +def test_rm_nonmergeable_source_never_merges( mock_srt, mock_srs, mock_prfb, @@ -1556,9 +1575,10 @@ def test_rm_divergent_never_merges( mock_update_pull_spec, mock_cof, mock_rdc, + source_kind, tmp_path, ): - """Divergent path must never fall back to ORAS and must never merge the MR.""" + """Nonmergeable sources use their extracted index.db and never merge the MR.""" request_id = 789 operators = ['operator1'] from_index = 'quay.io/namespace/index-image:v4.99' @@ -1587,7 +1607,7 @@ def test_rm_divergent_never_merges( localized_git_catalog_path=localized_git_catalog_path, index_db_path=index_db_path, target_branch='v4.14', - is_divergent=True, + source_kind=source_kind, ) mock_voe.return_value = ({'operator1'}, index_db_path) @@ -1600,9 +1620,15 @@ def test_rm_divergent_never_merges( 'source_branch': 'iib-request-789-v4.14', } mock_git_commit.return_value = (mr_details, 'commit_sha_789') - mock_monitor.return_value = 'quay.io/konflux/built-image@sha256:xyz789' + mock_monitor.return_value = 'quay.io/konflux/built-image@sha256:' + 'a' * 64 mock_replicate.return_value = ['registry.example.com/final-image:789'] + resolved_output = 'registry.example.com/final-image@sha256:' + 'b' * 64 + mock_update_pull_spec.return_value = resolved_output mock_push_index_db.return_value = None + publication = mock.Mock() + publication.attach_mock(mock_update_pull_spec, 'metadata') + publication.attach_mock(mock_push_index_db, 'artifact') + publication.attach_mock(mock_cleanup_mr, 'close') build_containerized_rm.handle_containerized_rm_request( operators=operators, @@ -1619,14 +1645,14 @@ def test_rm_divergent_never_merges( # overwrite_from_index=True here: the divergent BuildSources bypasses Task 4's # entry-point overwrite rejection (mocked directly), so the ONLY thing that can - # prevent a merge is the handler-level `and not sources.is_divergent` guard. If + # prevent a merge is the handler-level `sources.merge_allowed` guard. If # that guard were removed, this MR would be merged and this assertion would fail. mock_merge_mr.assert_not_called() - mock_cleanup_mr.assert_called_once() + mock_cleanup_mr.assert_called_once_with(mr_details, index_git_repo, raise_on_error=True) mock_orrf.assert_called_once() assert mock_orrf.call_args.kwargs['index_db_path'] == index_db_path mock_cof.assert_not_called() - expected_output_image = 'quay.io/konflux/built-image@sha256:xyz789' - assert mock_push_index_db.call_args.kwargs['output_image'] == expected_output_image + assert mock_push_index_db.call_args.kwargs['output_image'] == resolved_output + assert [call[0] for call in publication.mock_calls] == ['metadata', 'artifact', 'close'] diff --git a/tests/test_workers/test_tasks/test_containerized_utils.py b/tests/test_workers/test_tasks/test_containerized_utils.py index e87f6d608..9d13d9609 100644 --- a/tests/test_workers/test_tasks/test_containerized_utils.py +++ b/tests/test_workers/test_tasks/test_containerized_utils.py @@ -11,8 +11,13 @@ from iib.exceptions import ArtifactNotFoundError, IIBError, FileNotFoundInImageError from iib.workers.tasks import containerized_utils as cu from iib.workers.tasks.containerized_utils import ( + ChainedBuildSource, + get_iib_output_request_id, + resolve_chained_build_source, + extract_catalog_from_image, extract_catalog_and_db_from_image, extract_files_from_image_non_privileged, + fetch_and_verify_request_index_db_artifact, pull_index_db_artifact, push_index_db_artifact, write_build_metadata, @@ -24,6 +29,178 @@ ) +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@pytest.mark.parametrize( + ('image', 'expected'), + ( + ('registry.internal/iib-build:42', 42), + ('registry.external/iib-build:42', 42), + ('registry.internal/iib-build:latest', None), + ('registry.other/iib-build:42', None), + ('registry.internal/not-iib-build:42', None), + ), +) +def test_get_iib_output_request_id(mock_config, image, expected): + mock_config.return_value = { + 'iib_registry': 'registry.internal', + 'iib_index_image_output_registry': 'registry.external', + 'iib_image_push_template': '{registry}/iib-build:{request_id}', + } + assert get_iib_output_request_id(image) == expected + + +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +def test_get_iib_output_request_id_honors_custom_template(mock_config): + mock_config.return_value = { + 'iib_registry': 'registry.internal/team', + 'iib_index_image_output_registry': None, + 'iib_image_push_template': '{registry}/request-{request_id}/index:latest', + } + assert get_iib_output_request_id('registry.internal/team/request-73/index:latest') == 73 + + +def test_prepare_git_repository_rejects_iib_output_before_git_lookup(tmp_path): + """Unsupported request types reject tagged IIB outputs before Git side effects.""" + config = { + 'iib_registry': 'registry.internal', + 'iib_index_image_output_registry': None, + 'iib_image_push_template': '{registry}/iib-build:{request_id}', + } + with mock.patch( + 'iib.workers.tasks.containerized_utils.get_worker_config', return_value=config + ), mock.patch( + 'iib.workers.tasks.containerized_utils.resolve_git_url', return_value=None + ) as mock_resolve_git_url: + with pytest.raises( + IIBError, + match='Chaining is only supported for add, rm, and fbc-operations requests', + ): + cu.prepare_git_repository_for_build( + request_id=100, + from_index='registry.internal/iib-build:42', + temp_dir=str(tmp_path), + branch='v4.19', + index_to_gitlab_push_map={}, + ) + + mock_resolve_git_url.assert_not_called() + + +def _chain_request(request_id, from_index, **changes): + output = f'registry.internal/iib-build:{request_id}' + request = { + 'id': request_id, + 'state': 'complete', + 'request_type': 'add', + 'from_index': from_index, + 'index_image': output, + 'index_image_resolved': f'{output}@sha256:{request_id:064x}', + } + request.update(changes) + return request + + +@mock.patch('iib.workers.tasks.containerized_utils.get_request') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +def test_resolve_chained_build_source_multi_hop(mock_config, mock_get_request): + mock_config.return_value = { + 'iib_registry': 'registry.internal', + 'iib_index_image_output_registry': None, + 'iib_image_push_template': '{registry}/iib-build:{request_id}', + } + requests = { + 55: _chain_request(55, 'registry.internal/iib-build:42', request_type='rm'), + 42: _chain_request(42, 'quay.io/ns/index:v4.17', request_type='fbc-operations'), + } + mock_get_request.side_effect = requests.__getitem__ + + result = resolve_chained_build_source('registry.internal/iib-build:55', False, None) + + assert isinstance(result, ChainedBuildSource) + assert result.parent_request_id == 55 + assert result.parent_index_image == 'registry.internal/iib-build:55' + assert result.parent_index_image_resolved == ( + 'registry.internal/iib-build:55@sha256:' f'{55:064x}' + ) + assert result.original_from_index == 'quay.io/ns/index:v4.17' + assert result.ancestry == (55, 42) + + +@mock.patch('iib.workers.tasks.containerized_utils.get_request') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +def test_resolve_chained_build_source_returns_none_for_standard(mock_config, mock_get_request): + mock_config.return_value = { + 'iib_registry': 'registry.internal', + 'iib_index_image_output_registry': None, + 'iib_image_push_template': '{registry}/iib-build:{request_id}', + } + assert resolve_chained_build_source('quay.io/ns/index:v4.17', False, None) is None + mock_get_request.assert_not_called() + + +@mock.patch('iib.workers.tasks.containerized_utils.get_request') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@pytest.mark.parametrize( + ('overwrite', 'token'), + ((True, None), (False, 'user:token'), (True, 'user:token')), +) +def test_resolve_chained_build_source_rejects_overwrite( + mock_config, mock_get_request, overwrite, token +): + mock_config.return_value = { + 'iib_registry': 'registry.internal', + 'iib_index_image_output_registry': None, + 'iib_image_push_template': '{registry}/iib-build:{request_id}', + } + with pytest.raises(IIBError, match='cannot be overwritten'): + resolve_chained_build_source('registry.internal/iib-build:42', overwrite, token) + mock_get_request.assert_not_called() + + +@mock.patch('iib.workers.tasks.containerized_utils.get_request') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@pytest.mark.parametrize( + ('change', 'message'), + ( + ({'state': 'in_progress'}, 'is not complete'), + ({'request_type': 'merge-index-image'}, 'unsupported request type'), + ({'index_image': 'registry.internal/iib-build:999'}, 'does not match'), + ({'index_image_resolved': None}, 'has no resolved index image'), + ({'from_index': None}, 'has no from_index'), + ), +) +def test_resolve_chained_build_source_rejects_invalid_parent( + mock_config, mock_get_request, change, message +): + mock_config.return_value = { + 'iib_registry': 'registry.internal', + 'iib_index_image_output_registry': None, + 'iib_image_push_template': '{registry}/iib-build:{request_id}', + } + parent = _chain_request(42, 'quay.io/ns/index:v4.17') + parent.update(change) + mock_get_request.return_value = parent + with pytest.raises(IIBError, match=message): + resolve_chained_build_source('registry.internal/iib-build:42', False, None) + + +@mock.patch('iib.workers.tasks.containerized_utils.get_request') +@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +def test_resolve_chained_build_source_rejects_cycle(mock_config, mock_get_request): + mock_config.return_value = { + 'iib_registry': 'registry.internal', + 'iib_index_image_output_registry': None, + 'iib_image_push_template': '{registry}/iib-build:{request_id}', + } + requests = { + 42: _chain_request(42, 'registry.internal/iib-build:55'), + 55: _chain_request(55, 'registry.internal/iib-build:42'), + } + mock_get_request.side_effect = requests.__getitem__ + with pytest.raises(IIBError, match='cycle'): + resolve_chained_build_source('registry.internal/iib-build:42', False, None) + + @patch('iib.workers.tasks.containerized_utils.get_worker_config') @patch('iib.workers.tasks.containerized_utils.log') @patch('iib.workers.tasks.containerized_utils.refresh_indexdb_cache_for_image') @@ -326,94 +503,193 @@ def test_pull_reports_registry_failure_as_distinct_from_missing(m_gwc, m_ref, m_ assert '503 Service Unavailable' in str(exc_info.value) -@mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') -@mock.patch('iib.workers.tasks.containerized_utils._get_index_digest') -@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.get_oras_artifact') +@mock.patch('iib.workers.tasks.containerized_utils.get_request_indexdb_artifact_pullspec') +@pytest.mark.parametrize('filename', ('index.db', 'extracted_index.db')) +def test_fetch_request_index_db_artifact(mock_ref, mock_pull, filename, tmp_path): + artifact_dir = tmp_path / 'artifact' + artifact_dir.mkdir() + index_db = artifact_dir / filename + index_db.write_bytes(b'sqlite') + mock_ref.return_value = 'quay.io/iib/index-db:idb-abc-42' + mock_pull.return_value = str(artifact_dir) + + result = fetch_and_verify_request_index_db_artifact( + 'registry.internal/iib-build:42@sha256:abc', 42, str(tmp_path) + ) + + assert result == str(index_db) + mock_ref.assert_called_once_with('registry.internal/iib-build:42@sha256:abc', 42) + + +@mock.patch('iib.workers.tasks.containerized_utils.get_oras_artifact') +@mock.patch('iib.workers.tasks.containerized_utils.get_request_indexdb_artifact_pullspec') +def test_fetch_request_index_db_artifact_rejects_unknown_filename(mock_ref, mock_pull, tmp_path): + (tmp_path / 'other.db').write_bytes(b'sqlite') + mock_ref.return_value = 'quay.io/iib/index-db:idb-abc-42' + mock_pull.return_value = str(tmp_path) + + with pytest.raises(IIBError, match='Index.db file not found'): + fetch_and_verify_request_index_db_artifact( + 'registry.internal/iib-build:42@sha256:abc', 42, str(tmp_path) + ) + + +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') +@mock.patch('iib.workers.tasks.oras_utils.get_image_digest', return_value='sha256:abc') @mock.patch('iib.workers.tasks.containerized_utils.set_request_state') -@mock.patch('pathlib.Path.exists', return_value=True) -def test_push_keys_current_artifact_on_output_digest( - m_exists, m_state, m_gwc, m_digest, m_push, tmp_path +@pytest.mark.parametrize('filename', ('index.db', 'extracted_index.db')) +@pytest.mark.parametrize('overwrite', (False, True)) +def test_push_request_artifact_normalizes_payload( + mock_state, mock_digest, mock_config, filename, overwrite, tmp_path, monkeypatch ): - m_gwc.return_value = { + mock_config.return_value = { 'iib_index_db_artifact_registry': 'quay.io/iib', 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', 'iib_index_db_artifact_tag_template': 'idb-{digest}', } - # digest resolved from the OUTPUT image, not from_index - m_digest.return_value = 'f' * 64 + source_db = tmp_path / filename + source_db.write_bytes(b'parent database contents') + artifacts = {} + + def publish_artifact(artifact_ref, local_path, cwd, annotations): + artifacts[artifact_ref] = {local_path: (Path(cwd) / local_path).read_bytes()} + + def pull_artifact(artifact_ref, base_dir): + artifact_dir = Path(base_dir) / 'pulled' + artifact_dir.mkdir() + for name, contents in artifacts[artifact_ref].items(): + (artifact_dir / name).write_bytes(contents) + return str(artifact_dir) + + monkeypatch.setattr(cu, 'push_oras_artifact', publish_artifact) + monkeypatch.setattr(cu, 'get_oras_artifact', pull_artifact) + push_index_db_artifact( + request_id=42, + from_index='quay.io/ns/index:test', + index_db_path=str(source_db), + operators=['op1'], + output_image='registry.internal/iib-build:42@sha256:abc', + overwrite_from_index=overwrite, + request_type='add', + ) + + expected_refs = {'quay.io/iib/index-db:idb-abc-42'} + if overwrite: + expected_refs.add('quay.io/iib/index-db:idb-abc') + assert set(artifacts) == expected_refs + for payload in artifacts.values(): + assert payload == {'index.db': b'parent database contents'} + assert source_db.read_bytes() == b'parent database contents' + + fetched = fetch_and_verify_request_index_db_artifact( + 'registry.internal/iib-build:42@sha256:abc', 42, str(tmp_path) + ) + assert Path(fetched).name == 'index.db' + assert Path(fetched).read_bytes() == b'parent database contents' + + +@mock.patch('iib.workers.tasks.containerized_utils.get_oras_artifact') +@mock.patch('iib.workers.tasks.containerized_utils.get_request_indexdb_artifact_pullspec') +@pytest.mark.parametrize( + ('error', 'message'), + ( + (ArtifactNotFoundError('missing'), 'not found'), + (IIBError('registry timeout'), 'Failed to pull'), + ), +) +def test_fetch_request_index_db_artifact_preserves_failure_kind( + mock_ref, mock_pull, error, message, tmp_path +): + mock_ref.return_value = 'quay.io/iib/index-db:idb-abc-42' + mock_pull.side_effect = error + with pytest.raises(IIBError, match=message): + fetch_and_verify_request_index_db_artifact( + 'registry.internal/iib-build:42@sha256:abc', 42, str(tmp_path) + ) + + +@mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') +@mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_request_indexdb_artifact_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +@mock.patch('pathlib.Path.exists', return_value=True) +def test_push_keys_current_artifact_on_output_digest( + m_exists, m_state, m_request_ref, m_current_ref, m_push, tmp_path +): db = tmp_path / 'index.db' db.write_text('x') + output_image = 'quay.io/ns/foo@sha256:' + 'f' * 64 + m_request_ref.return_value = 'quay.io/iib/index-db:idb-' + 'f' * 64 + '-42' + m_current_ref.return_value = 'quay.io/iib/index-db:idb-' + 'f' * 64 result = push_index_db_artifact( request_id=42, from_index='quay.io/ns/foo:v4.17', index_db_path=str(db), operators=['op1'], - output_image='quay.io/ns/foo@sha256:' + 'f' * 64, + output_image=output_image, overwrite_from_index=True, request_type='add', ) assert result is None - m_digest.assert_called_with('quay.io/ns/foo@sha256:' + 'f' * 64) + m_request_ref.assert_called_once_with(output_image, 42) + m_current_ref.assert_called_once_with(output_image) pushed_refs = {c.kwargs['artifact_ref'] for c in m_push.call_args_list} assert 'quay.io/iib/index-db:idb-' + 'f' * 64 in pushed_refs # warm-push (overwrite) assert 'quay.io/iib/index-db:idb-' + 'f' * 64 + '-42' in pushed_refs # per-request tag @mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') -@mock.patch('iib.workers.tasks.containerized_utils._get_index_digest') -@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_request_indexdb_artifact_pullspec') @mock.patch('iib.workers.tasks.containerized_utils.set_request_state') @mock.patch('pathlib.Path.exists', return_value=True) def test_push_throwaway_skips_current_artifact( - m_exists, m_state, m_gwc, m_digest, m_push, tmp_path + m_exists, m_state, m_request_ref, m_current_ref, m_push, tmp_path ): - m_gwc.return_value = { - 'iib_index_db_artifact_registry': 'quay.io/iib', - 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', - 'iib_index_db_artifact_tag_template': 'idb-{digest}', - } - m_digest.return_value = 'a' * 64 db = tmp_path / 'index.db' db.write_text('x') + output_image = 'quay.io/ns/foo@sha256:' + 'a' * 64 + m_request_ref.return_value = 'quay.io/iib/index-db:idb-' + 'a' * 64 + '-7' push_index_db_artifact( request_id=7, from_index='quay.io/ns/foo:v4.17', index_db_path=str(db), operators=[], - output_image='quay.io/ns/foo@sha256:' + 'a' * 64, + output_image=output_image, overwrite_from_index=False, request_type='add', ) + m_request_ref.assert_called_once_with(output_image, 7) + m_current_ref.assert_not_called() pushed_refs = {c.kwargs['artifact_ref'] for c in m_push.call_args_list} assert pushed_refs == {'quay.io/iib/index-db:idb-' + 'a' * 64 + '-7'} # only per-request tag @mock.patch('iib.workers.tasks.containerized_utils.push_oras_artifact') -@mock.patch('iib.workers.tasks.containerized_utils._get_index_digest') -@mock.patch('iib.workers.tasks.containerized_utils.get_worker_config') +@mock.patch('iib.workers.tasks.containerized_utils.get_indexdb_artifact_pullspec') +@mock.patch('iib.workers.tasks.containerized_utils.get_request_indexdb_artifact_pullspec') @mock.patch('iib.workers.tasks.containerized_utils.set_request_state') @mock.patch('pathlib.Path.exists', return_value=True) -def test_push_honors_configured_tag_template(m_exists, m_state, m_gwc, m_digest, m_push, tmp_path): - # The write path must derive its tag from the same config template the read - # path uses, or an operator override leaves pushed and looked-up tags - # disagreeing and every lookup misses. - m_gwc.return_value = { - 'iib_index_db_artifact_registry': 'quay.io/iib', - 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', - 'iib_index_db_artifact_tag_template': 'cache-{digest}', - } - m_digest.return_value = 'b' * 64 +def test_push_honors_helper_defined_artifact_refs( + m_exists, m_state, m_request_ref, m_current_ref, m_push, tmp_path +): db = tmp_path / 'index.db' db.write_text('x') + output_image = 'quay.io/ns/foo@sha256:' + 'b' * 64 + m_request_ref.return_value = 'quay.io/iib/index-db:cache-' + 'b' * 64 + '-9' + m_current_ref.return_value = 'quay.io/iib/index-db:cache-' + 'b' * 64 push_index_db_artifact( request_id=9, from_index='quay.io/ns/foo:v4.17', index_db_path=str(db), operators=[], - output_image='quay.io/ns/foo@sha256:' + 'b' * 64, + output_image=output_image, overwrite_from_index=True, request_type='add', ) + m_request_ref.assert_called_once_with(output_image, 9) + m_current_ref.assert_called_once_with(output_image) pushed_refs = {c.kwargs['artifact_ref'] for c in m_push.call_args_list} assert pushed_refs == { 'quay.io/iib/index-db:cache-' + 'b' * 64, @@ -694,6 +970,30 @@ def test_cleanup_on_failure_has_no_rollback_param(): assert 'original_index_db_digest' not in params +@patch('iib.workers.tasks.containerized_utils.close_mr') +def test_cleanup_merge_request_failure_is_best_effort_by_default(mock_close_mr): + """Failure cleanup keeps the original error authoritative.""" + mock_close_mr.side_effect = IIBError('GitLab unavailable') + + cu.cleanup_merge_request_if_exists( + {'mr_id': '12', 'mr_url': 'https://gitlab.example.com/mr/12'}, + 'https://gitlab.example.com/project', + ) + + +@patch('iib.workers.tasks.containerized_utils.close_mr') +def test_cleanup_merge_request_failure_propagates_in_strict_mode(mock_close_mr): + """Successful throw-away builds fail when their MR cannot be closed.""" + mock_close_mr.side_effect = IIBError('GitLab unavailable') + + with pytest.raises(IIBError, match='GitLab unavailable'): + cu.cleanup_merge_request_if_exists( + {'mr_id': '12', 'mr_url': 'https://gitlab.example.com/mr/12'}, + 'https://gitlab.example.com/project', + raise_on_error=True, + ) + + @patch('iib.workers.tasks.containerized_utils.skopeo_inspect') def test_validate_bundles_in_parallel_success_single_bundle(mock_skopeo_inspect): """Test validate_bundles_in_parallel with a single bundle successfully.""" @@ -1396,85 +1696,85 @@ def test_merge_mr_after_build_failure_closes_mr(mock_merge_mr, mock_close_mr): mock_close_mr.assert_called_once_with(mr_details, 'https://gitlab.example.com/project') -@patch('iib.workers.tasks.containerized_utils.get_image_label') -@patch('iib.workers.tasks.containerized_utils.extract_files_from_image_non_privileged') -def test_extract_catalog_and_db_prefers_hidden_db(mock_extract, mock_label, tmp_path): - """When a hidden index.db exists, it is preferred over the labeled db.""" +@mock.patch('iib.workers.tasks.containerized_utils._reject_escaping_symlinks') +@mock.patch('iib.workers.tasks.containerized_utils.extract_files_from_image_non_privileged') +@mock.patch('iib.workers.tasks.containerized_utils.get_image_label') +def test_extract_catalog_from_image(mock_label, mock_extract, mock_reject, tmp_path): + image = 'registry.internal/iib-build:42@sha256:abc' + mock_label.return_value = '/configs' + result = extract_catalog_from_image(image, str(tmp_path)) + assert result == str(tmp_path / 'extracted_configs') + mock_extract.assert_called_once_with(image, '/configs', result) + mock_reject.assert_called_once_with(result, image, 'FBC configs directory') - def label_side_effect(image, label): - return { - 'operators.operatorframework.io.index.configs.v1': '/configs', - 'operators.operatorframework.io.index.database.v1': '/database/index.db', - }[label] - mock_label.side_effect = label_side_effect +@mock.patch('iib.workers.tasks.containerized_utils._reject_escaping_symlinks') +@mock.patch('iib.workers.tasks.containerized_utils.extract_files_from_image_non_privileged') +@mock.patch('iib.workers.tasks.containerized_utils.get_image_label') +def test_extract_catalog_from_image_accepts_declared_empty_catalog( + mock_label, mock_extract, mock_reject, tmp_path +): + mock_label.return_value = '/configs' + mock_extract.side_effect = FileNotFoundInImageError('empty') + result = extract_catalog_from_image('registry.internal/iib-build:42@sha256:abc', str(tmp_path)) + assert Path(result).is_dir() + mock_reject.assert_called_once() + + +@mock.patch('iib.workers.tasks.containerized_utils.get_image_label') +def test_extract_catalog_from_image_rejects_non_fbc_image(mock_label, tmp_path): + mock_label.return_value = None + with pytest.raises(IIBError, match='does not contain a file-based catalog'): + extract_catalog_from_image('registry/image@sha256:abc', str(tmp_path)) + +@patch( + 'iib.workers.tasks.containerized_utils.extract_catalog_from_image', + return_value='/tmp/extracted-configs', +) +@patch('iib.workers.tasks.containerized_utils.extract_files_from_image_non_privileged') +def test_extract_catalog_and_db_prefers_hidden_db(mock_extract, mock_catalog, tmp_path): + """When a hidden index.db exists, it is preferred over the labeled db.""" configs_dir, index_db = extract_catalog_and_db_from_image( 'quay.io/redhat/my-index:test', str(tmp_path) ) assert configs_dir.endswith('configs') assert index_db.endswith('index.db') - # Two extractions: configs dir and the hidden db file. - assert mock_extract.call_count == 2 + mock_catalog.assert_called_once_with('quay.io/redhat/my-index:test', str(tmp_path)) + # Only the hidden db is extracted here; the catalog helper owns config extraction. + mock_extract.assert_called_once() -@patch('iib.workers.tasks.containerized_utils.get_image_label') +@patch( + 'iib.workers.tasks.containerized_utils.extract_catalog_from_image', + return_value='/tmp/extracted-configs', +) @patch('iib.workers.tasks.containerized_utils.extract_files_from_image_non_privileged') -def test_extract_catalog_and_db_raises_when_no_hidden_db(mock_extract, mock_label, tmp_path): +def test_extract_catalog_and_db_raises_when_no_hidden_db(mock_extract, mock_catalog, tmp_path): """When the hidden db is absent, the request fails; there is no labeled-db or empty-db fallback. An image with no hidden index.db has not been onboarded to the containerized build flow, so the request must fail rather than degrade to a labeled db or a synthesised empty db. """ - mock_label.side_effect = lambda image, label: { - 'operators.operatorframework.io.index.configs.v1': '/configs', - 'operators.operatorframework.io.index.database.v1': '/database/index.db', - }[label] - # First call (configs) ok; second call (hidden db) raises FileNotFoundInImageError. - mock_extract.side_effect = [None, FileNotFoundInImageError('no hidden db')] + mock_extract.side_effect = FileNotFoundInImageError('no hidden db') with pytest.raises(IIBError, match='No index.db found in image'): extract_catalog_and_db_from_image('quay.io/redhat/my-index:test', str(tmp_path)) - # Only two extraction attempts: configs dir and the failed hidden db lookup. # The labeled database.v1 path is never read. - assert mock_extract.call_count == 2 - - -@patch('iib.workers.tasks.containerized_utils.get_image_label') -@patch('iib.workers.tasks.containerized_utils.extract_files_from_image_non_privileged') -def test_extract_catalog_and_db_empty_configs_uses_empty_dir(mock_extract, mock_label, tmp_path): - """A declared-but-empty /configs (empty index) yields an empty catalog, not a failure. - - 'oc image extract' cannot represent an empty directory, so extracting an empty - index's /configs raises FileNotFoundInImageError. Because the image declares a - configs label, this is treated as an empty catalog directory; the hidden db is - still extracted normally. - """ - mock_label.side_effect = lambda image, label: { - 'operators.operatorframework.io.index.configs.v1': '/configs', - }.get(label, '') - # First call (configs) reports nothing under the declared path; second call - # (hidden db) succeeds. - mock_extract.side_effect = [FileNotFoundInImageError('empty /configs'), None] - - configs_dir, index_db = extract_catalog_and_db_from_image( - 'quay.io/redhat/empty-index:test', str(tmp_path) - ) - - assert configs_dir.endswith('extracted_configs') - assert os.path.isdir(configs_dir) - assert os.listdir(configs_dir) == [] # empty catalog - assert index_db.endswith('index.db') - assert mock_extract.call_count == 2 + mock_catalog.assert_called_once_with('quay.io/redhat/my-index:test', str(tmp_path)) + mock_extract.assert_called_once() -@patch('iib.workers.tasks.containerized_utils.get_image_label') +@patch( + 'iib.workers.tasks.containerized_utils.extract_catalog_from_image', + return_value='/tmp/extracted-configs', +) @patch('iib.workers.tasks.containerized_utils.extract_files_from_image_non_privileged') def test_extract_catalog_and_db_propagates_real_extraction_error( - mock_extract, mock_label, tmp_path + mock_extract, mock_catalog, tmp_path ): """A genuine extraction failure (not a missing path) must propagate, not degrade. @@ -1482,26 +1782,12 @@ def test_extract_catalog_and_db_propagates_real_extraction_error( (registry/OCI/layer/tar failure) must not be silently swallowed as "no hidden db", which would build an image from an incomplete index.db. """ - mock_label.side_effect = lambda image, label: { - 'operators.operatorframework.io.index.configs.v1': '/configs', - 'operators.operatorframework.io.index.database.v1': '/database/index.db', - }[label] - # First call (configs) ok; second call (hidden db) raises a real error. - mock_extract.side_effect = [None, IIBError('registry unreachable')] + mock_extract.side_effect = IIBError('registry unreachable') with pytest.raises(IIBError, match='registry unreachable'): extract_catalog_and_db_from_image('quay.io/redhat/my-index:test', str(tmp_path)) -@patch('iib.workers.tasks.containerized_utils.get_image_label') -def test_extract_catalog_and_db_raises_without_configs_label(mock_label): - """If the image has no FBC configs label, an IIBError is raised.""" - mock_label.return_value = '' - - with pytest.raises(IIBError, match='does not contain a file-based catalog'): - extract_catalog_and_db_from_image('quay.io/redhat/my-index:test', '/tmp/does-not-matter') - - @mock.patch('iib.workers.tasks.containerized_utils.set_request_state') @mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') @mock.patch('iib.workers.tasks.containerized_utils.remote_branch_exists', return_value=True) @@ -1522,11 +1808,103 @@ def test_prepare_build_sources_normal( index_to_gitlab_push_map={'quay.io/redhat/my-index': 'https://gitlab/x.git'}, overwrite_from_index=True, ) - assert src.is_divergent is False + assert src.source_kind is cu.BuildSourceKind.STANDARD + assert src.merge_allowed assert src.target_branch == 'v4.14' assert src.index_db_path is None # normal path pulls from ORAS +def test_build_sources_merge_policy(): + common = { + 'index_git_repo': 'https://gitlab.example/repo.git', + 'local_git_repo_path': '/tmp/git/v4.17', + 'localized_git_catalog_path': '/tmp/git/v4.17/configs', + 'index_db_path': None, + 'target_branch': 'v4.17', + } + assert cu.BuildSources(**common, source_kind=cu.BuildSourceKind.STANDARD).merge_allowed + assert not cu.BuildSources(**common, source_kind=cu.BuildSourceKind.DIVERGENT).merge_allowed + assert not cu.BuildSources(**common, source_kind=cu.BuildSourceKind.CHAINED).merge_allowed + + +@pytest.mark.parametrize( + ('ancestor_from_index', 'branch_results', 'expected_branches'), + ( + ('quay.io/ns/index:v4.17', (True,), ('v4.17',)), + ('quay.io/ns/index:test', (False, True), ('test', 'v4.17')), + ), +) +@mock.patch('iib.workers.tasks.containerized_utils.set_request_state') +@mock.patch('iib.workers.tasks.containerized_utils.fetch_and_verify_request_index_db_artifact') +@mock.patch('iib.workers.tasks.containerized_utils.extract_catalog_from_image') +@mock.patch('iib.workers.tasks.containerized_utils.resolve_chained_build_source') +@mock.patch('iib.workers.tasks.containerized_utils.clone_git_repo') +@mock.patch('iib.workers.tasks.containerized_utils.remote_branch_exists') +@mock.patch('iib.workers.tasks.containerized_utils.get_git_token') +@mock.patch('iib.workers.tasks.containerized_utils.resolve_git_url') +def test_prepare_build_sources_chained_selects_ancestor_scaffolding( + mock_resolve_git, + mock_token, + mock_branch_exists, + mock_clone, + mock_resolve_chain, + mock_extract_catalog, + mock_fetch_db, + mock_set_state, + ancestor_from_index, + branch_results, + expected_branches, + tmp_path, +): + chained = ChainedBuildSource( + parent_request_id=42, + parent_index_image='registry.internal/iib-build:42', + parent_index_image_resolved='registry.internal/iib-build:42@sha256:abc', + original_from_index=ancestor_from_index, + ancestry=(42,), + ) + mock_resolve_chain.return_value = chained + mock_resolve_git.return_value = 'https://gitlab.example/repo.git' + mock_token.return_value = ('token-name', 'token-value') + if len(branch_results) == 1: + mock_branch_exists.return_value = branch_results[0] + else: + mock_branch_exists.side_effect = branch_results + + clone_catalog = tmp_path / 'git' / 'v4.17' / 'configs' + clone_catalog.mkdir(parents=True) + (clone_catalog / 'stale.yaml').write_text('stale') + parent_catalog = tmp_path / 'parent-configs' + (parent_catalog / 'operator').mkdir(parents=True) + (parent_catalog / 'operator' / 'catalog.json').write_text('{}') + mock_extract_catalog.return_value = str(parent_catalog) + mock_fetch_db.return_value = str(tmp_path / 'parent-index.db') + + result = cu.prepare_build_sources( + request_id=99, + from_index='registry.internal/iib-build:42', + from_index_resolved='registry.internal/iib-build:42@sha256:abc', + temp_dir=str(tmp_path), + ocp_version='v4.17', + index_to_gitlab_push_map={'quay.io/ns/index': 'https://gitlab.example/repo.git'}, + overwrite_from_index=False, + overwrite_from_index_token=None, + ) + + assert result.source_kind is cu.BuildSourceKind.CHAINED + assert result.target_branch == 'v4.17' + assert result.index_db_path == str(tmp_path / 'parent-index.db') + assert not (clone_catalog / 'stale.yaml').exists() + assert (clone_catalog / 'operator' / 'catalog.json').is_file() + mock_resolve_git.assert_called_once_with( + from_index=chained.original_from_index, + index_repo_map={'quay.io/ns/index': 'https://gitlab.example/repo.git'}, + ) + mock_extract_catalog.assert_called_once_with(chained.parent_index_image_resolved, str(tmp_path)) + mock_fetch_db.assert_called_once_with(chained.parent_index_image_resolved, 42, str(tmp_path)) + assert tuple(call.args[1] for call in mock_branch_exists.call_args_list) == expected_branches + + @mock.patch('iib.workers.tasks.containerized_utils.set_request_state') @mock.patch('iib.workers.tasks.containerized_utils.get_git_token', return_value=('n', 't')) @mock.patch('iib.workers.tasks.containerized_utils.remote_branch_exists', return_value=False) @@ -1576,7 +1954,8 @@ def test_prepare_build_sources_divergent_extracts( index_to_gitlab_push_map={'quay.io/redhat/my-index': 'https://gitlab/x.git'}, overwrite_from_index=False, ) - assert src.is_divergent is True + assert src.source_kind is cu.BuildSourceKind.DIVERGENT + assert not src.merge_allowed assert src.target_branch == 'v4.14' assert src.index_db_path == str(tmp_path / 'ex.db') # Divergent extraction must read the resolved digest, not the mutable tag, so diff --git a/tests/test_workers/test_tasks/test_oras_utils.py b/tests/test_workers/test_tasks/test_oras_utils.py index c77a52cf1..718ffe29a 100644 --- a/tests/test_workers/test_tasks/test_oras_utils.py +++ b/tests/test_workers/test_tasks/test_oras_utils.py @@ -16,6 +16,7 @@ _get_index_digest, _get_content_addressed_artifact_tag, get_indexdb_artifact_pullspec, + get_request_indexdb_artifact_pullspec, get_index_tag, ) @@ -764,6 +765,20 @@ def test_get_indexdb_artifact_pullspec(mock_digest, mock_gwc, from_index, digest mock_digest.assert_called_once_with(from_index) +@mock.patch('iib.workers.tasks.oras_utils._get_content_addressed_artifact_tag') +@mock.patch('iib.workers.tasks.oras_utils.get_worker_config') +def test_get_request_indexdb_artifact_pullspec(mock_config, mock_tag): + mock_config.return_value = { + 'iib_index_db_artifact_registry': 'quay.io/iib', + 'iib_index_db_artifact_template': '{registry}/index-db:{tag}', + } + mock_tag.return_value = 'idb-' + 'a' * 64 + result = get_request_indexdb_artifact_pullspec( + 'registry.internal/iib-build:42@sha256:' + 'a' * 64, 42 + ) + assert result == 'quay.io/iib/index-db:idb-' + 'a' * 64 + '-42' + + @mock.patch('iib.workers.tasks.oras_utils.get_worker_config') @mock.patch('iib.workers.tasks.oras_utils.get_image_digest') def test_get_indexdb_artifact_pullspec_digest_resolution_failure(mock_digest, mock_gwc):