Skip to content

feat: support chaining containerized index builds #1394

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yashvardhannanavati wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: support chaining containerized index builds #1394

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions docker/containerized/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,24 @@ If a request targets a tag with no corresponding Git branch — for example a ti

This lets IIB build and validate a one-off tag without requiring per-tag branch/Component provisioning.

### Chaining IIB Build Outputs

Containerized `add`, `rm`, and `fbc-operations` requests may use the completed
`index_image` from an earlier request as `from_index`. IIB resolves request
ancestry to locate the original onboarded Git repository and its Konflux
scaffolding, but reconstructs the immediate parent's state from:

- FBC configs embedded in the parent's resolved output image; and
- `index-db:idb-<parent-output-digest>-<parent-request-id>` in Quay.

Chained requests are always throw-away. Supplying `overwrite_from_index` or
`overwrite_from_index_token` for an IIB output image fails the request because
users cannot overwrite IIB's output registry. The Konflux MR is always closed
and never merged.

Chaining is not supported for `merge-index-image`, `create-empty-index`,
`regenerate-bundle`, or legacy worker requests.

## Index DB Artifact and ImageStream Tag Naming

Cached `index.db` artifact tags (ORAS) and ImageStream tags are keyed on the index image's content (manifest) digest — `idb-<sha256>` — resolved via `skopeo inspect`, not on its pullspec. Because the key is the content itself, it is both namespace-safe and promotion-safe:
Expand Down
5 changes: 4 additions & 1 deletion iib/workers/tasks/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ def _update_index_image_pull_spec(
is_image_fbc: bool = False,
index_repo_map: Optional[Dict[str, str]] = None,
rm_operators: Optional[List[str]] = None,
) -> None:
) -> Optional[str]:
"""
Update the request with the modified index image.

Expand All @@ -283,6 +283,8 @@ def _update_index_image_pull_spec(
required if ``is_image_fbc`` is ``True``.
:param list(str) rm_operators: List of operator package names to remove from the Git
catalog during overwrite. Used by RM requests and ADD requests with deprecations.
:return: The recorded index_image_resolved when add_or_rm is True, otherwise None.
:rtype: Optional[str]
:raises IIBError: if the manifest list couldn't be created and pushed
"""
conf = get_worker_config()
Expand Down Expand Up @@ -322,6 +324,7 @@ def _update_index_image_pull_spec(
payload['internal_index_image_copy_resolved'] = get_resolved_image(output_pull_spec)

update_request(request_id, payload, exc_msg='Failed setting the index image on the request')
return payload.get('index_image_resolved')


def _get_external_arch_pull_spec(
Expand Down
18 changes: 9 additions & 9 deletions iib/workers/tasks/build_containerized_add.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ def handle_containerized_add_request(
ocp_version=prebuild_info['ocp_version'],
index_to_gitlab_push_map=index_to_gitlab_push_map,
overwrite_from_index=overwrite_from_index,
overwrite_from_index_token=overwrite_from_index_token,
)
index_git_repo = sources.index_git_repo
local_git_repo_path = sources.local_git_repo_path
Expand Down Expand Up @@ -335,7 +336,7 @@ def handle_containerized_add_request(
"This should not happen if the pipeline completed successfully."
)

_update_index_image_pull_spec(
index_image_resolved = _update_index_image_pull_spec(
output_pull_spec=output_pull_spec,
request_id=request_id,
arches=arches,
Expand All @@ -351,6 +352,8 @@ def handle_containerized_add_request(
# a Konflux pipelinerun. So the old workflow isn't needed.
index_repo_map={},
)
if index_image_resolved is None:
raise IIBError(f'request {request_id} has no resolved index image.')

# Push updated index.db before merging the MR so that on failure both
# git and the index.db artifact remain consistent (MR stays open,
Expand All @@ -360,7 +363,7 @@ def handle_containerized_add_request(
from_index=str(from_index),
index_db_path=artifact_index_db_file,
operators=operators,
output_image=image_url,
output_image=index_image_resolved,
overwrite_from_index=overwrite_from_index,
request_type='add',
)
Expand All @@ -369,17 +372,14 @@ def handle_containerized_add_request(
# (replication, metadata, index.db push) have succeeded before git
# is advanced. This prevents git/index.db divergence on partial failure.
#
# The `not sources.is_divergent` half is defense-in-depth: prepare_build_sources
# already rejects overwrite_from_index on the divergent path, so this cannot be
# reached with both set. It stays because a divergent build reuses the base OCP
# branch's Konflux Component -- merging its MR would publish one tag's content
# onto the shared branch.
if overwrite_from_index and not sources.is_divergent:
# Only standard sources may publish their MR. Divergent and chained builds
# reuse shared Git/Konflux scaffolding and must remain throw-away.
if overwrite_from_index and sources.merge_allowed:
Comment thread
qodo-for-releng[bot] marked this conversation as resolved.
merge_mr_after_build(mr_details, index_git_repo)
# Prevent cleanup_on_failure from trying to close an already-merged MR
mr_details = None
else:
cleanup_merge_request_if_exists(mr_details, index_git_repo)
cleanup_merge_request_if_exists(mr_details, index_git_repo, raise_on_error=True)

set_request_state(
request_id,
Expand Down
18 changes: 9 additions & 9 deletions iib/workers/tasks/build_containerized_fbc_operations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ def handle_containerized_fbc_operation_request(
ocp_version=prebuild_info['ocp_version'],
index_to_gitlab_push_map=index_to_gitlab_push_map,
overwrite_from_index=overwrite_from_index,
overwrite_from_index_token=overwrite_from_index_token,
)
index_git_repo = sources.index_git_repo
local_git_repo_path = sources.local_git_repo_path
Expand Down Expand Up @@ -228,7 +229,7 @@ def handle_containerized_fbc_operation_request(
"This should not happen if the pipeline completed successfully."
)

_update_index_image_pull_spec(
index_image_resolved = _update_index_image_pull_spec(
output_pull_spec=output_pull_spec,
request_id=request_id,
arches=arches,
Expand All @@ -244,6 +245,8 @@ def handle_containerized_fbc_operation_request(
# a Konflux pipelinerun. So the old workflow isn't needed.
index_repo_map={},
)
if index_image_resolved is None:
raise IIBError(f'request {request_id} has no resolved index image.')

# Push updated index.db before merging the MR so that on failure both
# git and the index.db artifact remain consistent (MR stays open,
Expand All @@ -253,7 +256,7 @@ def handle_containerized_fbc_operation_request(
from_index=from_index,
index_db_path=index_db_path,
operators=operators_in_db,
output_image=image_url,
output_image=index_image_resolved,
overwrite_from_index=overwrite_from_index,
request_type='fbc_operations',
)
Expand All @@ -262,17 +265,14 @@ def handle_containerized_fbc_operation_request(
# (replication, metadata, index.db push) have succeeded before git
# is advanced. This prevents git/index.db divergence on partial failure.
#
# The `not sources.is_divergent` half is defense-in-depth: prepare_build_sources
# already rejects overwrite_from_index on the divergent path, so this cannot be
# reached with both set. It stays because a divergent build reuses the base OCP
# branch's Konflux Component -- merging its MR would publish one tag's content
# onto the shared branch.
if overwrite_from_index and not sources.is_divergent:
# Only standard sources may publish their MR. Divergent and chained builds
# reuse shared Git/Konflux scaffolding and must remain throw-away.
if overwrite_from_index and sources.merge_allowed:
merge_mr_after_build(mr_details, index_git_repo)
# Prevent cleanup_on_failure from trying to close an already-merged MR
mr_details = None
else:
cleanup_merge_request_if_exists(mr_details, index_git_repo)
cleanup_merge_request_if_exists(mr_details, index_git_repo, raise_on_error=True)

# Summarize every action the request took. The per-step messages (extracting
# fragments, removing operators from index.db, adding packages) are reported
Expand Down
18 changes: 9 additions & 9 deletions iib/workers/tasks/build_containerized_rm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ def handle_containerized_rm_request(
ocp_version=ocp_version,
index_to_gitlab_push_map=index_to_gitlab_push_map or {},
overwrite_from_index=overwrite_from_index,
overwrite_from_index_token=overwrite_from_index_token,
)
index_git_repo = sources.index_git_repo
local_git_repo_path = sources.local_git_repo_path
Expand Down Expand Up @@ -272,7 +273,7 @@ def handle_containerized_rm_request(

# Send an empty index_repo_map because the Git repository is already
# updated with the changes
_update_index_image_pull_spec(
index_image_resolved = _update_index_image_pull_spec(
output_pull_spec=output_pull_spec,
request_id=request_id,
arches=arches,
Expand All @@ -289,6 +290,8 @@ def handle_containerized_rm_request(
index_repo_map={},
rm_operators=operators,
)
if index_image_resolved is None:
raise IIBError(f'request {request_id} has no resolved index image.')

# Push updated index.db before merging the MR so that on failure both
# git and the index.db artifact remain consistent (MR stays open,
Expand All @@ -298,7 +301,7 @@ def handle_containerized_rm_request(
from_index=from_index,
index_db_path=index_db_path,
operators=operators,
output_image=image_url,
output_image=index_image_resolved,
overwrite_from_index=overwrite_from_index,
request_type='rm',
)
Expand All @@ -307,17 +310,14 @@ def handle_containerized_rm_request(
# (replication, metadata, index.db push) have succeeded before git
# is advanced. This prevents git/index.db divergence on partial failure.
#
# The `not sources.is_divergent` half is defense-in-depth: prepare_build_sources
# already rejects overwrite_from_index on the divergent path, so this cannot be
# reached with both set. It stays because a divergent build reuses the base OCP
# branch's Konflux Component -- merging its MR would publish one tag's content
# onto the shared branch.
if overwrite_from_index and not sources.is_divergent:
# Only standard sources may publish their MR. Divergent and chained builds
# reuse shared Git/Konflux scaffolding and must remain throw-away.
if overwrite_from_index and sources.merge_allowed:
merge_mr_after_build(mr_details, index_git_repo)
# Prevent cleanup_on_failure from trying to close an already-merged MR
mr_details = None
else:
cleanup_merge_request_if_exists(mr_details, index_git_repo)
cleanup_merge_request_if_exists(mr_details, index_git_repo, raise_on_error=True)

operators_str = ', '.join(operators)
set_request_state(
Expand Down
Loading
Loading