From a9cb43126f0e45d10323754865f858720876fb74 Mon Sep 17 00:00:00 2001 From: Stephen Gallagher Date: Sep 13 2024 17:22:20 +0000 Subject: Add determine_rhel_state() and does_branch_exist() These replace determine_active_y_version and does_divergent_branch_exist() The return value from determine_rhel_state checks for both Product Pages state and the presence of the prior release branch to return a complete set of information about how the current cXs branch will behave. This patch updates the cli.request_current_state() and cli._build() functions to use these two new utility features. It also refactors those two functions to reuse code better. Signed-off-by: Stephen Gallagher --- diff --git a/src/centpkg/cli.py b/src/centpkg/cli.py index c654c8c..5225c29 100755 --- a/src/centpkg/cli.py +++ b/src/centpkg/cli.py @@ -157,55 +157,35 @@ class centpkgClient(cliClient): # FIXME: This is alot of duplication. # Get it working now. De-duplicate later def request_current_state(self): - # Only run if we have internal configuraions, or scratch + # Only run if we have internal configurations, or scratch internal_config_file = "/etc/rpkg/centpkg_internal.conf" if os.path.exists(internal_config_file): # Get our internal only variables cfg = ConfigParser() cfg.read(internal_config_file) pp_api_url = config_get_safely(cfg, "centpkg.internal", "pp_api_url") - gitbz_query_url = config_get_safely( - cfg, "centpkg.internal", "gitbz_query_url" - ) rhel_dist_git = config_get_safely(cfg, "centpkg.internal", "rhel_dist_git") - # Find out divergent branch and stabalization - self.log.info("Checking current state:") + # Find out divergent branch and stabilization + self.log.info("Checking current state...") stream_version = self.cmd.target.split("-")[0] rhel_version = centpkg.utils.stream_mapping(stream_version) + try: - x_version, active_y, is_beta, in_stabilization = ( - centpkg.utils.determine_active_y_version(rhel_version, pp_api_url) + rhel_state = centpkg.utils.determine_rhel_state( + rhel_dist_git=rhel_dist_git, + namespace=self.cmd.ns, + repo_name=self.cmd.repo_name, + cs_branch=stream_version, + pp_api_url=pp_api_url, ) - except AssertionError as e: - self.log.error( - " Error: centpkg cannot determine the development phase." - ) - self.log.error( - " Please file an issue at https://git.centos.org/centos/centpkg" - ) - self.log.error(" Workaround: Use the --rhel-target option") - self.log.error("Exiting") + except centpkg.utils.RHELError as e: + self.log.error("Could not determine RHEL state of this package") + self.log.error(str(e)) raise SystemExit(1) - if is_beta: - # Special case: for X.0 betas, there will never be a prior branch - # In this case, always work on the active branch. - divergent_branch = True - else: - divergent_branch = centpkg.utils.does_divergent_branch_exist( - self.cmd.repo_name, x_version, active_y, rhel_dist_git, "rpms" - ) - # Good to know - if in_stabilization: - self.log.info(" we are in stabilization mode.") - else: - self.log.info(" we are not in stabilization mode.") - if divergent_branch and not is_beta: - self.log.info(" a divergent branch was found.") - elif divergent_branch and is_beta: - self.log.info(" we are working on a beta release.") - else: - self.log.info(" a divergent branch was not found.") + + # Output the current state + self.log.info(centpkg.utils.format_current_state_message(rhel_state)) else: self.log.error("NO INTERNAL CONFIGURATION") self.log.error( @@ -214,6 +194,27 @@ class centpkgClient(cliClient): self.log.error("Exiting") raise SystemExit(1) + def get_custom_metadata(self): + # If custom-user-metadata set, add onto it + if ( + hasattr(self.args, "custom_user_metadata") + and self.args.custom_user_metadata + ): + try: + temp_custom_user_metadata = json.loads(self.args.custom_user_metadata) + # Use ValueError instead of json.JSONDecodeError for Python 2 and 3 compatibility + except ValueError as e: + self.parser.error("--custom-user-metadata is not valid JSON: %s" % e) + if not isinstance(temp_custom_user_metadata, dict): + self.parser.error("--custom-user-metadata must be a JSON object") + else: + temp_custom_user_metadata = dict() + + return temp_custom_user_metadata + + def update_custom_metadata(self, metadata): + self.args.custom_user_metadata = json.dumps(metadata) + # Overloaded build def register_build(self): # Do all the work from the super class @@ -249,7 +250,6 @@ class centpkgClient(cliClient): # Only do rhel-target if we are centpkg, not centpkg-sig, or if we are doing scratch if not os.path.basename(sys.argv[0]).endswith("-sig") and not self.args.scratch: - # Only run if we have internal configuraions internal_config_file = "/etc/rpkg/centpkg_internal.conf" if os.path.exists(internal_config_file): @@ -258,34 +258,9 @@ class centpkgClient(cliClient): if hasattr(self.args, "rhel_target") and self.args.rhel_target: # If rhel-target set to none, do nothing with metadata if self.args.rhel_target.lower() != "none": - # If custom-user-metadata set, add onto it - if ( - hasattr(self.args, "custom_user_metadata") - and self.args.custom_user_metadata - ): - try: - temp_custom_user_metadata = json.loads( - self.args.custom_user_metadata - ) - # Use ValueError instead of json.JSONDecodeError for Python 2 and 3 compatibility - except ValueError as e: - self.parser.error( - "--custom-user-metadata is not valid JSON: %s" % e - ) - if not isinstance(temp_custom_user_metadata, dict): - self.parser.error( - "--custom-user-metadata must be a JSON object" - ) - temp_custom_user_metadata["rhel-target"] = ( - self.args.rhel_target - ) - else: - temp_custom_user_metadata = { - "rhel-target": self.args.rhel_target - } - self.args.custom_user_metadata = json.dumps( - temp_custom_user_metadata - ) + temp_custom_user_metadata = self.get_custom_metadata() + temp_custom_user_metadata["rhel-target"] = self.args.rhel_target + self.update_custom_metadata(temp_custom_user_metadata) else: # Get our internal only variables @@ -294,9 +269,6 @@ class centpkgClient(cliClient): pp_api_url = config_get_safely( cfg, "centpkg.internal", "pp_api_url" ) - gitbz_query_url = config_get_safely( - cfg, "centpkg.internal", "gitbz_query_url" - ) rhel_dist_git = config_get_safely( cfg, "centpkg.internal", "rhel_dist_git" ) @@ -307,12 +279,19 @@ class centpkgClient(cliClient): rhel_version = centpkg.utils.stream_mapping(stream_version) try: - x_version, active_y, is_beta, in_stabilization = ( - centpkg.utils.determine_active_y_version( - rhel_version, pp_api_url - ) + rhel_state = centpkg.utils.determine_rhel_state( + rhel_dist_git=rhel_dist_git, + namespace=self.cmd.ns, + repo_name=self.cmd.repo_name, + cs_branch=stream_version, + pp_api_url=pp_api_url, ) - except AssertionError as e: + except centpkg.utils.RHELError as e: + self.log.error("Could not determine RHEL state of this package") + self.log.error(str(e)) + raise SystemExit(1) + + except: self.log.error( " Error: centpkg cannot determine the development phase." ) @@ -323,79 +302,31 @@ class centpkgClient(cliClient): self.log.error("Exiting") raise SystemExit(1) - if is_beta: - # Special case: for X.0 betas, there will never be a prior branch - # In this case, always work on the active branch. - divergent_branch = True - else: - divergent_branch = centpkg.utils.does_divergent_branch_exist( - self.cmd.repo_name, - x_version, - active_y, - rhel_dist_git, - "rpms", - ) - - # Good to know - if divergent_branch and not is_beta: - self.log.info(" a divergent branch was found.") - elif divergent_branch and is_beta: - self.log.info(" we are working on a beta release.") - else: - self.log.info(" a divergent branch was not found.") - if in_stabilization: - self.log.info(" we are in stabilization mode.") - else: - self.log.info(" we are not in stabilization mode.") + # Output the current state + self.log.info( + centpkg.utils.format_current_state_message(rhel_state) + ) # Update args.custom_user_metadata + temp_custom_user_metadata = self.get_custom_metadata() + if ( - hasattr(self.args, "custom_user_metadata") - and self.args.custom_user_metadata + rhel_state.phase == centpkg.utils.pp_phase_stabilization + or not rhel_state.rhel_target_default ): - try: - temp_custom_user_metadata = json.loads( - self.args.custom_user_metadata - ) - # Use ValueError instead of json.JSONDecodeError for Python 2 and 3 compatibility - except ValueError as e: - self.parser.error( - "--custom-user-metadata is not valid JSON: %s" % e - ) - if not isinstance(temp_custom_user_metadata, dict): - self.parser.error( - "--custom-user-metadata must be a JSON object" - ) - if divergent_branch: - temp_custom_user_metadata["rhel-target"] = "latest" - elif not divergent_branch and not in_stabilization: - temp_custom_user_metadata["rhel-target"] = "zstream" - else: - self.log.error("We are currently in Stabalization mode") - self.log.error( - "You must either set the rhel-target (--rhel-target)" - ) - self.log.error("or branch for the previous version.") - self.log.error("Exiting") - raise SystemExit(1) - self.args.custom_user_metadata = json.dumps( - temp_custom_user_metadata - ) - else: - if divergent_branch: - self.args.custom_user_metadata = '{"rhel-target": "latest"}' - elif not divergent_branch and not in_stabilization: - self.args.custom_user_metadata = ( - '{"rhel-target": "zstream"}' - ) - else: - self.log.error("We are currently in Stabalization mode") - self.log.error( - "You must either set the rhel-target (--rhel-target)" - ) - self.log.error("or branch for the previous version.") - self.log.error("Exiting") - raise SystemExit(1) + # It shouldn't be possible for rhel_target_default + # to be None except during Stabilization phase, + # but we'll check anyway. + self.log.error("We are currently in Stabilization phase") + self.log.error("You must set the rhel-target (--rhel-target)") + self.log.error("to 'zstream' (for 0day) or 'exception'.") + self.log.error("Exiting") + raise SystemExit(1) + + temp_custom_user_metadata["rhel-target"] = ( + rhel_state.rhel_target_default + ) + self.update_custom_metadata(temp_custom_user_metadata) # Good to know self.log.info( diff --git a/src/centpkg/utils.py b/src/centpkg/utils.py index fd7c9eb..8ef80bc 100644 --- a/src/centpkg/utils.py +++ b/src/centpkg/utils.py @@ -17,6 +17,7 @@ import os import re import requests import sys +from collections import namedtuple from datetime import date, datetime from http import HTTPStatus from pyrpkg import rpkgError @@ -28,6 +29,40 @@ import git as gitpython dist_git_config = None +# RHEL Product Pages Phase Identifiers +pp_phase_name_lookup = dict() +# Phase 230 is "Planning / Development / Testing" (AKA DevTestDoc) +pp_phase_devtestdoc = 230 +pp_phase_name_lookup[pp_phase_devtestdoc] = "DevTestDoc" + +# Phase 450 is "Stabilization" (AKA Exception Phase) +pp_phase_stabilization = 450 +pp_phase_name_lookup[pp_phase_stabilization] = "Stabilization" + +# Phase 600 is "Maintenance" (AKA Z-stream Phase) +pp_phase_maintenance = 600 +pp_phase_name_lookup[pp_phase_maintenance] = "Maintenance" + +rhel_state_nt = namedtuple( + "RHELState", + [ + "latest_version", + "target_version", + "rule_branch", + "phase", + "rhel_target_default", + "enforcing", + ], +) + + +# Super-class for errors related to internal RHEL infrastructure +class RHELError(Exception): + pass + + +logger = logging.getLogger(__name__) + def do_fork(logger, base_url, token, repo_name, namespace, cli_name): """ @@ -312,43 +347,27 @@ def stream_mapping(csname): Returns ------- str - Correspoinding RHEL name. + Corresponding RHEL name. """ if csname == "c8s" or csname == "cs8": - return "rhel-8" + return 8, "rhel-8" if csname == "c9s" or csname == "cs9": - return "rhel-9" + return 9, "rhel-9" if csname == "c10s" or csname == "cs10": - return "rhel-10" + return 10, "rhel-10" if csname == "c11s" or csname == "cs11": - return "rhel-11" + return 11, "rhel-11" return None -def does_divergent_branch_exist( - repo_name, x_version, active_y, rhel_dist_git, namespace -): - logger = logging.getLogger(__name__) - +def does_branch_exist(rhel_dist_git, namespace, repo_name, branch): # Determine if the Y-1 branch exists for this repo - - if x_version >= 10 and active_y <= 0: - # For 10.0 and later X.0 releases, check for a rhel-X.0-beta branch - divergent_branch = "rhel-{}.0-beta".format(x_version) - elif x_version <= 9: - divergent_branch = "rhel-{}.{}.0".format(x_version, active_y - 1) - else: - # Starting with RHEL 10, the branch names have dropped the extra .0 - divergent_branch = "rhel-{}.{}".format(x_version, active_y - 1) - - logger.debug("Divergent branch: {}".format(divergent_branch)) - g = gitpython.cmd.Git() try: g.ls_remote( "--exit-code", os.path.join(rhel_dist_git, namespace, repo_name), - divergent_branch, + branch, ) branch_exists = True except gitpython.GitCommandError as e: @@ -381,30 +400,28 @@ def parse_rhel_shortname(shortname): return major_version, minor_version, extra_version -def determine_active_y_version(rhel_version, api_url): - """ - Returns: A 4-tuple containing: - 0. The major release version(int) - 1. The active Y-stream version(int) - 2. Whether the active release is the pre-X.0 beta - 3. Whether we are in the Exception Phase(bool) - """ - logger = logging.getLogger(__name__) +def parse_rhel_branchname(shortname): + # The branchname is in the form rhel-9-1.0 or rhel-10.0[-beta] + m = re.match( + "rhel-(?P[0-9]+)[.-](?P[0-9]+)([.]0|[-](?P.*))?", shortname + ) + if not m: + raise RuntimeError("Could not parse version from {}".format(shortname)) + + major_version = int(m.group("major")) + minor_version = int(m.group("minor")) + extra_version = m.group("extra") or None - # Phase Identifiers - # Phase 230 is "Planning / Development / Testing" (AKA DevTestDoc) - # Phase 450 is "Stabilization" - phase_devtestdoc = 230 - phase_stabilization = 450 + return major_version, minor_version, extra_version - # Query the "package pages" API for the current active Y-stream release - request_params = { - "phase__in": "{},{}".format(phase_devtestdoc, phase_stabilization), - "product__shortname": "rhel", - "relgroup__shortname": rhel_version, - "format": "json", - } +def query_package_pages(api_url, request_params): + """ + api_url: A URL to the API endpoing of the Product Pages (e.g. + "https://example.com/pp/api/") + request_params: A set of python-requests-compatible URL parameters to + focus the query. + """ res = requests.get( os.path.join(api_url, "latest", "releases"), params=request_params, @@ -413,37 +430,241 @@ def determine_active_y_version(rhel_version, api_url): res.raise_for_status() payload = json.loads(res.text) logger.debug("Response from PP API: {}".format(json.dumps(payload, indent=2))) - if len(payload) < 1: + + return payload + + +def format_branch(x_version, y_version, is_beta): + if x_version <= 9: + # 9.x and older releases include an excess .0 in the branch name + if is_beta: + branch = "rhel-{}.{}.0-beta".format(x_version, y_version) + else: + branch = "rhel-{}.{}.0".format(x_version, y_version) + else: + # Starting with RHEL 10, the branch names have dropped the extra .0 + if is_beta: + branch = "rhel-{}.{}-beta".format(x_version, y_version) + else: + branch = "rhel-{}.{}".format(x_version, y_version) + return branch + + +def determine_rhel_state(rhel_dist_git, namespace, repo_name, cs_branch, pp_api_url): + """ + Arguments: + * rhel_dist_git: an https URL to the RHEL dist-git. Used for determining + the presence of the prior release's Z-stream branch. + * namespace: The dist-git namespace (rpms, containers, modules, etc.). + Used for determining the presence of the prior release's Z-stream + branch. + * repo_name: The name of the repo in the namespace from which we will + determine status. Used for determining the presence of the prior + release's Z-stream branch. + * cs_branch: The CentOS Stream branch for this repo. Used to determine the + RHEL major release. + * pp_api_url: The URL to the RHEL Product Pages API. Used for determining + the current development phase. + + Returns: a namedtuple containing key information about the RHEL release + associated with this CentOS Stream branch. It has the following members: + + * latest_version: The most recent major and minor release of RHEL. This + is a presentation string and its format is not guaranteed. + * target_version: The major and minor release of RHEL that is currently + targeted by this CentOS Stream branch. This is a presentation string + and its format is not guaranteed. + * rule_branch: The branch to be used for check-tickets rules (str) + * rhel_target_default: The default `--rhel-target` (str) or + None (NoneType). The possible values if not None are "latest" or + "zstream". + * enforcing: Whether ticket approvals should be enforced. (bool) + """ + + x_version, rhel_version = stream_mapping(cs_branch) + + # Query the "package pages" API for the current active Y-stream release + request_params = { + "phase__in": "{},{}".format( + pp_phase_devtestdoc, pp_phase_stabilization, pp_phase_maintenance + ), + "product__shortname": "rhel", + "relgroup__shortname": rhel_version, + "format": "json", + } + + try: + pp_response = query_package_pages( + api_url=pp_api_url, request_params=request_params + ) + except (ConnectionError, HTTPError) as e: + raise RHELError("Could not contact Product Pages. Are you on the VPN?") + + if len(pp_response) < 1: # Received zero potential release matches logger.warning("Didn't match any active releases. Assuming pre-Beta.") # Fake up a Beta payload - payload = [ + pp_response = [ { - "shortname": "{}.0.beta".format(rhel_version), - "phase": phase_devtestdoc, + "shortname": "{}.0-beta".format(rhel_version), + "phase": pp_phase_devtestdoc, } ] active_y_version = -1 beta = False - for entry in payload: + phase_lookup = dict() + for entry in pp_response: shortname = entry["shortname"] # The shortname is in the form rhel-9-1.0 or rhel-10.0[.beta] # Extract the active Y-stream version x_version, y_version, extra_version = parse_rhel_shortname(shortname) - - if y_version > active_y_version: + entry_is_beta = bool(extra_version and "beta" in extra_version) + + # Enable looking up the phase later + branch_name = format_branch(x_version, y_version, entry_is_beta) + phase_lookup[branch_name] = entry["phase"] + + if y_version > active_y_version or ( + y_version == active_y_version and beta and not entry_is_beta + ): + # Replace the saved values with a higher Y version if we + # see one. Also check whether we have the same Y version + # but without the Beta indicator active_y_version = y_version - beta = bool(extra_version and "beta" in extra_version) + beta = entry_is_beta + + if beta: + latest_version = "{}.{} Beta".format(x_version, active_y_version) + else: + latest_version = "{}.{}".format(x_version, active_y_version) + + logger.debug("Latest version: {}".format(latest_version)) + + # Next we need to find out if we're actually USING the latest version or + # the previous one, by asking RHEL dist-git if the rhel-X.(Y-1).0 branch + # exists. (Or rhel-X.Y.0-beta in the special case of Y=0) + + # If the latest release is the Beta, we can skip checking for a prior + # release branch, since none can exist and we know it cannot be in + # the Stabilization Phase yet. Thus, we return the CS branch and + # --rhel-target=latest + if beta: + return rhel_state_nt( + latest_version=latest_version, + target_version=latest_version, + rule_branch=cs_branch, + phase=pp_phase_devtestdoc, + rhel_target_default="latest", + enforcing=False, + ) - in_exception_phase = entry["phase"] == 450 + # First, check if this is the special case of Y=0 + # Note: since this is being written during the 10.0 Beta timeframe, there + # is no need to special case older formats like 9.0.0-beta. We can just + # use rhel-X.0-beta instead. + if active_y_version == 0: + prior_release_branch = format_branch(x_version, active_y_version, is_beta=True) + else: + prior_release_branch = format_branch( + x_version, active_y_version - 1, is_beta=False + ) + + logger.debug("Prior release branch: {}".format(prior_release_branch)) + + try: + branch_exists = does_branch_exist( + rhel_dist_git, namespace, repo_name, prior_release_branch + ) + except gitpython.GitCommandError as e: + raise RHELError("Could not read from RHEL dist-git. Are you on the VPN?") + + if branch_exists: + # The branch is there, so work on the active Y-stream, which is always + # in DevTestDoc Phase + phase = pp_phase_devtestdoc + check_tickets_branch = cs_branch + rhel_target_default = "latest" + enforcing = False + target_version = latest_version + else: + # The branch is not present, so we'll work on the prior Y-stream + check_tickets_branch = prior_release_branch + + target_x, target_y, target_extra = parse_rhel_branchname(prior_release_branch) + target_version = "{}.{}{}".format( + target_x, + target_y, + " Beta" if target_extra and "beta" in target_extra else "", + ) + + # The prior Y-stream is always in either Stabilization or Maintenance + # phase, so it always enforces. + enforcing = True + + # Determine which phase the prior release is in: + phase = phase_lookup[prior_release_branch] + + if phase == pp_phase_stabilization: + # We're in the Stabilization phase, so we can't automatically determine + # between the "zstream" and "exception" targets. + rhel_target_default = None + else: + # We must be in Maintenance phase + rhel_target_default = "zstream" + + return rhel_state_nt( + latest_version=latest_version, + target_version=target_version, + rule_branch=check_tickets_branch, + phase=phase, + rhel_target_default=rhel_target_default, + enforcing=enforcing, + ) + + +def format_current_state_message(rhel_state): + """ + Returns a human-readable string providing actionable information about the + current state of this repository. Useful for `centpkg current-state` and + the check-tickets function in merge requests + """ + + message = ( + f"Current RHEL status:\n" + f"\tThe latest active Y-stream release is RHEL {rhel_state.latest_version}\n" + f"\tThis project is targeting RHEL {rhel_state.target_version}\n" + ) + + if rhel_state.latest_version != rhel_state.target_version: + zstream_active_msg = ( + f"\t\tThe latest and targeted versions differ.\n" + f"\t\tIf this is not intentional, please see\n" + f"\t\thttps://one.redhat.com/rhel-development-guide/#proc_centos-stream-first_assembly_rhel-9-development\n" + f"\t\tfor details on how to unlock Y-stream development by creating the {rhel_state.rule_branch} branch.\n" + ) + message = "".join((message, zstream_active_msg)) + + target_phase = pp_phase_name_lookup[rhel_state.phase] + message = "".join( + ( + message, + f"\tThe {rhel_state.target_version} release is currently in {target_phase} phase\n", + ) + ) + + if rhel_state.phase == pp_phase_stabilization: + message = "".join( + (message, f"\t\tThe --rhel-target argument must be used when building.\n") + ) - logger.debug( - "Active Y-stream: {}, Enforcing: {}, Beta: {}".format( - active_y_version, in_exception_phase, beta + message = "".join( + ( + message, + f"\tTicket approvals are {'' if rhel_state.enforcing else 'not '}currently required for merge request approval.", ) ) - return x_version, active_y_version, beta, in_exception_phase + return message