Blame src/centpkg/utils.py

80f38d
# -*- coding: utf-8 -*-
80f38d
# utils.py - a module with support methods for centpkg
80f38d
#
80f38d
# Copyright (C) 2021 Red Hat Inc.
80f38d
# Author(s): Ondrej Nosek <onosek@redhat.com>
80f38d
#
80f38d
# This program is free software; you can redistribute it and/or modify it
80f38d
# under the terms of the GNU General Public License as published by the
80f38d
# Free Software Foundation; either version 2 of the License, or (at your
80f38d
# option) any later version.  See http://www.gnu.org/copyleft/gpl.html for
80f38d
# the full text of the license.
80f38d
80f38d
import git
1e7ef8
import json
1e7ef8
import logging
1e7ef8
import os
1e7ef8
import pytz
1e7ef8
import re
80f38d
import requests
1e7ef8
import sys
1e7ef8
from datetime import date, datetime
80f38d
from pyrpkg import rpkgError
80f38d
from requests.exceptions import ConnectionError
80f38d
from six.moves.configparser import NoOptionError, NoSectionError
80f38d
from six.moves.urllib.parse import quote_plus, urlparse
80f38d
1e7ef8
import git as gitpython
1e7ef8
5a7f92
dist_git_config = None
80f38d
80f38d
def do_fork(logger, base_url, token, repo_name, namespace, cli_name):
80f38d
    """
80f38d
    Creates a fork of the project.
80f38d
    :param logger: A logger object
80f38d
    :param base_url: a string of the URL repository
80f38d
    :param token: a string of the API token that has rights to make a fork
80f38d
    :param repo_name: a string of the repository name
80f38d
    :param namespace: a string determines a type of the repository
80f38d
    :param cli_name: string of the CLI's name (e.g. centpkg)
5b7b46
    :return: a tuple consisting of whether the fork needed to be created (bool)
5b7b46
        and the fork path (string)
80f38d
    """
80f38d
    api_url = '{0}/api/v4'.format(base_url.rstrip('/'))
80f38d
    project_id = quote_plus("redhat/centos-stream/{0}/{1}".format(namespace, repo_name))
80f38d
    fork_url = '{0}/projects/{1}/fork'.format(api_url, project_id)
80f38d
80f38d
    headers = {
80f38d
        'PRIVATE-TOKEN': token,
80f38d
        'Accept': 'application/json',
80f38d
        'Content-Type': 'application/json'
80f38d
    }
9a602e
    # define a new repository name/path to avoid collision with other projects
9a602e
    safe_name = "centos_{0}_{1}".format(namespace, repo_name)
9a602e
    payload = json.dumps({
9a602e
        'name': safe_name,  # name of the project after forking
9a602e
        'path': safe_name,
9a602e
    })
80f38d
    try:
80f38d
        rv = requests.post(
80f38d
            fork_url, headers=headers, data=payload, timeout=60)
80f38d
    except ConnectionError as error:
80f38d
        error_msg = ('The connection to API failed while trying to '
80f38d
                     'create a new fork. The error was: {0}'.format(str(error)))
80f38d
        raise rpkgError(error_msg)
80f38d
80f38d
    try:
80f38d
        # Extract response json for debugging
80f38d
        rv_json = rv.json()
5b7b46
        logger.debug("GitLab API response: '{0}'".format(rv_json))
80f38d
    except Exception:
80f38d
        pass
80f38d
464a71
    if rv.ok:
464a71
        fork_id = rv.json()['id']
464a71
        try:
464a71
            # Unprotect c9s in fork
cb72cf
            rv = requests.delete('{0}/projects/{1}/protected_branches/{2}'.format(api_url, fork_id, 'c9s'), headers=headers)
464a71
        except ConnectionError as error:
464a71
            error_msg = ('The connection to API failed while trying to unprotect c9s branch'
464a71
                         'in the fork. The error was: {0}'.format(str(error)))
464a71
            raise rpkgError(error_msg)
464a71
464a71
464a71
        try:
464a71
            # Reprotect c9s to disable pushes
464a71
            # Only maintainers in gitlab are allowed to push with the following config
464a71
            # In CS, every pkg maintainer is considered as a developer in gitlab
464a71
            data = {'id': fork_id,
464a71
                    'name': 'c9s',
464a71
                    'allowed_to_push': [{'access_level': 40}],
464a71
                    'allowed_to_merge': [{'access_level': 40}],
464a71
                    }
cb72cf
            rv = requests.post('{0}/projects/{1}/protected_branches'.format(api_url, fork_id), json=data, headers=headers)
464a71
        except ConnectionError as error:
464a71
            error_msg = ('The connection to API failed while trying to reprotect c9s branch'
464a71
                         'in the fork fork. The error was: {0}'.format(str(error)))
464a71
            raise rpkgError(error_msg)
464a71
80f38d
    base_error_msg = ('The following error occurred while creating a new fork: {0}')
80f38d
    if not rv.ok:
80f38d
        # fork was already created
80f38d
        if rv.status_code == 409 or rv.reason == "Conflict":
5b7b46
            # When the repo already exists, the return doesn't contain the repo
5b7b46
            # path or username.  Make one more API call to get the username of
5b7b46
            # the token to construct the repo path.
5b7b46
            rv = requests.get('{0}/user'.format(api_url), headers=headers)
5b7b46
            username = rv.json()['username']
5b7b46
            return False, '{0}/{1}'.format(username, safe_name)
80f38d
        # show hint for invalid, expired or revoked token
80f38d
        elif rv.status_code == 401 or rv.reason == "Unauthorized":
80f38d
            base_error_msg += '\nFor invalid or expired token refer to ' \
80f38d
                '"{0} fork -h" to set a token in your user ' \
80f38d
                'configuration.'.format(cli_name)
80f38d
        raise rpkgError(base_error_msg.format(rv.text))
80f38d
5b7b46
    return True, rv_json['path_with_namespace']
80f38d
80f38d
5b7b46
def do_add_remote(base_url, remote_base_url, repo, repo_path, remote_name):
80f38d
    """
80f38d
    Adds remote tracked repository
80f38d
    :param base_url: a string of the URL repository
80f38d
    :param remote_base_url: a string of the remote tracked repository
80f38d
    :param repo: object, current project git repository
5b7b46
    :param repo_path: a string of the repository path
5b7b46
    :param remote_name: a string of the remote name
80f38d
    :return: a bool; True if remote was created, False when already exists
80f38d
    """
80f38d
    parsed_url = urlparse(remote_base_url)
5b7b46
    remote_url = '{0}://{1}/{2}.git'.format(
80f38d
        parsed_url.scheme,
80f38d
        parsed_url.netloc,
5b7b46
        repo_path,
80f38d
    )
80f38d
80f38d
    # check already existing remote
80f38d
    for remote in repo.remotes:
5b7b46
        if remote.name == remote_name:
80f38d
            return False
80f38d
80f38d
    try:
5b7b46
        repo.create_remote(remote_name, url=remote_url)
80f38d
    except git.exc.GitCommandError as e:
80f38d
        error_msg = "During create remote:\n  {0}\n  {1}".format(
80f38d
            " ".join(e.command), e.stderr)
80f38d
        raise rpkgError(error_msg)
80f38d
    return True
80f38d
80f38d
80f38d
def config_get_safely(config, section, option):
80f38d
    """
80f38d
    Returns option from the user's configuration file. In case of missing
80f38d
    section or option method throws an exception with a human-readable
80f38d
    warning and a possible hint.
80f38d
    The method should be used especially in situations when there are newly
80f38d
    added sections/options into the config. In this case, there is a risk that
80f38d
    the user's config wasn't properly upgraded.
80f38d
80f38d
    :param config: ConfigParser object
80f38d
    :param section: section name in the config
80f38d
    :param option: name of the option
80f38d
    :return: option value from the right section
80f38d
    :rtype: str
80f38d
    """
80f38d
80f38d
    hint = (
80f38d
        "First (if possible), refer to the help of the current command "
80f38d
        "(-h/--help).\n"
80f38d
        "There also might be a new version of the config after upgrade.\n"
80f38d
        "Hint: you can check if you have 'centpkg.conf.rpmnew' or "
80f38d
        "'centpkg.conf.rpmsave' in the config directory. If yes, try to merge "
80f38d
        "your changes to the config with the maintainer provided version "
80f38d
        "(or replace centpkg.conf file with 'centpkg.conf.rpmnew')."
80f38d
    )
80f38d
80f38d
    try:
80f38d
        return config.get(section, option)
80f38d
    except NoSectionError:
80f38d
        msg = "Missing section '{0}' in the config file.".format(section)
80f38d
        raise rpkgError("{0}\n{1}".format(msg, hint))
80f38d
    except NoOptionError:
80f38d
        msg = "Missing option '{0}' in the section '{1}' of the config file.".format(
80f38d
            option, section
80f38d
        )
80f38d
        raise rpkgError("{0}\n{1}".format(msg, hint))
80f38d
    except Exception:
80f38d
        raise
29dd93
29dd93
5a7f92
def get_canonical_repo_name(config, repo_url):
5a7f92
    """
5a7f92
    Check whether the current repo is a fork and if so, retrieve the parent
5a7f92
    fork to get the proper name.
5a7f92
    """
5a7f92
5a7f92
    # Look up the repo and query for forked_from_project
5a7f92
    cli_name = config_get_safely(dist_git_config, '__default', 'cli_name')
5a7f92
    distgit_section = '{0}.distgit'.format(cli_name)
5a7f92
    distgit_api_base_url = config_get_safely(dist_git_config, distgit_section, "apibaseurl")
5a7f92
5a7f92
    # Make sure the fork comes from the same Gitlab instance
5a7f92
    parsed_repo_url = urlparse(repo_url)
5a7f92
    parsed_base_url = urlparse(distgit_api_base_url)
5a7f92
5a7f92
    try:
5a7f92
        distgit_token = config_get_safely(dist_git_config, distgit_section, 'token')
5a7f92
5a7f92
        api_url = '{0}/api/v4'.format(distgit_api_base_url.rstrip('/'))
5a7f92
        project_url = '{0}/projects/{1}'.format(api_url, quote_plus(parsed_repo_url.path.lstrip('/')))
5a7f92
5a7f92
        headers = {
5a7f92
            'PRIVATE-TOKEN': distgit_token,
5a7f92
            'Accept': 'application/json',
5a7f92
            'Content-Type': 'application/json'
5a7f92
        }
5a7f92
5a7f92
        rv = requests.get(project_url, headers=headers)
5a7f92
        rv.raise_for_status()
5a7f92
5a7f92
        # Extract response json for debugging
5a7f92
        rv_json = rv.json()
5a7f92
5a7f92
        canonical_repo_name = rv_json['forked_from_project']['name']
5a7f92
    except KeyError as e:
5a7f92
        # There was no 'forked_from_project' key, likely meaning the
5a7f92
        # user lacked permissions to read the API. Usually this means
5a7f92
        # they haven't supplied a token or it is expired.
5a7f92
        raise rpkgError("Insufficient Gitlab API permissions. Missing token?")
5a7f92
5a7f92
    except Exception as e:
5a7f92
        # For any other exception, just fall back to using the last segment
5a7f92
        # of the URL path.
5a7f92
        canonical_repo_name = parsed_repo_url.path.split('/')[-1]
5a7f92
5a7f92
    # Chop off a trailing .git if any
5a7f92
    return canonical_repo_name.rsplit('.git', 1)[0]
5a7f92
29dd93
def get_repo_name(name, org='rpms'):
29dd93
    """
29dd93
    Try to parse the repository name in case it is a git url.
29dd93
29dd93
    Parameters
29dd93
    ----------
29dd93
    name: str
29dd93
        The repository name, including the org name.
29dd93
        It will try to retrieve both  repository name and org in case "name" is an url.
29dd93
29dd93
    org: str
29dd93
        The org to use in case name parsing is needed.
29dd93
29dd93
    Returns
29dd93
    -------
29dd93
    str
29dd93
        A string containing the repository name: $ORG/$REPO`.
29dd93
        It will return the original `name` parameter in case of regex match failure.
29dd93
    """
29dd93
    if name.startswith(org):
29dd93
        return name
29dd93
5a7f92
    # This is probably a renamed fork, so try to find the fork's parent
5a7f92
    repo_name = get_canonical_repo_name(dist_git_config, name)
29dd93
29dd93
    return '%s/%s' % (org, repo_name)
1e7ef8
1e7ef8
def stream_mapping(csname):
1e7ef8
    """
1e7ef8
    Given a CentOS Stream name, map it to the corresponding RHEL name.
1e7ef8
1e7ef8
    Parameters
1e7ef8
    ----------
1e7ef8
    csname: str
1e7ef8
        The CentOS Stream name.
1e7ef8
1e7ef8
    Returns
1e7ef8
    -------
1e7ef8
    str
1e7ef8
        Correspoinding RHEL name.
1e7ef8
    """
1e7ef8
    if csname == "c8s" or csname == "cs8" :
1e7ef8
        return "rhel-8"
1e7ef8
    if csname == "c9s" or csname == "cs9" :
1e7ef8
        return "rhel-9"
1e7ef8
    if csname == "c10s" or csname == "cs10" :
1e7ef8
        return "rhel-10"
1e7ef8
    if csname == "c11s" or csname == "cs11" :
1e7ef8
        return "rhel-11"
1e7ef8
    return None
1e7ef8
1e7ef8
def does_divergent_branch_exist(repo_name, rhel_version, rhel_dist_git, pp_api_url, namespace):
1e7ef8
    logger = logging.getLogger(__name__)
1e7ef8
1e7ef8
    # Determine if the Y-1 branch exists for this repo
1e7ef8
1e7ef8
    # Look up the Y-1 branch name
1e7ef8
    divergent_branch = determine_divergent_branch(
1e7ef8
        rhel_version,
1e7ef8
        pp_api_url,
1e7ef8
        namespace,
1e7ef8
    )
1e7ef8
    logger.debug("Divergent branch: {}".format(divergent_branch))
1e7ef8
    
1e7ef8
    g = gitpython.cmd.Git()
1e7ef8
    try:
1e7ef8
        g.ls_remote(
1e7ef8
            "--exit-code",
1e7ef8
            os.path.join(rhel_dist_git, namespace, repo_name),
1e7ef8
            divergent_branch,
1e7ef8
        )
1e7ef8
        branch_exists = True
1e7ef8
    except gitpython.GitCommandError as e:
1e7ef8
        t, v, tb = sys.exc_info()
1e7ef8
        # `git ls-remote --exit-code` returns "2" if it cannot find the ref
1e7ef8
        if e.status == 2:
1e7ef8
            branch_exists = False
1e7ef8
        else:
1e7ef8
            raise
1e7ef8
    return branch_exists
1e7ef8
1e7ef8
def determine_divergent_branch(rhel_version, pp_api_url, namespace):
1e7ef8
    logger = logging.getLogger(__name__)
1e7ef8
1e7ef8
    # Query the "package pages" API for the current active Y-stream release
1e7ef8
    # Phase 230 is "Planning / Development / Testing" (AKA DeveTestDoc)
1e7ef8
    request_params = {
1e7ef8
        "phase": 230,
1e7ef8
        "product__shortname": "rhel",
1e7ef8
        "relgroup__shortname": rhel_version,
1e7ef8
        "format": "json",
1e7ef8
    }
1e7ef8
1e7ef8
    res = requests.get(
1e7ef8
        os.path.join(pp_api_url, "latest", "releases"),
1e7ef8
        params=request_params,
1e7ef8
        timeout=60,
1e7ef8
    )
1e7ef8
    res.raise_for_status()
1e7ef8
    payload = json.loads(res.text)
1e7ef8
    logger.debug(
1e7ef8
        "Response from PP API: {}".format(json.dumps(payload, indent=2))
1e7ef8
    )
1e7ef8
    if len(payload) < 1:
1e7ef8
        raise RuntimeError("Received zero potential release matches)")
1e7ef8
1e7ef8
    active_y_version = -1
1e7ef8
    for entry in payload:
1e7ef8
        shortname = entry["shortname"]
1e7ef8
1e7ef8
        # The shortname is in the form rhel-9-1.0
1e7ef8
        # Extract the active Y-stream version
1e7ef8
        m = re.search("(?<={}-)\d+(?=\.0)".format(rhel_version), shortname)
1e7ef8
        if not m:
1e7ef8
            raise RuntimeError(
1e7ef8
                "Could not determine active Y-stream version from shortname"
1e7ef8
            )
1e7ef8
        y_version = int(m.group(0))
1e7ef8
        if y_version > active_y_version:
1e7ef8
            active_y_version = y_version
1e7ef8
1e7ef8
    # The divergent branch is Y-1
1e7ef8
    return "{}.{}.0".format(rhel_version, active_y_version - 1)
1e7ef8
1e7ef8
def _datesplit(isodate):
1e7ef8
    date_string_tuple = isodate.split('-')
1e7ef8
    return [ int(x) for x in date_string_tuple ]
1e7ef8
1e7ef8
1e7ef8
def determine_active_y_version(rhel_version, pp_api_url):
1e7ef8
    """
1e7ef8
    Returns: A 2-tuple of the active Y-stream version(int) and whether we are
1e7ef8
    in the Exception Phase(bool)
1e7ef8
    """
1e7ef8
    logger = logging.getLogger(__name__)
1e7ef8
1e7ef8
    # Query the "package pages" API for the current active Y-stream release
1e7ef8
    # Phase 230 is "Planning / Development / Testing" (AKA DeveTestDoc)
1e7ef8
    request_params = {
1e7ef8
        "phase": 230,
1e7ef8
        "product__shortname": "rhel",
1e7ef8
        "relgroup__shortname": rhel_version,
1e7ef8
        "format": "json",
1e7ef8
    }
1e7ef8
1e7ef8
    res = requests.get(
1e7ef8
        os.path.join(pp_api_url, "latest", "releases"),
1e7ef8
        params=request_params,
1e7ef8
        timeout=60,
1e7ef8
    )
1e7ef8
    res.raise_for_status()
1e7ef8
    payload = json.loads(res.text)
1e7ef8
    logger.debug(
1e7ef8
        "Response from PP API: {}".format(json.dumps(payload, indent=2))
1e7ef8
    )
1e7ef8
    if len(payload) < 1:
1e7ef8
        raise RuntimeError("Received zero potential release matches)")
1e7ef8
1e7ef8
    release_id = -1
1e7ef8
    active_y_version = -1
1e7ef8
    for entry in payload:
1e7ef8
        shortname = entry["shortname"]
1e7ef8
1e7ef8
        # The shortname is in the form rhel-9-1.0
1e7ef8
        # Extract the active Y-stream version
1e7ef8
        m = re.search("(?<={}-)\d+(?=\.0)".format(rhel_version), shortname)
1e7ef8
        if not m:
1e7ef8
            raise RuntimeError(
1e7ef8
                "Could not determine active Y-stream version from shortname"
1e7ef8
            )
1e7ef8
        y_version = int(m.group(0))
1e7ef8
        if y_version > active_y_version:
1e7ef8
            active_y_version = y_version
1e7ef8
            release_id = entry["id"]
1e7ef8
1e7ef8
    # Now look up whether we are in the Exception Phase for this Y-stream release
1e7ef8
    request_params = {
1e7ef8
        "name__regex": "Exception Phase",
1e7ef8
        "format": "json",
1e7ef8
    }
1e7ef8
    res = requests.get(os.path.join(pp_api_url, "latest", "releases", str(release_id), "schedule-tasks"), params=request_params)
1e7ef8
    res.raise_for_status()
1e7ef8
    payload = json.loads(res.text)
1e7ef8
1e7ef8
    # This lookup *must* return exactly one value or the Product Pages are
1e7ef8
    # wrong and must be fixed.
1e7ef8
    assert len(payload) == 1
1e7ef8
1e7ef8
    # Determine if this Y-stream release is in the exception phase
1e7ef8
    today = datetime.now(tz=pytz.utc).date()
1e7ef8
    exception_start_date = date(*_datesplit(payload[0]["date_start"]))
1e7ef8
    in_exception_phase = today >= exception_start_date
1e7ef8
1e7ef8
    logger.debug("Active Y-stream: {}, Enforcing: {}".format(active_y_version, in_exception_phase))
1e7ef8
1e7ef8
    return active_y_version, in_exception_phase
1e7ef8