asaleh / centos / centpkg

Forked from centos/centpkg a year ago
Clone

Blame src/centpkg/utils.py

80f38d
# -*- coding: utf-8 -*-
80f38d
# utils.py - a module with support methods for centpkg
80f38d
#
80f38d
# Copyright (C) 2021 Red Hat Inc.
80f38d
# Author(s): Ondrej Nosek <onosek@redhat.com>
80f38d
#
80f38d
# This program is free software; you can redistribute it and/or modify it
80f38d
# under the terms of the GNU General Public License as published by the
80f38d
# Free Software Foundation; either version 2 of the License, or (at your
80f38d
# option) any later version.  See http://www.gnu.org/copyleft/gpl.html for
80f38d
# the full text of the license.
80f38d
Adam Saleh dadf6a
import os
Adam Saleh dadf6a
from pathlib import Path
29dd93
import re
80f38d
import json
80f38d
80f38d
import git
80f38d
import requests
80f38d
from pyrpkg import rpkgError
80f38d
from requests.exceptions import ConnectionError
80f38d
from six.moves.configparser import NoOptionError, NoSectionError
80f38d
from six.moves.urllib.parse import quote_plus, urlparse
80f38d
5a7f92
dist_git_config = None
80f38d
80f38d
def do_fork(logger, base_url, token, repo_name, namespace, cli_name):
80f38d
    """
80f38d
    Creates a fork of the project.
80f38d
    :param logger: A logger object
80f38d
    :param base_url: a string of the URL repository
80f38d
    :param token: a string of the API token that has rights to make a fork
80f38d
    :param repo_name: a string of the repository name
80f38d
    :param namespace: a string determines a type of the repository
80f38d
    :param cli_name: string of the CLI's name (e.g. centpkg)
5b7b46
    :return: a tuple consisting of whether the fork needed to be created (bool)
5b7b46
        and the fork path (string)
80f38d
    """
80f38d
    api_url = '{0}/api/v4'.format(base_url.rstrip('/'))
80f38d
    project_id = quote_plus("redhat/centos-stream/{0}/{1}".format(namespace, repo_name))
80f38d
    fork_url = '{0}/projects/{1}/fork'.format(api_url, project_id)
80f38d
80f38d
    headers = {
80f38d
        'PRIVATE-TOKEN': token,
80f38d
        'Accept': 'application/json',
80f38d
        'Content-Type': 'application/json'
80f38d
    }
9a602e
    # define a new repository name/path to avoid collision with other projects
9a602e
    safe_name = "centos_{0}_{1}".format(namespace, repo_name)
9a602e
    payload = json.dumps({
9a602e
        'name': safe_name,  # name of the project after forking
9a602e
        'path': safe_name,
9a602e
    })
80f38d
    try:
80f38d
        rv = requests.post(
80f38d
            fork_url, headers=headers, data=payload, timeout=60)
80f38d
    except ConnectionError as error:
80f38d
        error_msg = ('The connection to API failed while trying to '
80f38d
                     'create a new fork. The error was: {0}'.format(str(error)))
80f38d
        raise rpkgError(error_msg)
80f38d
80f38d
    try:
80f38d
        # Extract response json for debugging
80f38d
        rv_json = rv.json()
5b7b46
        logger.debug("GitLab API response: '{0}'".format(rv_json))
80f38d
    except Exception:
80f38d
        pass
80f38d
464a71
    if rv.ok:
464a71
        fork_id = rv.json()['id']
464a71
        try:
464a71
            # Unprotect c9s in fork
cb72cf
            rv = requests.delete('{0}/projects/{1}/protected_branches/{2}'.format(api_url, fork_id, 'c9s'), headers=headers)
464a71
        except ConnectionError as error:
464a71
            error_msg = ('The connection to API failed while trying to unprotect c9s branch'
464a71
                         'in the fork. The error was: {0}'.format(str(error)))
464a71
            raise rpkgError(error_msg)
464a71
464a71
464a71
        try:
464a71
            # Reprotect c9s to disable pushes
464a71
            # Only maintainers in gitlab are allowed to push with the following config
464a71
            # In CS, every pkg maintainer is considered as a developer in gitlab
464a71
            data = {'id': fork_id,
464a71
                    'name': 'c9s',
464a71
                    'allowed_to_push': [{'access_level': 40}],
464a71
                    'allowed_to_merge': [{'access_level': 40}],
464a71
                    }
cb72cf
            rv = requests.post('{0}/projects/{1}/protected_branches'.format(api_url, fork_id), json=data, headers=headers)
464a71
        except ConnectionError as error:
464a71
            error_msg = ('The connection to API failed while trying to reprotect c9s branch'
464a71
                         'in the fork fork. The error was: {0}'.format(str(error)))
464a71
            raise rpkgError(error_msg)
464a71
80f38d
    base_error_msg = ('The following error occurred while creating a new fork: {0}')
80f38d
    if not rv.ok:
80f38d
        # fork was already created
80f38d
        if rv.status_code == 409 or rv.reason == "Conflict":
5b7b46
            # When the repo already exists, the return doesn't contain the repo
5b7b46
            # path or username.  Make one more API call to get the username of
5b7b46
            # the token to construct the repo path.
5b7b46
            rv = requests.get('{0}/user'.format(api_url), headers=headers)
5b7b46
            username = rv.json()['username']
5b7b46
            return False, '{0}/{1}'.format(username, safe_name)
80f38d
        # show hint for invalid, expired or revoked token
80f38d
        elif rv.status_code == 401 or rv.reason == "Unauthorized":
80f38d
            base_error_msg += '\nFor invalid or expired token refer to ' \
80f38d
                '"{0} fork -h" to set a token in your user ' \
80f38d
                'configuration.'.format(cli_name)
80f38d
        raise rpkgError(base_error_msg.format(rv.text))
80f38d
5b7b46
    return True, rv_json['path_with_namespace']
80f38d
80f38d
5b7b46
def do_add_remote(base_url, remote_base_url, repo, repo_path, remote_name):
80f38d
    """
80f38d
    Adds remote tracked repository
80f38d
    :param base_url: a string of the URL repository
80f38d
    :param remote_base_url: a string of the remote tracked repository
80f38d
    :param repo: object, current project git repository
5b7b46
    :param repo_path: a string of the repository path
5b7b46
    :param remote_name: a string of the remote name
80f38d
    :return: a bool; True if remote was created, False when already exists
80f38d
    """
80f38d
    parsed_url = urlparse(remote_base_url)
5b7b46
    remote_url = '{0}://{1}/{2}.git'.format(
80f38d
        parsed_url.scheme,
80f38d
        parsed_url.netloc,
5b7b46
        repo_path,
80f38d
    )
80f38d
80f38d
    # check already existing remote
80f38d
    for remote in repo.remotes:
5b7b46
        if remote.name == remote_name:
80f38d
            return False
80f38d
80f38d
    try:
5b7b46
        repo.create_remote(remote_name, url=remote_url)
80f38d
    except git.exc.GitCommandError as e:
80f38d
        error_msg = "During create remote:\n  {0}\n  {1}".format(
80f38d
            " ".join(e.command), e.stderr)
80f38d
        raise rpkgError(error_msg)
80f38d
    return True
80f38d
Adam Saleh dadf6a
def get_names_from_specs(specdir=None):
Adam Saleh dadf6a
    prev_cwd = Path.cwd()
Adam Saleh dadf6a
    if specdir:
Adam Saleh dadf6a
        os.chdir(specdir)
Adam Saleh dadf6a
    try:
Adam Saleh dadf6a
        return [f[:-5] for f in os.listdir() if f.endswith('.spec')]
Adam Saleh dadf6a
    except ValueError as e:
Adam Saleh dadf6a
        raise ValueError("Didn't find source file with name.")
Adam Saleh dadf6a
    finally:
Adam Saleh dadf6a
        os.chdir(prev_cwd)
Adam Saleh dadf6a
Adam Saleh dadf6a
def get_names_from_git(gitdir=None):
Adam Saleh dadf6a
    prev_cwd = Path.cwd()
Adam Saleh dadf6a
    return  list(Path(urlparse(u).path).with_suffix('').name
Adam Saleh dadf6a
        for r in git.Repo(gitdir or Path.cwd()).remotes
Adam Saleh dadf6a
        for u in r.urls if "redhat" in u)
Adam Saleh dadf6a
Adam Saleh dadf6a
def reset_git_origin(old_name, new_name, gitdir=None):
Adam Saleh dadf6a
    for r in git.Repo(gitdir or Path.cwd()).remotes:
Adam Saleh dadf6a
        if r.name == 'origin':
Adam Saleh dadf6a
            current_url = r.url
Adam Saleh dadf6a
            new_url = current_url.replace(old_name, new_name)
Adam Saleh dadf6a
            r.set_url(new_url)
Adam Saleh dadf6a
Adam Saleh dadf6a
def parse_project_dir(repo):
Adam Saleh dadf6a
    return Path(repo).with_suffix('').name
Adam Saleh dadf6a
Adam Saleh dadf6a
def fix_repo(project_dir=None):
Adam Saleh dadf6a
    path = Path(project_dir) if project_dir else Path.cwd()
Adam Saleh dadf6a
    spec_name, git_name, dir_name = get_project_names(path)
Adam Saleh dadf6a
    if spec_name != git_name:
Adam Saleh dadf6a
        #self.log.info("Changing the git-remote {} to match {}".format(git_name, spec_name))
Adam Saleh dadf6a
        reset_git_origin(git_name,spec_name,path)
Adam Saleh dadf6a
    if spec_name != dir_name:
Adam Saleh dadf6a
        #self.log.info("Changing the directory {} to match the specfile {}.spec".format(dir_name, spec_name))
Adam Saleh dadf6a
        os.rename(path,path.parent / spec_name)
Adam Saleh dadf6a
Adam Saleh dadf6a
def get_project_names(projectdir=None):
Adam Saleh dadf6a
    cwd = Path(projectdir) if projectdir else Path.cwd()
Adam Saleh dadf6a
    spec_names = get_names_from_specs(cwd)
Adam Saleh dadf6a
    git_names = get_names_from_git(cwd)
Adam Saleh dadf6a
    dir_name = cwd.name
Adam Saleh dadf6a
    lcase_names = set(s.lower() for s in spec_names).intersection(set(s.lower() for s in git_names))
Adam Saleh dadf6a
    try:
Adam Saleh dadf6a
        [spec_name] = [s for s in spec_names if s.lower() in lcase_names]
Adam Saleh dadf6a
        [git_name,*_] = [s for s in git_names if s.lower() in lcase_names]
Adam Saleh dadf6a
        return spec_name, git_name, dir_name
Adam Saleh dadf6a
    except ValueError as e:
Adam Saleh dadf6a
        raise ValueError("Instead of a single project name found {}".format(spec_names))
Adam Saleh dadf6a
80f38d
80f38d
def config_get_safely(config, section, option):
80f38d
    """
80f38d
    Returns option from the user's configuration file. In case of missing
80f38d
    section or option method throws an exception with a human-readable
80f38d
    warning and a possible hint.
80f38d
    The method should be used especially in situations when there are newly
80f38d
    added sections/options into the config. In this case, there is a risk that
80f38d
    the user's config wasn't properly upgraded.
80f38d
80f38d
    :param config: ConfigParser object
80f38d
    :param section: section name in the config
80f38d
    :param option: name of the option
80f38d
    :return: option value from the right section
80f38d
    :rtype: str
80f38d
    """
80f38d
80f38d
    hint = (
80f38d
        "First (if possible), refer to the help of the current command "
80f38d
        "(-h/--help).\n"
80f38d
        "There also might be a new version of the config after upgrade.\n"
80f38d
        "Hint: you can check if you have 'centpkg.conf.rpmnew' or "
80f38d
        "'centpkg.conf.rpmsave' in the config directory. If yes, try to merge "
80f38d
        "your changes to the config with the maintainer provided version "
80f38d
        "(or replace centpkg.conf file with 'centpkg.conf.rpmnew')."
80f38d
    )
80f38d
80f38d
    try:
80f38d
        return config.get(section, option)
80f38d
    except NoSectionError:
80f38d
        msg = "Missing section '{0}' in the config file.".format(section)
80f38d
        raise rpkgError("{0}\n{1}".format(msg, hint))
80f38d
    except NoOptionError:
80f38d
        msg = "Missing option '{0}' in the section '{1}' of the config file.".format(
80f38d
            option, section
80f38d
        )
80f38d
        raise rpkgError("{0}\n{1}".format(msg, hint))
80f38d
    except Exception:
80f38d
        raise
29dd93
29dd93
5a7f92
def get_canonical_repo_name(config, repo_url):
5a7f92
    """
5a7f92
    Check whether the current repo is a fork and if so, retrieve the parent
5a7f92
    fork to get the proper name.
5a7f92
    """
5a7f92
5a7f92
    # Look up the repo and query for forked_from_project
5a7f92
    cli_name = config_get_safely(dist_git_config, '__default', 'cli_name')
5a7f92
    distgit_section = '{0}.distgit'.format(cli_name)
5a7f92
    distgit_api_base_url = config_get_safely(dist_git_config, distgit_section, "apibaseurl")
5a7f92
5a7f92
    # Make sure the fork comes from the same Gitlab instance
5a7f92
    parsed_repo_url = urlparse(repo_url)
5a7f92
    parsed_base_url = urlparse(distgit_api_base_url)
5a7f92
5a7f92
    try:
5a7f92
        distgit_token = config_get_safely(dist_git_config, distgit_section, 'token')
5a7f92
5a7f92
        api_url = '{0}/api/v4'.format(distgit_api_base_url.rstrip('/'))
5a7f92
        project_url = '{0}/projects/{1}'.format(api_url, quote_plus(parsed_repo_url.path.lstrip('/')))
5a7f92
5a7f92
        headers = {
5a7f92
            'PRIVATE-TOKEN': distgit_token,
5a7f92
            'Accept': 'application/json',
5a7f92
            'Content-Type': 'application/json'
5a7f92
        }
5a7f92
5a7f92
        rv = requests.get(project_url, headers=headers)
5a7f92
        rv.raise_for_status()
5a7f92
5a7f92
        # Extract response json for debugging
5a7f92
        rv_json = rv.json()
5a7f92
5a7f92
        canonical_repo_name = rv_json['forked_from_project']['name']
5a7f92
    except KeyError as e:
5a7f92
        # There was no 'forked_from_project' key, likely meaning the
5a7f92
        # user lacked permissions to read the API. Usually this means
5a7f92
        # they haven't supplied a token or it is expired.
5a7f92
        raise rpkgError("Insufficient Gitlab API permissions. Missing token?")
5a7f92
5a7f92
    except Exception as e:
5a7f92
        # For any other exception, just fall back to using the last segment
5a7f92
        # of the URL path.
5a7f92
        canonical_repo_name = parsed_repo_url.path.split('/')[-1]
5a7f92
5a7f92
    # Chop off a trailing .git if any
5a7f92
    return canonical_repo_name.rsplit('.git', 1)[0]
5a7f92
29dd93
def get_repo_name(name, org='rpms'):
29dd93
    """
29dd93
    Try to parse the repository name in case it is a git url.
29dd93
29dd93
    Parameters
29dd93
    ----------
29dd93
    name: str
29dd93
        The repository name, including the org name.
29dd93
        It will try to retrieve both  repository name and org in case "name" is an url.
29dd93
29dd93
    org: str
29dd93
        The org to use in case name parsing is needed.
29dd93
29dd93
    Returns
29dd93
    -------
29dd93
    str
29dd93
        A string containing the repository name: $ORG/$REPO`.
29dd93
        It will return the original `name` parameter in case of regex match failure.
29dd93
    """
29dd93
    if name.startswith(org):
29dd93
        return name
29dd93
5a7f92
    # This is probably a renamed fork, so try to find the fork's parent
5a7f92
    repo_name = get_canonical_repo_name(dist_git_config, name)
29dd93
29dd93
    return '%s/%s' % (org, repo_name)