Blame SOURCES/pyproject_buildrequires.py

7ef706
import os
7ef706
import sys
7ef706
import importlib.metadata
7ef706
import argparse
7ef706
import traceback
7ef706
import contextlib
7ef706
from io import StringIO
7ef706
import json
7ef706
import subprocess
7ef706
import re
7ef706
import tempfile
7ef706
import email.parser
7ef706
import pathlib
7ef706
7ef706
from pyproject_requirements_txt import convert_requirements_txt
7ef706
7ef706
7ef706
# Some valid Python version specifiers are not supported.
7ef706
# Allow only the forms we know we can handle.
7ef706
VERSION_RE = re.compile(r'[a-zA-Z0-9.-]+(\.\*)?')
7ef706
7ef706
7ef706
class EndPass(Exception):
7ef706
    """End current pass of generating requirements"""
7ef706
7ef706
7ef706
# nb: we don't use functools.partial to be able to use pytest's capsys
7ef706
# see https://github.com/pytest-dev/pytest/issues/8900
7ef706
def print_err(*args, **kwargs):
7ef706
    kwargs.setdefault('file', sys.stderr)
7ef706
    print(*args, **kwargs)
7ef706
7ef706
7ef706
try:
7ef706
    from packaging.requirements import Requirement, InvalidRequirement
7ef706
    from packaging.utils import canonicalize_name
7ef706
except ImportError as e:
7ef706
    print_err('Import error:', e)
7ef706
    # already echoed by the %pyproject_buildrequires macro
7ef706
    sys.exit(0)
7ef706
7ef706
# uses packaging, needs to be imported after packaging is verified to be present
7ef706
from pyproject_convert import convert
7ef706
7ef706
7ef706
@contextlib.contextmanager
7ef706
def hook_call():
7ef706
    captured_out = StringIO()
7ef706
    with contextlib.redirect_stdout(captured_out):
7ef706
        yield
7ef706
    for line in captured_out.getvalue().splitlines():
7ef706
        print_err('HOOK STDOUT:', line)
7ef706
7ef706
7ef706
def guess_reason_for_invalid_requirement(requirement_str):
7ef706
    if ':' in requirement_str:
7ef706
        message = (
7ef706
            'It might be an URL. '
7ef706
            '%pyproject_buildrequires cannot handle all URL-based requirements. '
7ef706
            'Add PackageName@ (see PEP 508) to the URL to at least require any version of PackageName.'
7ef706
        )
7ef706
        if '@' in requirement_str:
7ef706
            message += ' (but note that URLs might not work well with other features)'
7ef706
        return message
7ef706
    if '/' in requirement_str:
7ef706
        return (
7ef706
            'It might be a local path. '
7ef706
            '%pyproject_buildrequires cannot handle local paths as requirements. '
7ef706
            'Use an URL with PackageName@ (see PEP 508) to at least require any version of PackageName.'
7ef706
        )
7ef706
    # No more ideas
7ef706
    return None
7ef706
7ef706
7ef706
class Requirements:
7ef706
    """Requirement printer"""
7ef706
    def __init__(self, get_installed_version, extras=None,
7ef706
                 generate_extras=False, python3_pkgversion='3'):
7ef706
        self.get_installed_version = get_installed_version
7ef706
        self.extras = set()
7ef706
7ef706
        if extras:
7ef706
            for extra in extras:
7ef706
                self.add_extras(*extra.split(','))
7ef706
7ef706
        self.missing_requirements = False
7ef706
7ef706
        self.generate_extras = generate_extras
7ef706
        self.python3_pkgversion = python3_pkgversion
7ef706
7ef706
    def add_extras(self, *extras):
7ef706
        self.extras |= set(e.strip() for e in extras)
7ef706
7ef706
    @property
7ef706
    def marker_envs(self):
7ef706
        if self.extras:
7ef706
            return [{'extra': e} for e in sorted(self.extras)]
7ef706
        return [{'extra': ''}]
7ef706
7ef706
    def evaluate_all_environamnets(self, requirement):
7ef706
        for marker_env in self.marker_envs:
7ef706
            if requirement.marker.evaluate(environment=marker_env):
7ef706
                return True
7ef706
        return False
7ef706
7ef706
    def add(self, requirement_str, *, source=None):
7ef706
        """Output a Python-style requirement string as RPM dep"""
7ef706
        print_err(f'Handling {requirement_str} from {source}')
7ef706
7ef706
        try:
7ef706
            requirement = Requirement(requirement_str)
7ef706
        except InvalidRequirement:
7ef706
            hint = guess_reason_for_invalid_requirement(requirement_str)
7ef706
            message = f'Requirement {requirement_str!r} from {source} is invalid.'
7ef706
            if hint:
7ef706
                message += f' Hint: {hint}'
7ef706
            raise ValueError(message)
7ef706
7ef706
        if requirement.url:
7ef706
            print_err(
7ef706
                f'WARNING: Simplifying {requirement_str!r} to {requirement.name!r}.'
7ef706
            )
7ef706
7ef706
        name = canonicalize_name(requirement.name)
7ef706
        if (requirement.marker is not None and
7ef706
                not self.evaluate_all_environamnets(requirement)):
7ef706
            print_err(f'Ignoring alien requirement:', requirement_str)
7ef706
            return
7ef706
7ef706
        # We need to always accept pre-releases as satisfying the requirement
7ef706
        # Otherwise e.g. installed cffi version 1.15.0rc2 won't even satisfy the requirement for "cffi"
7ef706
        # https://bugzilla.redhat.com/show_bug.cgi?id=2014639#c3
7ef706
        requirement.specifier.prereleases = True
7ef706
7ef706
        try:
7ef706
            # TODO: check if requirements with extras are satisfied
7ef706
            installed = self.get_installed_version(requirement.name)
7ef706
        except importlib.metadata.PackageNotFoundError:
7ef706
            print_err(f'Requirement not satisfied: {requirement_str}')
7ef706
            installed = None
7ef706
        if installed and installed in requirement.specifier:
7ef706
            print_err(f'Requirement satisfied: {requirement_str}')
7ef706
            print_err(f'   (installed: {requirement.name} {installed})')
7ef706
            if requirement.extras:
7ef706
                print_err(f'   (extras are currently not checked)')
7ef706
        else:
7ef706
            self.missing_requirements = True
7ef706
7ef706
        if self.generate_extras:
7ef706
            extra_names = [f'{name}[{extra.lower()}]' for extra in sorted(requirement.extras)]
7ef706
        else:
7ef706
            extra_names = []
7ef706
7ef706
        for name in [name] + extra_names:
7ef706
            together = []
7ef706
            for specifier in sorted(
7ef706
                requirement.specifier,
7ef706
                key=lambda s: (s.operator, s.version),
7ef706
            ):
7ef706
                if not VERSION_RE.fullmatch(str(specifier.version)):
7ef706
                    raise ValueError(
7ef706
                        f'Unknown character in version: {specifier.version}. '
7ef706
                        + '(This might be a bug in pyproject-rpm-macros.)',
7ef706
                    )
7ef706
                together.append(convert(python3dist(name, python3_pkgversion=self.python3_pkgversion),
7ef706
                                        specifier.operator, specifier.version))
7ef706
            if len(together) == 0:
7ef706
                print(python3dist(name,
7ef706
                                  python3_pkgversion=self.python3_pkgversion))
7ef706
            elif len(together) == 1:
7ef706
                print(together[0])
7ef706
            else:
7ef706
                print(f"({' with '.join(together)})")
7ef706
7ef706
    def check(self, *, source=None):
7ef706
        """End current pass if any unsatisfied dependencies were output"""
7ef706
        if self.missing_requirements:
7ef706
            print_err(f'Exiting dependency generation pass: {source}')
7ef706
            raise EndPass(source)
7ef706
7ef706
    def extend(self, requirement_strs, **kwargs):
7ef706
        """add() several requirements"""
7ef706
        for req_str in requirement_strs:
7ef706
            self.add(req_str, **kwargs)
7ef706
7ef706
7ef706
def get_backend(requirements):
7ef706
    try:
7ef706
        f = open('pyproject.toml')
7ef706
    except FileNotFoundError:
7ef706
        pyproject_data = {}
7ef706
    else:
7ef706
        try:
7ef706
            # lazy import toml here, not needed without pyproject.toml
7ef706
            import toml
7ef706
        except ImportError as e:
7ef706
            print_err('Import error:', e)
7ef706
            # already echoed by the %pyproject_buildrequires macro
7ef706
            sys.exit(0)
7ef706
        with f:
7ef706
            pyproject_data = toml.load(f)
7ef706
7ef706
    buildsystem_data = pyproject_data.get('build-system', {})
7ef706
    requirements.extend(
7ef706
        buildsystem_data.get('requires', ()),
7ef706
        source='build-system.requires',
7ef706
    )
7ef706
7ef706
    backend_name = buildsystem_data.get('build-backend')
7ef706
    if not backend_name:
7ef706
        # https://www.python.org/dev/peps/pep-0517/:
7ef706
        # If the pyproject.toml file is absent, or the build-backend key is
7ef706
        # missing, the source tree is not using this specification, and tools
7ef706
        # should revert to the legacy behaviour of running setup.py
7ef706
        # (either directly, or by implicitly invoking the [following] backend).
7ef706
        # If setup.py is also not present program will mimick pip's behavior
7ef706
        # and end with an error.
7ef706
        if not os.path.exists('setup.py'):
7ef706
            raise FileNotFoundError('File "setup.py" not found for legacy project.')
7ef706
        backend_name = 'setuptools.build_meta:__legacy__'
7ef706
7ef706
        # Note: For projects without pyproject.toml, this was already echoed
7ef706
        # by the %pyproject_buildrequires macro, but this also handles cases
7ef706
        # with pyproject.toml without a specified build backend.
7ef706
        # If the default requirements change, also change them in the macro!
7ef706
        requirements.add('setuptools >= 40.8', source='default build backend')
7ef706
        requirements.add('wheel', source='default build backend')
7ef706
7ef706
    requirements.check(source='build backend')
7ef706
7ef706
    backend_path = buildsystem_data.get('backend-path')
7ef706
    if backend_path:
7ef706
        # PEP 517 example shows the path as a list, but some projects don't follow that
7ef706
        if isinstance(backend_path, str):
7ef706
            backend_path = [backend_path]
7ef706
        sys.path = backend_path + sys.path
7ef706
7ef706
    module_name, _, object_name = backend_name.partition(":")
7ef706
    backend_module = importlib.import_module(module_name)
7ef706
7ef706
    if object_name:
7ef706
        return getattr(backend_module, object_name)
7ef706
7ef706
    return backend_module
7ef706
7ef706
7ef706
def generate_build_requirements(backend, requirements):
7ef706
    get_requires = getattr(backend, 'get_requires_for_build_wheel', None)
7ef706
    if get_requires:
7ef706
        with hook_call():
7ef706
            new_reqs = get_requires()
7ef706
        requirements.extend(new_reqs, source='get_requires_for_build_wheel')
7ef706
        requirements.check(source='get_requires_for_build_wheel')
7ef706
7ef706
7ef706
def generate_run_requirements(backend, requirements):
7ef706
    hook_name = 'prepare_metadata_for_build_wheel'
7ef706
    prepare_metadata = getattr(backend, hook_name, None)
7ef706
    if not prepare_metadata:
7ef706
        raise ValueError(
7ef706
            'build backend cannot provide build metadata '
7ef706
            + '(incl. runtime requirements) before build'
7ef706
        )
7ef706
    with hook_call():
7ef706
        dir_basename = prepare_metadata('.')
7ef706
    with open(dir_basename + '/METADATA') as f:
7ef706
        message = email.parser.Parser().parse(f, headersonly=True)
7ef706
    for key in 'Requires', 'Requires-Dist':
7ef706
        requires = message.get_all(key, ())
7ef706
        requirements.extend(requires, source=f'wheel metadata: {key}')
7ef706
7ef706
7ef706
def generate_tox_requirements(toxenv, requirements):
7ef706
    toxenv = ','.join(toxenv)
7ef706
    requirements.add('tox-current-env >= 0.0.6', source='tox itself')
7ef706
    requirements.check(source='tox itself')
7ef706
    with tempfile.NamedTemporaryFile('r') as deps, \
7ef706
        tempfile.NamedTemporaryFile('r') as extras, \
7ef706
            tempfile.NamedTemporaryFile('r') as provision:
7ef706
        r = subprocess.run(
7ef706
            [sys.executable, '-m', 'tox',
7ef706
             '--print-deps-to', deps.name,
7ef706
             '--print-extras-to', extras.name,
7ef706
             '--no-provision', provision.name,
7ef706
             '-qre', toxenv],
7ef706
            check=False,
7ef706
            encoding='utf-8',
7ef706
            stdout=subprocess.PIPE,
7ef706
            stderr=subprocess.STDOUT,
7ef706
        )
7ef706
        if r.stdout:
7ef706
            print_err(r.stdout, end='')
7ef706
7ef706
        provision_content = provision.read()
7ef706
        if provision_content and r.returncode != 0:
7ef706
            provision_requires = json.loads(provision_content)
7ef706
            if 'minversion' in provision_requires:
7ef706
                requirements.add(f'tox >= {provision_requires["minversion"]}',
7ef706
                                 source='tox provision (minversion)')
7ef706
            if 'requires' in provision_requires:
7ef706
                requirements.extend(provision_requires["requires"],
7ef706
                                    source='tox provision (requires)')
7ef706
            requirements.check(source='tox provision')  # this terminates the script
7ef706
            raise RuntimeError(
7ef706
                'Dependencies requested by tox provisioning appear installed, '
7ef706
                'but tox disagreed.')
7ef706
        else:
7ef706
            r.check_returncode()
7ef706
7ef706
        deplines = deps.read().splitlines()
7ef706
        packages = convert_requirements_txt(deplines)
7ef706
        requirements.add_extras(*extras.read().splitlines())
7ef706
        requirements.extend(packages,
7ef706
                            source=f'tox --print-deps-only: {toxenv}')
7ef706
7ef706
7ef706
def python3dist(name, op=None, version=None, python3_pkgversion="3"):
7ef706
    prefix = f"python{python3_pkgversion}dist"
7ef706
7ef706
    if op is None:
7ef706
        if version is not None:
7ef706
            raise AssertionError('op and version go together')
7ef706
        return f'{prefix}({name})'
7ef706
    else:
7ef706
        return f'{prefix}({name}) {op} {version}'
7ef706
7ef706
7ef706
def generate_requires(
7ef706
    *, include_runtime=False, toxenv=None, extras=None,
7ef706
    get_installed_version=importlib.metadata.version,  # for dep injection
7ef706
    generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True
7ef706
):
7ef706
    """Generate the BuildRequires for the project in the current directory
7ef706
7ef706
    This is the main Python entry point.
7ef706
    """
7ef706
    requirements = Requirements(
7ef706
        get_installed_version, extras=extras or [],
7ef706
        generate_extras=generate_extras,
7ef706
        python3_pkgversion=python3_pkgversion
7ef706
    )
7ef706
7ef706
    try:
7ef706
        if (include_runtime or toxenv) and not use_build_system:
7ef706
            raise ValueError('-N option cannot be used in combination with -r, -e, -t, -x options')
7ef706
        if requirement_files:
7ef706
            for req_file in requirement_files:
7ef706
                requirements.extend(
7ef706
                    convert_requirements_txt(req_file, pathlib.Path(req_file.name)),
7ef706
                    source=f'requirements file {req_file.name}'
7ef706
                )
7ef706
            requirements.check(source='all requirements files')
7ef706
        if use_build_system:
7ef706
            backend = get_backend(requirements)
7ef706
            generate_build_requirements(backend, requirements)
7ef706
        if toxenv:
7ef706
            include_runtime = True
7ef706
            generate_tox_requirements(toxenv, requirements)
7ef706
        if include_runtime:
7ef706
            generate_run_requirements(backend, requirements)
7ef706
    except EndPass:
7ef706
        return
7ef706
7ef706
7ef706
def main(argv):
7ef706
    parser = argparse.ArgumentParser(
7ef706
        description='Generate BuildRequires for a Python project.'
7ef706
    )
7ef706
    parser.add_argument(
7ef706
        '-r', '--runtime', action='store_true', default=True,
7ef706
        help='Generate run-time requirements (default, disable with -R)',
7ef706
    )
7ef706
    parser.add_argument(
7ef706
        '-R', '--no-runtime', action='store_false', dest='runtime',
7ef706
        help="Don't generate run-time requirements (implied by -N)",
7ef706
    )
7ef706
    parser.add_argument(
7ef706
        '-e', '--toxenv', metavar='TOXENVS', action='append',
7ef706
        help=('specify tox environments (comma separated and/or repeated)'
7ef706
              '(implies --tox)'),
7ef706
    )
7ef706
    parser.add_argument(
7ef706
        '-t', '--tox', action='store_true',
7ef706
        help=('generate test tequirements from tox environment '
7ef706
              '(implies --runtime)'),
7ef706
    )
7ef706
    parser.add_argument(
7ef706
        '-x', '--extras', metavar='EXTRAS', action='append',
7ef706
        help='comma separated list of "extras" for runtime requirements '
7ef706
             '(e.g. -x testing,feature-x) (implies --runtime, can be repeated)',
7ef706
    )
7ef706
    parser.add_argument(
7ef706
        '--generate-extras', action='store_true',
7ef706
        help='Generate build requirements on Python Extras',
7ef706
    )
7ef706
    parser.add_argument(
7ef706
        '-p', '--python3_pkgversion', metavar='PYTHON3_PKGVERSION',
7ef706
        default="3", help=('Python version for pythonXdist()'
7ef706
                           'or pythonX.Ydist() requirements'),
7ef706
    )
7ef706
    parser.add_argument(
7ef706
        '-N', '--no-use-build-system', dest='use_build_system',
7ef706
        action='store_false', help='Use -N to indicate that project does not use any build system',
7ef706
    )
7ef706
    parser.add_argument(
7ef706
       'requirement_files', nargs='*', type=argparse.FileType('r'),
7ef706
        help=('Add buildrequires from file'),
7ef706
    )
7ef706
7ef706
    args = parser.parse_args(argv)
7ef706
7ef706
    if not args.use_build_system:
7ef706
        args.runtime = False
7ef706
7ef706
    if args.toxenv:
7ef706
        args.tox = True
7ef706
7ef706
    if args.tox:
7ef706
        args.runtime = True
7ef706
        if not args.toxenv:
7ef706
            _default = f'py{sys.version_info.major}{sys.version_info.minor}'
7ef706
            args.toxenv = [os.getenv('RPM_TOXENV', _default)]
7ef706
7ef706
    if args.extras:
7ef706
        args.runtime = True
7ef706
7ef706
    try:
7ef706
        generate_requires(
7ef706
            include_runtime=args.runtime,
7ef706
            toxenv=args.toxenv,
7ef706
            extras=args.extras,
7ef706
            generate_extras=args.generate_extras,
7ef706
            python3_pkgversion=args.python3_pkgversion,
7ef706
            requirement_files=args.requirement_files,
7ef706
            use_build_system=args.use_build_system,
7ef706
        )
7ef706
    except Exception:
7ef706
        # Log the traceback explicitly (it's useful debug info)
7ef706
        traceback.print_exc()
7ef706
        exit(1)
7ef706
7ef706
7ef706
if __name__ == '__main__':
7ef706
    main(sys.argv[1:])