From 6f4bc51b3e180ba1e001787f163e9b4e08ee974a Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: Jan 11 2022 17:11:20 +0000 Subject: import python-rpm-macros-3.9-48.el9 --- diff --git a/SOURCES/import_all_modules.py b/SOURCES/import_all_modules.py new file mode 100644 index 0000000..3930236 --- /dev/null +++ b/SOURCES/import_all_modules.py @@ -0,0 +1,171 @@ +'''Script to perform import of each module given to %%py_check_import +''' +import argparse +import importlib +import fnmatch +import os +import re +import site +import sys + +from contextlib import contextmanager +from pathlib import Path + + +def read_modules_files(file_paths): + '''Read module names from the files (modules must be newline separated). + + Return the module names list or, if no files were provided, an empty list. + ''' + + if not file_paths: + return [] + + modules = [] + for file in file_paths: + file_contents = file.read_text() + modules.extend(file_contents.split()) + return modules + + +def read_modules_from_cli(argv): + '''Read module names from command-line arguments (space or comma separated). + + Return the module names list. + ''' + + if not argv: + return [] + + # %%py3_check_import allows to separate module list with comma or whitespace, + # we need to unify the output to a list of particular elements + modules_as_str = ' '.join(argv) + modules = re.split(r'[\s,]+', modules_as_str) + # Because of shell expansion in some less typical cases it may happen + # that a trailing space will occur at the end of the list. + # Remove the empty items from the list before passing it further + modules = [m for m in modules if m] + return modules + + +def filter_top_level_modules_only(modules): + '''Filter out entries with nested modules (containing dot) ie. 'foo.bar'. + + Return the list of top-level modules. + ''' + + return [module for module in modules if '.' not in module] + + +def any_match(text, globs): + '''Return True if any of given globs fnmatchcase's the given text.''' + + return any(fnmatch.fnmatchcase(text, g) for g in globs) + + +def exclude_unwanted_module_globs(globs, modules): + '''Filter out entries which match the either of the globs given as argv. + + Return the list of filtered modules. + ''' + + return [m for m in modules if not any_match(m, globs)] + + +def read_modules_from_all_args(args): + '''Return a joined list of modules from all given command-line arguments. + ''' + + modules = read_modules_files(args.filename) + modules.extend(read_modules_from_cli(args.modules)) + if args.exclude: + modules = exclude_unwanted_module_globs(args.exclude, modules) + + if args.top_level: + modules = filter_top_level_modules_only(modules) + + # Error when someone accidentally managed to filter out everything + if len(modules) == 0: + raise ValueError('No modules to check were left') + + return modules + + +def import_modules(modules): + '''Procedure to perform import check for each module name from the given list of modules. + ''' + + for module in modules: + print('Check import:', module, file=sys.stderr) + importlib.import_module(module) + + +def argparser(): + parser = argparse.ArgumentParser( + description='Generate list of all importable modules for import check.' + ) + parser.add_argument( + 'modules', nargs='*', + help=('Add modules to check the import (space or comma separated).'), + ) + parser.add_argument( + '-f', '--filename', action='append', type=Path, + help='Add importable module names list from file.', + ) + parser.add_argument( + '-t', '--top-level', action='store_true', + help='Check only top-level modules.', + ) + parser.add_argument( + '-e', '--exclude', action='append', + help='Provide modules globs to be excluded from the check.', + ) + return parser + + +@contextmanager +def remove_unwanteds_from_sys_path(): + '''Remove cwd and this script's parent from sys.path for the import test. + Bring the original contents back after import is done (or failed) + ''' + + cwd_absolute = Path.cwd().absolute() + this_file_parent = Path(__file__).parent.absolute() + old_sys_path = list(sys.path) + for path in old_sys_path: + if Path(path).absolute() in (cwd_absolute, this_file_parent): + sys.path.remove(path) + try: + yield + finally: + sys.path = old_sys_path + + +def addsitedirs_from_environ(): + '''Load directories from the _PYTHONSITE environment variable (separated by :) + and load the ones already present in sys.path via site.addsitedir() + to handle .pth files in them. + + This is needed to properly import old-style namespace packages with nspkg.pth files. + See https://bugzilla.redhat.com/2018551 for a more detailed rationale.''' + for path in os.getenv('_PYTHONSITE', '').split(':'): + if path in sys.path: + site.addsitedir(path) + + +def main(argv=None): + + cli_args = argparser().parse_args(argv) + + if not cli_args.modules and not cli_args.filename: + raise ValueError('No modules to check were provided') + + modules = read_modules_from_all_args(cli_args) + + with remove_unwanteds_from_sys_path(): + addsitedirs_from_environ() + import_modules(modules) + + +if __name__ == '__main__': + main() diff --git a/SOURCES/macros.python b/SOURCES/macros.python index 04c47c5..f0d431d 100644 --- a/SOURCES/macros.python +++ b/SOURCES/macros.python @@ -1,12 +1,14 @@ # unversioned macros: used with user defined __python, no longer part of rpm >= 4.15 # __python is defined to error by default in the srpm macros -%python_sitelib %(%{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib'))") -%python_sitearch %(%{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib'))") -%python_version %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python_version_nodots %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python_platform %(%{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") -%python_platform_triplet %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") -%python_ext_suffix %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +# nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) +# so we set it manually (to empty string), making our Python prefer the correct install scheme location +%python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib'))") +%python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib'))") +%python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") +%python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") %py_setup setup.py %py_shbang_opts -s @@ -67,16 +69,28 @@ } # With $PATH and $PYTHONPATH set to the %%buildroot, -# try to import the given Python module(s). +# try to import the Python module(s) given as command-line args or read from file (-f). +# Respect the custom values of %%py_shebang_flags or set nothing if it's undefined. +# Filter and check import on only top-level modules using -t flag. +# Exclude unwanted modules by passing their globs to -e option. # Useful as a smoke test in %%check when running tests is not feasible. -# Use spaces or commas as separators. -%py_check_import() %{expand:\\\ - (cd %{_topdir} &&\\\ +# Use spaces or commas as separators if providing list directly. +# Use newlines as separators if providing list in a file. +%py_check_import(e:tf:) %{expand:\\\ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ + _PYTHONSITE="%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{__python} -c "import %{lua:local m=rpm.expand('%{?*}'):gsub('[%s,]+', ', ');print(m)}" - ) + %{lua: + local command = "%{__python} " + if rpm.expand("%{?py_shebang_flags}") ~= "" then + command = command .. "-%{py_shebang_flags}" + end + command = command .. " %{_rpmconfigdir}/redhat/import_all_modules.py " + -- handle multiline arguments correctly, see https://bugzilla.redhat.com/2018809 + local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ") + print(command .. args) + } } %python_provide() %{lua: diff --git a/SOURCES/macros.python-srpm b/SOURCES/macros.python-srpm index 8401975..b30a6c8 100644 --- a/SOURCES/macros.python-srpm +++ b/SOURCES/macros.python-srpm @@ -1,22 +1,3 @@ -# Define the Python interpreter paths in the SRPM macros so that -# - they can be used in Build/Requires -# - they can be used in non-Python packages where requiring pythonX-devel would -# be an overkill - -# use the underscored macros to redefine the behavior of %%python3_version etc. -%__python2 /usr/bin/python2 -%__python3 /usr/bin/python3 - -# use the non-underscored macros to refer to Python in spec, etc. -%python2 %__python2 -%python3 %__python3 - -# See https://fedoraproject.org/wiki/Changes/PythonMacroError -%__python %{error:attempt to use unversioned python, define %%__python to %{__python2} or %{__python3} explicitly} - -# Users can use %%python only if they redefined %%__python (e.g. to %%__python3) -%python %__python - # There are multiple Python 3 versions packaged, but only one can be the "main" version # That means that it owns the "python3" namespace: # - python3 package name @@ -49,6 +30,25 @@ # Alternatively, it can be overridden in spec (e.g. to "3.8") when building for alternate Python stacks. %python3_pkgversion 3 +# Define the Python interpreter paths in the SRPM macros so that +# - they can be used in Build/Requires +# - they can be used in non-Python packages where requiring pythonX-devel would +# be an overkill + +# use the underscored macros to redefine the behavior of %%python3_version etc. +%__python2 /usr/bin/python2 +%__python3 /usr/bin/python%{python3_pkgversion} + +# use the non-underscored macros to refer to Python in spec, etc. +%python2 %__python2 +%python3 %__python3 + +# See https://fedoraproject.org/wiki/Changes/PythonMacroError +%__python %{error:attempt to use unversioned python, define %%__python to %{__python2} or %{__python3} explicitly} + +# Users can use %%python only if they redefined %%__python (e.g. to %%__python3) +%python %__python + # Define where Python wheels will be stored and the prefix of -wheel packages # - In Fedora we want wheel subpackages named e.g. `python-pip-wheel` that # install packages into `/usr/share/python-wheels`. Both names are not diff --git a/SOURCES/macros.python3 b/SOURCES/macros.python3 index 93b4199..0ccafd9 100644 --- a/SOURCES/macros.python3 +++ b/SOURCES/macros.python3 @@ -1,10 +1,12 @@ -%python3_sitelib %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib'))") -%python3_sitearch %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib'))") -%python3_version %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python3_version_nodots %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python3_platform %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") -%python3_platform_triplet %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") -%python3_ext_suffix %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +# nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) +# so we set it manually (to empty string), making our Python prefer the correct install scheme location +%python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib'))") +%python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib'))") +%python3_version %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") +%python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %py3_shbang_opts -s @@ -65,16 +67,28 @@ } # With $PATH and $PYTHONPATH set to the %%buildroot, -# try to import the given Python 3 module(s). +# try to import the Python 3 module(s) given as command-line args or read from file (-f). +# Respect the custom values of %%py3_shebang_flags or set nothing if it's undefined. +# Filter and check import on only top-level modules using -t flag. +# Exclude unwanted modules by passing their globs to -e option. # Useful as a smoke test in %%check when running tests is not feasible. -# Use spaces or commas as separators. -%py3_check_import() %{expand:\\\ - (cd %{_topdir} &&\\\ +# Use spaces or commas as separators if providing list directly. +# Use newlines as separators if providing list in a file. +%py3_check_import(e:tf:) %{expand:\\\ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ + _PYTHONSITE="%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{__python3} -c "import %{lua:local m=rpm.expand('%{?*}'):gsub('[%s,]+', ', ');print(m)}" - ) + %{lua: + local command = "%{__python3} " + if rpm.expand("%{?py3_shebang_flags}") ~= "" then + command = command .. "-%{py3_shebang_flags}" + end + command = command .. " %{_rpmconfigdir}/redhat/import_all_modules.py " + -- handle multiline arguments correctly, see https://bugzilla.redhat.com/2018809 + local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ") + print(command .. args) + } } # This only supports Python 3.5+ and will never work with Python 2. diff --git a/SPECS/python-rpm-macros.spec b/SPECS/python-rpm-macros.spec index 4290cf0..138b69b 100644 --- a/SPECS/python-rpm-macros.spec +++ b/SPECS/python-rpm-macros.spec @@ -1,10 +1,12 @@ Name: python-rpm-macros Version: 3.9 -Release: 43%{?dist} +Release: 48%{?dist} Summary: The common Python RPM macros URL: https://src.fedoraproject.org/rpms/python-rpm-macros/ -# macros and lua: MIT, compileall2.py: PSFv2 +# macros and lua: MIT +# import_all_modules.py: MIT +# compileall2.py: PSFv2 License: MIT and Python # Macros: @@ -19,6 +21,7 @@ Source201: python.lua # Python code %global compileall2_version 0.7.1 Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py +Source302: import_all_modules.py BuildArch: noarch @@ -54,7 +57,7 @@ Summary: RPM macros for building Python 3 packages # For %%__python3 and %%python3 Requires: python-srpm-macros = %{version}-%{release} -# For %%py_setup +# For %%py_setup and import_all_modules.py Requires: python-rpm-macros = %{version}-%{release} %description -n python3-rpm-macros @@ -75,6 +78,7 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua mkdir -p %{buildroot}%{_rpmconfigdir}/redhat install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ +install -m 644 import_all_modules.py %{buildroot}%{_rpmconfigdir}/redhat/ %check @@ -85,6 +89,7 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %files %{rpmmacrodir}/macros.python %{rpmmacrodir}/macros.pybytecompile +%{_rpmconfigdir}/redhat/import_all_modules.py %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm @@ -96,6 +101,28 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Tue Dec 21 2021 Karolina Surma - 3.9-48 +- Fix CI test configuration, so that pytest can import the package code + +* Wed Dec 08 2021 Miro Hrončok - 3.9-47 +- Set %%__python3 value according to %%python3_pkgversion + I.e. when %%python3_pkgversion is 3.12, %%__python3 is /usr/bin/python3.12 + +* Mon Nov 01 2021 Karolina Surma - 3.9-46 +- Fix multiline arguments processing for %%py_check_import +- Fix %%py_shebang_flags handling within %%py_check_import +- Process .pth files in buildroot's sitedirs in %%py_check_import +- Move import_all_modules.py from python-srpm-macros to python-rpm-macros + +* Mon Oct 25 2021 Karolina Surma - 3.9-45 +- Introduce -f (read from file) option to %%py{3}_check_import +- Introduce -t (filter top-level modules) option to %%py{3}_check_import +- Introduce -e (exclude module globs) option to %%py{3}_check_import + +* Thu Sep 09 2021 Miro Hrončok - 3.9-44 +- Set $RPM_BUILD_ROOT in %%{python3_...} macros + to allow selecting alternate sysconfig install scheme based on that variable + * Wed Aug 11 2021 Tomas Orsava - 3.9-43 - Define a new macros %%python_wheel_dir and %%python_wheel_pkg_prefix - Related: rhbz#1982668