|
|
f992f7 |
diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py
|
|
|
f992f7 |
index 14c9adb..e20104e 100644
|
|
|
f992f7 |
--- a/Lib/ensurepip/__init__.py
|
|
|
f992f7 |
+++ b/Lib/ensurepip/__init__.py
|
|
|
f992f7 |
@@ -7,6 +7,7 @@ import pkgutil
|
|
|
f992f7 |
import shutil
|
|
|
f992f7 |
import sys
|
|
|
f992f7 |
import tempfile
|
|
|
f992f7 |
+from ensurepip import rewheel
|
|
|
f992f7 |
|
|
|
f992f7 |
|
|
|
f992f7 |
__all__ = ["version", "bootstrap"]
|
|
|
f992f7 |
@@ -43,6 +44,8 @@ def _run_pip(args, additional_paths=None):
|
|
|
f992f7 |
|
|
|
f992f7 |
# Install the bundled software
|
|
|
f992f7 |
import pip
|
|
|
f992f7 |
+ if args[0] in ["install", "list", "wheel"]:
|
|
|
f992f7 |
+ args.append('--pre')
|
|
|
f992f7 |
pip.main(args)
|
|
|
f992f7 |
|
|
|
f992f7 |
|
|
|
f992f7 |
@@ -93,21 +96,40 @@ def bootstrap(root=None, upgrade=False, user=False,
|
|
|
f992f7 |
# omit pip and easy_install
|
|
|
f992f7 |
os.environ["ENSUREPIP_OPTIONS"] = "install"
|
|
|
f992f7 |
|
|
|
f992f7 |
+ whls = []
|
|
|
f992f7 |
+ rewheel_dir = None
|
|
|
f992f7 |
+ # try to see if we have system-wide versions of _PROJECTS
|
|
|
f992f7 |
+ dep_records = rewheel.find_system_records([p[0] for p in _PROJECTS])
|
|
|
f992f7 |
+ # TODO: check if system-wide versions are the newest ones
|
|
|
f992f7 |
+ # if --upgrade is used?
|
|
|
f992f7 |
+ if all(dep_records):
|
|
|
f992f7 |
+ # if we have all _PROJECTS installed system-wide, we'll recreate
|
|
|
f992f7 |
+ # wheels from them and install those
|
|
|
f992f7 |
+ rewheel_dir = tempfile.mkdtemp()
|
|
|
f992f7 |
+ for dr in dep_records:
|
|
|
f992f7 |
+ new_whl = rewheel.rewheel_from_record(dr, rewheel_dir)
|
|
|
f992f7 |
+ whls.append(os.path.join(rewheel_dir, new_whl))
|
|
|
f992f7 |
+ else:
|
|
|
f992f7 |
+ # if we don't have all the _PROJECTS installed system-wide,
|
|
|
f992f7 |
+ # let's just fall back to bundled wheels
|
|
|
f992f7 |
+ for project, version in _PROJECTS:
|
|
|
f992f7 |
+ whl = os.path.join(
|
|
|
f992f7 |
+ os.path.dirname(__file__),
|
|
|
f992f7 |
+ "_bundled",
|
|
|
f992f7 |
+ "{}-{}-py2.py3-none-any.whl".format(project, version)
|
|
|
f992f7 |
+ )
|
|
|
f992f7 |
+ whls.append(whl)
|
|
|
f992f7 |
+
|
|
|
f992f7 |
tmpdir = tempfile.mkdtemp()
|
|
|
f992f7 |
try:
|
|
|
f992f7 |
# Put our bundled wheels into a temporary directory and construct the
|
|
|
f992f7 |
# additional paths that need added to sys.path
|
|
|
f992f7 |
additional_paths = []
|
|
|
f992f7 |
- for project, version in _PROJECTS:
|
|
|
f992f7 |
- wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
|
|
|
f992f7 |
- whl = pkgutil.get_data(
|
|
|
f992f7 |
- "ensurepip",
|
|
|
f992f7 |
- "_bundled/{}".format(wheel_name),
|
|
|
f992f7 |
- )
|
|
|
f992f7 |
- with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
|
|
|
f992f7 |
- fp.write(whl)
|
|
|
f992f7 |
-
|
|
|
f992f7 |
- additional_paths.append(os.path.join(tmpdir, wheel_name))
|
|
|
f992f7 |
+ for whl in whls:
|
|
|
f992f7 |
+ shutil.copy(whl, tmpdir)
|
|
|
f992f7 |
+ additional_paths.append(os.path.join(tmpdir, os.path.basename(whl)))
|
|
|
f992f7 |
+ if rewheel_dir:
|
|
|
f992f7 |
+ shutil.rmtree(rewheel_dir)
|
|
|
f992f7 |
|
|
|
f992f7 |
# Construct the arguments to be passed to the pip command
|
|
|
f992f7 |
args = ["install", "--no-index", "--find-links", tmpdir]
|
|
|
f992f7 |
diff --git a/Lib/ensurepip/rewheel/__init__.py b/Lib/ensurepip/rewheel/__init__.py
|
|
|
f992f7 |
new file mode 100644
|
|
|
f992f7 |
index 0000000..75c2094
|
|
|
f992f7 |
--- /dev/null
|
|
|
f992f7 |
+++ b/Lib/ensurepip/rewheel/__init__.py
|
|
|
f992f7 |
@@ -0,0 +1,158 @@
|
|
|
f992f7 |
+import argparse
|
|
|
f992f7 |
+import codecs
|
|
|
f992f7 |
+import csv
|
|
|
f992f7 |
+import email.parser
|
|
|
f992f7 |
+import os
|
|
|
f992f7 |
+import io
|
|
|
f992f7 |
+import re
|
|
|
f992f7 |
+import site
|
|
|
f992f7 |
+import subprocess
|
|
|
f992f7 |
+import sys
|
|
|
f992f7 |
+import zipfile
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+def run():
|
|
|
f992f7 |
+ parser = argparse.ArgumentParser(description='Recreate wheel of package with given RECORD.')
|
|
|
f992f7 |
+ parser.add_argument('record_path',
|
|
|
f992f7 |
+ help='Path to RECORD file')
|
|
|
f992f7 |
+ parser.add_argument('-o', '--output-dir',
|
|
|
f992f7 |
+ help='Dir where to place the wheel, defaults to current working dir.',
|
|
|
f992f7 |
+ dest='outdir',
|
|
|
f992f7 |
+ default=os.path.curdir)
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+ ns = parser.parse_args()
|
|
|
f992f7 |
+ retcode = 0
|
|
|
f992f7 |
+ try:
|
|
|
f992f7 |
+ print(rewheel_from_record(**vars(ns)))
|
|
|
f992f7 |
+ except BaseException as e:
|
|
|
f992f7 |
+ print('Failed: {}'.format(e))
|
|
|
f992f7 |
+ retcode = 1
|
|
|
f992f7 |
+ sys.exit(1)
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+def find_system_records(projects):
|
|
|
f992f7 |
+ """Return list of paths to RECORD files for system-installed projects.
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+ If a project is not installed, the resulting list contains None instead
|
|
|
f992f7 |
+ of a path to its RECORD
|
|
|
f992f7 |
+ """
|
|
|
f992f7 |
+ records = []
|
|
|
f992f7 |
+ # get system site-packages dirs
|
|
|
f992f7 |
+ if hasattr(sys, 'real_prefix'):
|
|
|
f992f7 |
+ #we are in python2 virtualenv and sys.real_prefix is the original sys.prefix
|
|
|
f992f7 |
+ _orig_prefixes = site.PREFIXES
|
|
|
f992f7 |
+ setattr(site, 'PREFIXES', [sys.real_prefix]*2)
|
|
|
f992f7 |
+ sys_sitepack = site.getsitepackages()
|
|
|
f992f7 |
+ setattr(site, 'PREFIXES', _orig_prefixes)
|
|
|
f992f7 |
+ elif hasattr(sys, 'base_prefix'): # python3 venv doesn't inject real_prefix to sys
|
|
|
f992f7 |
+ # we are on python3 and base(_exec)_prefix is unchanged in venv
|
|
|
f992f7 |
+ sys_sitepack = site.getsitepackages([sys.base_prefix, sys.base_exec_prefix])
|
|
|
f992f7 |
+ else:
|
|
|
f992f7 |
+ # we are in python2 without virtualenv
|
|
|
f992f7 |
+ sys_sitepack = site.getsitepackages()
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+ sys_sitepack = [sp for sp in sys_sitepack if os.path.exists(sp)]
|
|
|
f992f7 |
+ # try to find all projects in all system site-packages
|
|
|
f992f7 |
+ for project in projects:
|
|
|
f992f7 |
+ path = None
|
|
|
f992f7 |
+ for sp in sys_sitepack:
|
|
|
f992f7 |
+ dist_info_re = os.path.join(sp, project) + '-[^\{0}]+\.dist-info'.format(os.sep)
|
|
|
f992f7 |
+ candidates = [os.path.join(sp, p) for p in os.listdir(sp)]
|
|
|
f992f7 |
+ # filter out candidate dirs based on the above regexp
|
|
|
f992f7 |
+ filtered = [c for c in candidates if re.match(dist_info_re, c)]
|
|
|
f992f7 |
+ # if we have 0 or 2 or more dirs, something is wrong...
|
|
|
f992f7 |
+ if len(filtered) == 1:
|
|
|
f992f7 |
+ path = filtered[0]
|
|
|
f992f7 |
+ if path is not None:
|
|
|
f992f7 |
+ records.append(os.path.join(path, 'RECORD'))
|
|
|
f992f7 |
+ else:
|
|
|
f992f7 |
+ records.append(None)
|
|
|
f992f7 |
+ return records
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+def rewheel_from_record(record_path, outdir):
|
|
|
f992f7 |
+ """Recreates a whee of package with given record_path and returns path
|
|
|
f992f7 |
+ to the newly created wheel."""
|
|
|
f992f7 |
+ site_dir = os.path.dirname(os.path.dirname(record_path))
|
|
|
f992f7 |
+ record_relpath = record_path[len(site_dir):].strip(os.path.sep)
|
|
|
f992f7 |
+ to_write, to_omit = get_records_to_pack(site_dir, record_relpath)
|
|
|
f992f7 |
+ new_wheel_name = get_wheel_name(record_path)
|
|
|
f992f7 |
+ new_wheel_path = os.path.join(outdir, new_wheel_name + '.whl')
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+ new_wheel = zipfile.ZipFile(new_wheel_path, mode='w', compression=zipfile.ZIP_DEFLATED)
|
|
|
f992f7 |
+ # we need to write a new record with just the files that we will write,
|
|
|
f992f7 |
+ # e.g. not binaries and *.pyc/*.pyo files
|
|
|
f992f7 |
+ if sys.version_info[0] < 3:
|
|
|
f992f7 |
+ new_record = io.BytesIO()
|
|
|
f992f7 |
+ else:
|
|
|
f992f7 |
+ new_record = io.StringIO()
|
|
|
f992f7 |
+ writer = csv.writer(new_record)
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+ # handle files that we can write straight away
|
|
|
f992f7 |
+ for f, sha_hash, size in to_write:
|
|
|
f992f7 |
+ new_wheel.write(os.path.join(site_dir, f), arcname=f)
|
|
|
f992f7 |
+ writer.writerow([f, sha_hash,size])
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+ # rewrite the old wheel file with a new computed one
|
|
|
f992f7 |
+ writer.writerow([record_relpath, '', ''])
|
|
|
f992f7 |
+ new_wheel.writestr(record_relpath, new_record.getvalue())
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+ new_wheel.close()
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+ return new_wheel.filename
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+def get_wheel_name(record_path):
|
|
|
f992f7 |
+ """Return proper name of the wheel, without .whl."""
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+ wheel_info_path = os.path.join(os.path.dirname(record_path), 'WHEEL')
|
|
|
f992f7 |
+ with codecs.open(wheel_info_path, encoding='utf-8') as wheel_info_file:
|
|
|
f992f7 |
+ wheel_info = email.parser.Parser().parsestr(wheel_info_file.read().encode('utf-8'))
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+ metadata_path = os.path.join(os.path.dirname(record_path), 'METADATA')
|
|
|
f992f7 |
+ with codecs.open(metadata_path, encoding='utf-8') as metadata_file:
|
|
|
f992f7 |
+ metadata = email.parser.Parser().parsestr(metadata_file.read().encode('utf-8'))
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+ # construct name parts according to wheel spec
|
|
|
f992f7 |
+ distribution = metadata.get('Name')
|
|
|
f992f7 |
+ version = metadata.get('Version')
|
|
|
f992f7 |
+ build_tag = '' # nothing for now
|
|
|
f992f7 |
+ lang_tag = []
|
|
|
f992f7 |
+ for t in wheel_info.get_all('Tag'):
|
|
|
f992f7 |
+ lang_tag.append(t.split('-')[0])
|
|
|
f992f7 |
+ lang_tag = '.'.join(lang_tag)
|
|
|
f992f7 |
+ abi_tag, plat_tag = wheel_info.get('Tag').split('-')[1:3]
|
|
|
f992f7 |
+ # leave out build tag, if it is empty
|
|
|
f992f7 |
+ to_join = filter(None, [distribution, version, build_tag, lang_tag, abi_tag, plat_tag])
|
|
|
f992f7 |
+ return '-'.join(list(to_join))
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+def get_records_to_pack(site_dir, record_relpath):
|
|
|
f992f7 |
+ """Accepts path of sitedir and path of RECORD file relative to it.
|
|
|
f992f7 |
+ Returns two lists:
|
|
|
f992f7 |
+ - list of files that can be written to new RECORD straight away
|
|
|
f992f7 |
+ - list of files that shouldn't be written or need some processing
|
|
|
f992f7 |
+ (pyc and pyo files, scripts)
|
|
|
f992f7 |
+ """
|
|
|
f992f7 |
+ record_file_path = os.path.join(site_dir, record_relpath)
|
|
|
f992f7 |
+ with codecs.open(record_file_path, encoding='utf-8') as record_file:
|
|
|
f992f7 |
+ record_contents = record_file.read()
|
|
|
f992f7 |
+ # temporary fix for https://github.com/pypa/pip/issues/1376
|
|
|
f992f7 |
+ # we need to ignore files under ".data" directory
|
|
|
f992f7 |
+ data_dir = os.path.dirname(record_relpath).strip(os.path.sep)
|
|
|
f992f7 |
+ data_dir = data_dir[:-len('dist-info')] + 'data'
|
|
|
f992f7 |
+
|
|
|
f992f7 |
+ to_write = []
|
|
|
f992f7 |
+ to_omit = []
|
|
|
f992f7 |
+ for l in record_contents.splitlines():
|
|
|
f992f7 |
+ spl = l.split(',')
|
|
|
f992f7 |
+ if len(spl) == 3:
|
|
|
f992f7 |
+ # new record will omit (or write differently):
|
|
|
f992f7 |
+ # - abs paths, paths with ".." (entry points),
|
|
|
f992f7 |
+ # - pyc+pyo files
|
|
|
f992f7 |
+ # - the old RECORD file
|
|
|
f992f7 |
+ # TODO: is there any better way to recognize an entry point?
|
|
|
f992f7 |
+ if os.path.isabs(spl[0]) or spl[0].startswith('..') or \
|
|
|
f992f7 |
+ spl[0].endswith('.pyc') or spl[0].endswith('.pyo') or \
|
|
|
f992f7 |
+ spl[0] == record_relpath or spl[0].startswith(data_dir):
|
|
|
f992f7 |
+ to_omit.append(spl)
|
|
|
f992f7 |
+ else:
|
|
|
f992f7 |
+ to_write.append(spl)
|
|
|
f992f7 |
+ else:
|
|
|
f992f7 |
+ pass # bad RECORD or empty line
|
|
|
f992f7 |
+ return to_write, to_omit
|
|
|
f992f7 |
diff --git a/Makefile.pre.in b/Makefile.pre.in
|
|
|
f992f7 |
index ca33158..44bdde5 100644
|
|
|
f992f7 |
--- a/Makefile.pre.in
|
|
|
f992f7 |
+++ b/Makefile.pre.in
|
|
|
f992f7 |
@@ -1066,7 +1066,7 @@ LIBSUBDIRS= lib-tk lib-tk/test lib-tk/test/test_tkinter \
|
|
|
f992f7 |
test/tracedmodules \
|
|
|
f992f7 |
encodings compiler hotshot \
|
|
|
f992f7 |
email email/mime email/test email/test/data \
|
|
|
f992f7 |
- ensurepip ensurepip/_bundled \
|
|
|
f992f7 |
+ ensurepip ensurepip/_bundled ensurepip/rewheel\
|
|
|
f992f7 |
json json/tests \
|
|
|
f992f7 |
sqlite3 sqlite3/test \
|
|
|
f992f7 |
logging bsddb bsddb/test csv importlib wsgiref \
|