From 0222c2fd50d22018c6acd70200dbaee6b7f60a10 Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: Aug 01 2017 03:39:01 +0000 Subject: import createrepo-0.9.9-28.el7 --- diff --git a/SOURCES/BZ-1125437-allow-xz-and-bz2-for-xml-files.patch b/SOURCES/BZ-1125437-allow-xz-and-bz2-for-xml-files.patch new file mode 100644 index 0000000..4ba51e2 --- /dev/null +++ b/SOURCES/BZ-1125437-allow-xz-and-bz2-for-xml-files.patch @@ -0,0 +1,163 @@ +commit 2f832212a0ba5fc8d2ac448b0053ae34b134352d +Author: Michal Domonkos +Date: Fri Nov 25 18:39:35 2016 +0100 + + createrepo: allow xz and bz2 for xml files. BZ 1125437 + + Currently, we don't honor --compress-type for the + primary/filelists/other.xml files and always force gz because libxml2 + (used by yum-metadata-parser) didn't use to support anything other than + gz. + + This has been worked around in yum since then by decompressing the files + first before passing them to y-m-p (commit cfe43e8). If we do the same + in createrepo (which uses y-m-p to generate the sqlite files), we can + enable these additional compress types for primary/filelists/other.xml + -- and that's what this commit does. + + Note that libxml2 also natively supports xz in addition to gz so we only + need to do the decompression for bz2. + +diff --git a/createrepo/__init__.py b/createrepo/__init__.py +index 9e89afc..6d56ff8 100644 +--- a/createrepo/__init__.py ++++ b/createrepo/__init__.py +@@ -447,11 +447,10 @@ class MetaDataGenerator: + + def _setupPrimary(self): + # setup the primary metadata file +- # FIXME - make this be conf.compress_type once y-m-p is fixed +- fpz = self.conf.primaryfile + '.' + 'gz' ++ fpz = self.conf.primaryfile + '.' + self.conf.compress_type + primaryfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, + fpz) +- fo = compressOpen(primaryfilepath, 'w', 'gz') ++ fo = compressOpen(primaryfilepath, 'w', self.conf.compress_type) + fo.write('\n') + fo.write('' % +@@ -460,11 +459,10 @@ class MetaDataGenerator: + + def _setupFilelists(self): + # setup the filelist file +- # FIXME - make this be conf.compress_type once y-m-p is fixed +- fpz = self.conf.filelistsfile + '.' + 'gz' ++ fpz = self.conf.filelistsfile + '.' + self.conf.compress_type + filelistpath = os.path.join(self.conf.outputdir, self.conf.tempdir, + fpz) +- fo = compressOpen(filelistpath, 'w', 'gz') ++ fo = compressOpen(filelistpath, 'w', self.conf.compress_type) + fo.write('\n') + fo.write('' % self.pkgcount) +@@ -472,11 +470,10 @@ class MetaDataGenerator: + + def _setupOther(self): + # setup the other file +- # FIXME - make this be conf.compress_type once y-m-p is fixed +- fpz = self.conf.otherfile + '.' + 'gz' ++ fpz = self.conf.otherfile + '.' + self.conf.compress_type + otherfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, + fpz) +- fo = compressOpen(otherfilepath, 'w', 'gz') ++ fo = compressOpen(otherfilepath, 'w', self.conf.compress_type) + fo.write('\n') + fo.write('' % +@@ -1217,21 +1214,34 @@ class MetaDataGenerator: + rp = sqlitecachec.RepodataParserSqlite(repopath, repomd.repoid, None) + + for (rpm_file, ftype) in workfiles: +- # when we fix y-m-p and non-gzipped xml files - then we can make this just add +- # self.conf.compress_type +- if ftype in ('other', 'filelists', 'primary'): +- rpm_file = rpm_file + '.' + 'gz' +- elif rpm_file.find('.') != -1 and rpm_file.split('.')[-1] not in _available_compression: ++ unpath = os.path.join(repopath, rpm_file) ++ if (ftype in ('other', 'filelists', 'primary') ++ or (rpm_file.find('.') != -1 and rpm_file.split('.')[-1] ++ not in _available_compression)): + rpm_file = rpm_file + '.' + self.conf.compress_type + complete_path = os.path.join(repopath, rpm_file) + zfo = compressOpen(complete_path) ++ dfo = None ++ if (self.conf.compress_type == 'bz2' and self.conf.database and ++ ftype in ('other', 'filelists', 'primary')): ++ # yum-metadata-parser doesn't understand bz2 so let's write the ++ # decompressed data to a file and pass that via gen_func ++ # instead of the compressed version ++ dfo = open(unpath, 'w') + # This is misc.checksum() done locally so we can get the size too. + data = misc.Checksums([sumtype]) +- while data.read(zfo, 2**16): +- pass ++ while True: ++ chunk = data.read(zfo, 2**16) ++ if not chunk: ++ break ++ if dfo is not None: ++ dfo.write(chunk) + uncsum = data.hexdigest(sumtype) + unsize = len(data) + zfo.close() ++ if dfo is not None: ++ dfo.close() ++ + csum = misc.checksum(sumtype, complete_path) + timestamp = os.stat(complete_path)[8] + +@@ -1244,21 +1254,29 @@ class MetaDataGenerator: + self.callback.log("Starting %s db creation: %s" % (ftype, + time.ctime())) + ++ gen_func = None + if ftype == 'primary': +- #FIXME - in theory some sort of try/except here +- # TypeError appears to be raised, sometimes :( +- rp.getPrimary(complete_path, csum) +- ++ gen_func = rp.getPrimary + elif ftype == 'filelists': +- #FIXME and here +- rp.getFilelists(complete_path, csum) +- ++ gen_func = rp.getFilelists + elif ftype == 'other': +- #FIXME and here +- rp.getOtherdata(complete_path, csum) ++ gen_func = rp.getOtherdata ++ if gen_func is not None: ++ if dfo is None: ++ #FIXME - in theory some sort of try/except here ++ # TypeError appears to be raised, sometimes :( ++ gen_func(complete_path, csum) ++ else: ++ #FIXME and here ++ gen_func(unpath, uncsum) ++ os.unlink(unpath) + + if ftype in ['primary', 'filelists', 'other']: +- tmp_result_name = '%s.xml.gz.sqlite' % ftype ++ if dfo is None: ++ compress_ext = '.%s' % self.conf.compress_type ++ else: ++ compress_ext = '' ++ tmp_result_name = '%s.xml%s.sqlite' % (ftype, compress_ext) + tmp_result_path = os.path.join(repopath, tmp_result_name) + good_name = '%s.sqlite' % ftype + resultpath = os.path.join(repopath, good_name) +@@ -1323,13 +1341,8 @@ class MetaDataGenerator: + data.openchecksum = (sumtype, uncsum) + + if self.conf.unique_md_filenames: +- if ftype in ('primary', 'filelists', 'other'): +- compress = 'gz' +- else: +- compress = self.conf.compress_type +- + main_name = '.'.join(rpm_file.split('.')[:-1]) +- res_file = '%s-%s.%s' % (csum, main_name, compress) ++ res_file = '%s-%s.%s' % (csum, main_name, self.conf.compress_type) + orig_file = os.path.join(repopath, rpm_file) + dest_file = os.path.join(repopath, res_file) + os.rename(orig_file, dest_file) diff --git a/SOURCES/BZ-1227782-modifyrepo-fix-already-compressed-input.patch b/SOURCES/BZ-1227782-modifyrepo-fix-already-compressed-input.patch new file mode 100644 index 0000000..b238bb1 --- /dev/null +++ b/SOURCES/BZ-1227782-modifyrepo-fix-already-compressed-input.patch @@ -0,0 +1,72 @@ +diff -up createrepo-0.9.9/modifyrepo.py.orig createrepo-0.9.9/modifyrepo.py +--- createrepo-0.9.9/modifyrepo.py.orig 2016-12-08 16:03:27.747862638 +0100 ++++ createrepo-0.9.9/modifyrepo.py 2016-12-08 16:04:15.499625562 +0100 +@@ -31,11 +31,12 @@ import sys + from createrepo import __version__ + from createrepo.utils import checksum_and_rename, compressOpen, MDError + from createrepo.utils import _available_compression +-from yum.misc import checksum ++from yum.misc import checksum, AutoFileChecksums + + from yum.repoMDObject import RepoMD, RepoMDError, RepoData + from xml.dom import minidom + from optparse import OptionParser ++from cStringIO import StringIO + + + class RepoMetadata: +@@ -96,29 +97,29 @@ class RepoMetadata: + if isinstance(metadata, minidom.Document): + md = metadata.toxml() + mdname = 'updateinfo.xml' ++ oldmd = AutoFileChecksums(StringIO(md), [self.checksum_type]) ++ oldmd.read() + elif isinstance(metadata, str): + if os.path.exists(metadata): +- if metadata.split('.')[-1] in ('gz', 'bz2', 'xz'): ++ mdname = os.path.basename(metadata) ++ if mdname.split('.')[-1] in ('gz', 'bz2', 'xz'): ++ mdname = mdname.rsplit('.', 1)[0] + oldmd = compressOpen(metadata, mode='rb') + else: + oldmd = file(metadata, 'r') ++ oldmd = AutoFileChecksums(oldmd, [self.checksum_type]) + md = oldmd.read() + oldmd.close() +- mdname = os.path.basename(metadata) + else: + raise MDError, '%s not found' % metadata + else: + raise MDError, 'invalid metadata type' + +- do_compress = False + ## Compress the metadata and move it into the repodata +- if self.compress and mdname.split('.')[-1] not in ('gz', 'bz2', 'xz'): +- do_compress = True +- mdname += '.' + self.compress_type + mdtype = self._get_mdtype(mdname, mdtype) +- + destmd = os.path.join(self.repodir, mdname) +- if do_compress: ++ if self.compress: ++ destmd += '.' + self.compress_type + newmd = compressOpen(destmd, mode='wb', compress_type=self.compress_type) + else: + newmd = open(destmd, 'wb') +@@ -127,7 +128,6 @@ class RepoMetadata: + newmd.close() + print "Wrote:", destmd + +- open_csum = checksum(self.checksum_type, metadata) + if self.unique_md_filenames: + csum, destmd = checksum_and_rename(destmd, self.checksum_type) + else: +@@ -141,7 +141,7 @@ class RepoMetadata: + new_rd.type = mdtype + new_rd.location = (None, 'repodata/' + base_destmd) + new_rd.checksum = (self.checksum_type, csum) +- new_rd.openchecksum = (self.checksum_type, open_csum) ++ new_rd.openchecksum = (self.checksum_type, oldmd.checksums.hexdigests().popitem()[1]) + new_rd.size = str(os.stat(destmd).st_size) + new_rd.timestamp = str(int(os.stat(destmd).st_mtime)) + self.repoobj.repoData[new_rd.type] = new_rd diff --git a/SOURCES/BZ-1287685-modifyrepo-handle-empty-file-lzma.patch b/SOURCES/BZ-1287685-modifyrepo-handle-empty-file-lzma.patch new file mode 100644 index 0000000..303016c --- /dev/null +++ b/SOURCES/BZ-1287685-modifyrepo-handle-empty-file-lzma.patch @@ -0,0 +1,24 @@ +commit c626c54074b36edb090254fe6f4985bc20893436 +Author: Michal Domonkos +Date: Thu Mar 23 16:05:16 2017 +0100 + + modifyrepo: handle empty file with LZMA. BZ 1287685 + + When trying to compress an empty string with LZMA, we will get the + unfriendly "LZMA.error: unknown error!". Let's handle this case + ourselves and raise a more user-friendly error instead. + +diff --git a/modifyrepo.py b/modifyrepo.py +index 34b0902..ade5607 100755 +--- a/modifyrepo.py ++++ b/modifyrepo.py +@@ -125,6 +125,9 @@ class RepoMetadata: + else: + raise MDError, 'invalid metadata type' + ++ if not md and self.compress_type == 'xz': ++ raise MDError, 'LZMA does not support compressing empty files' ++ + ## Compress the metadata and move it into the repodata + mdtype = self._get_mdtype(mdname, mdtype) + destmd = os.path.join(self.repodir, mdname) diff --git a/SOURCES/BZ-1287714-modifyrepo-docs-remove-compat.patch b/SOURCES/BZ-1287714-modifyrepo-docs-remove-compat.patch new file mode 100644 index 0000000..da0c75b --- /dev/null +++ b/SOURCES/BZ-1287714-modifyrepo-docs-remove-compat.patch @@ -0,0 +1,12 @@ +diff -up createrepo-0.9.9/docs/modifyrepo.1.orig createrepo-0.9.9/docs/modifyrepo.1 +--- createrepo-0.9.9/docs/modifyrepo.1.orig 2017-03-24 13:57:07.282663747 +0100 ++++ createrepo-0.9.9/docs/modifyrepo.1 2017-03-24 13:57:25.116548403 +0100 +@@ -24,7 +24,7 @@ Compress the new repodata before adding + Do not compress the new repodata before adding it to the repo. + + .IP "\fB\-\-compress-type \fP" +-Specify which compression type to use: compat (default), xz (may not be available), gz, bz2. ++Specify which compression type to use: gz (default), xz (may not be available), bz2. + + .IP "\fB\-s, \-\-checksum \fP" + Specify the checksum type to use. diff --git a/SOURCES/BZ-1404239-fix-update-for-same-nevra.patch b/SOURCES/BZ-1404239-fix-update-for-same-nevra.patch new file mode 100644 index 0000000..e6a1fa2 --- /dev/null +++ b/SOURCES/BZ-1404239-fix-update-for-same-nevra.patch @@ -0,0 +1,21 @@ +diff -up createrepo-0.9.9/createrepo/readMetadata.py.orig createrepo-0.9.9/createrepo/readMetadata.py +--- createrepo-0.9.9/createrepo/readMetadata.py.orig 2017-03-02 13:03:29.417076819 +0100 ++++ createrepo-0.9.9/createrepo/readMetadata.py 2017-03-02 13:03:30.420070445 +0100 +@@ -107,7 +107,16 @@ class MetadataIndex(object): + """ + if relpath in self.pkg_tups_by_path: + pkgtup = self.pkg_tups_by_path[relpath] +- return self._repo.sack.searchPkgTuple(pkgtup)[0] ++ pos = self._repo.sack.searchPkgTuple(pkgtup) ++ if len(pos) == 1: ++ return pos[0] ++ elif len(pos) > 1: ++ # Multiple matches for this pkgtup so look at their relpath ++ if self.opts.get('verbose'): ++ print _("Warning: Duplicate nevra detected for %s") % relpath ++ for po in pos: ++ if po.relativepath == relpath: ++ return po + return None + + def cleanup(self): diff --git a/SOURCES/BZ-1406418-cleanup-temporary-files-in-var-tmp-on-exit.patch b/SOURCES/BZ-1406418-cleanup-temporary-files-in-var-tmp-on-exit.patch new file mode 100644 index 0000000..50391c6 --- /dev/null +++ b/SOURCES/BZ-1406418-cleanup-temporary-files-in-var-tmp-on-exit.patch @@ -0,0 +1,106 @@ +diff -up createrepo-0.9.9/createrepo/__init__.py.orig createrepo-0.9.9/createrepo/__init__.py +--- createrepo-0.9.9/createrepo/__init__.py.orig 2017-02-20 14:53:58.885997101 +0100 ++++ createrepo-0.9.9/createrepo/__init__.py 2017-02-20 14:54:06.242935946 +0100 +@@ -1221,19 +1221,19 @@ class MetaDataGenerator: + msg = _('Could not restore old non-metadata file: %s -> %s') % (oldfile, finalfile) + msg += _('Error was %s') % e + raise MDError, msg ++ self._write_out_read_pkgs_list() ++ + ++ def cleanup(self): + self._cleanup_tmp_repodata_dir() +- self._cleanup_update_tmp_dir() +- self._write_out_read_pkgs_list() ++ self._cleanup_update_tmp_dir() + + + def _cleanup_update_tmp_dir(self): +- if not self.conf.update: +- return +- +- shutil.rmtree(self.oldData._repo.basecachedir, ignore_errors=True) +- shutil.rmtree(self.oldData._repo.base_persistdir, ignore_errors=True) +- ++ if self.conf.update and hasattr(self, 'oldData'): ++ self.oldData.cleanup() ++ ++ + def _write_out_read_pkgs_list(self): + # write out the read_pkgs_list file with self.read_pkgs + if self.conf.read_pkgs_list: +diff -up createrepo-0.9.9/createrepo/merge.py.orig createrepo-0.9.9/createrepo/merge.py +--- createrepo-0.9.9/createrepo/merge.py.orig 2017-02-20 14:53:58.884997110 +0100 ++++ createrepo-0.9.9/createrepo/merge.py 2017-02-20 14:54:06.241935954 +0100 +@@ -146,3 +146,4 @@ class RepoMergeBase: + mdgen.doPkgMetadata() + mdgen.doRepoMetadata() + mdgen.doFinalMove() ++ mdgen.cleanup() +diff -up createrepo-0.9.9/createrepo/readMetadata.py.orig createrepo-0.9.9/createrepo/readMetadata.py +--- createrepo-0.9.9/createrepo/readMetadata.py.orig 2017-02-20 14:53:58.877997168 +0100 ++++ createrepo-0.9.9/createrepo/readMetadata.py 2017-02-20 14:54:06.241935954 +0100 +@@ -16,6 +16,7 @@ + # Copyright 2006 Red Hat + + import os ++import shutil + import stat + from utils import errorprint, _ + +@@ -109,7 +110,11 @@ class MetadataIndex(object): + return self._repo.sack.searchPkgTuple(pkgtup)[0] + return None + +- ++ def cleanup(self): ++ """Delete temporary files in /var/tmp.""" ++ shutil.rmtree(self._repo.basecachedir, ignore_errors=True) ++ shutil.rmtree(self._repo.base_persistdir, ignore_errors=True) ++ + + if __name__ == "__main__": + cwd = os.getcwd() +@@ -122,4 +127,5 @@ if __name__ == "__main__": + print po.xml_dump_primary_metadata() + print po.xml_dump_filelists_metadata() + print po.xml_dump_other_metadata() ++ idx.cleanup() + +diff -up createrepo-0.9.9/genpkgmetadata.py.orig createrepo-0.9.9/genpkgmetadata.py +--- createrepo-0.9.9/genpkgmetadata.py.orig 2017-02-20 14:53:58.889997068 +0100 ++++ createrepo-0.9.9/genpkgmetadata.py 2017-02-20 14:54:06.241935954 +0100 +@@ -249,6 +249,7 @@ def main(args): + print ('start time: %0.3f' % (time.time() - start_st)) + + mid_st = time.time() ++ mdgen = None + try: + if conf.split: + mdgen = createrepo.SplitMetaDataGenerator(config_obj=conf, +@@ -277,12 +278,25 @@ def main(args): + mdgen.doFinalMove() + if conf.profile: + print ('fm time: %0.3f' % (time.time() - fm_st)) ++ cl_st = time.time() ++ mdgen.cleanup() ++ if conf.profile: ++ print ('cl time: %0.3f' % (time.time() - cl_st)) + + + except MDError, errormsg: + errorprint(_('%s') % errormsg) ++ # cleanup ++ tmp = os.path.join(conf.outputdir, conf.tempdir) ++ if os.path.exists(tmp): ++ for name in os.listdir(tmp): ++ os.unlink(os.path.join(tmp, name)) ++ os.rmdir(tmp) + sys.exit(1) + ++ finally: ++ if mdgen: ++ mdgen.cleanup() + + if __name__ == "__main__": + if len(sys.argv) > 1: diff --git a/SPECS/createrepo.spec b/SPECS/createrepo.spec index 3b1f2fe..21d923e 100644 --- a/SPECS/createrepo.spec +++ b/SPECS/createrepo.spec @@ -15,7 +15,7 @@ BuildRequires: bash-completion Summary: Creates a common metadata repository Name: createrepo Version: 0.9.9 -Release: 26%{?dist} +Release: 28%{?dist} License: GPLv2 Group: System Environment/Base Source: %{name}-%{version}.tar.gz @@ -27,6 +27,12 @@ Patch4: BZ-1178763-duplicates.patch Patch5: BZ-1270784-checksum-prefix.patch Patch6: BZ-1083182-traceback-on-nonexisting-pkglist.patch Patch7: BZ-1285761-modifyrepo-mergerepo-options-manpage.patch +Patch8: BZ-1406418-cleanup-temporary-files-in-var-tmp-on-exit.patch +Patch9: BZ-1404239-fix-update-for-same-nevra.patch +Patch10: BZ-1125437-allow-xz-and-bz2-for-xml-files.patch +Patch11: BZ-1227782-modifyrepo-fix-already-compressed-input.patch +Patch12: BZ-1287685-modifyrepo-handle-empty-file-lzma.patch +Patch13: BZ-1287714-modifyrepo-docs-remove-compat.patch URL: http://createrepo.baseurl.org/ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) BuildArchitectures: noarch @@ -48,6 +54,12 @@ packages. %patch5 -p1 %patch6 -p1 %patch7 -p1 +%patch8 -p1 +%patch9 -p1 +%patch10 -p1 +%patch11 -p1 +%patch12 -p1 +%patch13 -p1 %build @@ -72,6 +84,22 @@ rm -rf $RPM_BUILD_ROOT %{python_sitelib}/createrepo %changelog +* Fri Mar 24 2017 Valentina Mukhamedzhanova - 0.9.9-28 +- createrepo: allow xz and bz2 for xml files. +- Resolves: bug#1125437 +- modifyrepo: fix handling of already compressed input. +- Resolves: bug#1227782 +- modifyrepo: handle empty file with LZMA. +- Resolves: bug#1287685 +- modifyrepo: docs: remove compat compress type. +- Resolves: bug#1287714 + +* Thu Mar 02 2017 Valentina Mukhamedzhanova - 0.9.9-27 +- Fix temporary files cleanup. +- Resolves: bug#1406418 +- Fix --update for pkgs with same nevra. +- Resolves: bug#1404239 + * Mon Feb 29 2016 Valentina Mukhamedzhanova - 0.9.9-26 - Make sure filename doesn't already contain a checksum. - Resolves: bug#1270784