From 054cd9db81af13b5cdc669067775ff71aa82625f Mon Sep 17 00:00:00 2001
From: Matthew Almond <malmond@fb.com>
Date: Tue, 9 Mar 2021 11:01:40 -0800
Subject: [PATCH 1/2] Use libdnf.utils.checksum_{check,value}
libdnf has the canonical implementation of checksum handling. We aim to
replace all use of dnf.yum.misc.checksum() with this. In doing so, this
fixes installing previously downloaded and transcoded rpms to support
https://github.com/rpm-software-management/librepo/pull/222
This also has some minor performance benefits: librepo's checksum
handling employs caching of previously downloaded files via extended
attributes. This works for ordinary rpms in the dnf cache, but does not
work (yet) for rpm paths specified on the command line due to
https://github.com/rpm-software-management/librepo/issues/233. That
issue is pretty minor, and the fix ends up in libdnf later.
The previous implementation maps all runtime errors to MiscError. We do
this still by taking the libdnf.error.Error class (defined in SWIG) and
map it directly back to the Python exception as before.
---
dnf/package.py | 18 ++++----
dnf/yum/misc.py | 113 ------------------------------------------------
2 files changed, 10 insertions(+), 121 deletions(-)
diff --git a/dnf/package.py b/dnf/package.py
index b01e555eba..fc89cf98a8 100644
--- a/dnf/package.py
+++ b/dnf/package.py
@@ -30,6 +30,8 @@
import dnf.rpm
import dnf.yum.misc
import hawkey
+import libdnf.error
+import libdnf.utils
import logging
import os
import rpm
@@ -56,7 +58,10 @@ def _chksum(self):
return self._priv_chksum
if self._from_cmdline:
chksum_type = dnf.yum.misc.get_default_chksum_type()
- chksum_val = dnf.yum.misc.checksum(chksum_type, self.location)
+ try:
+ chksum_val = libdnf.utils.checksum_value(chksum_type, self.location)
+ except libdnf.error.Error as e:
+ raise dnf.exceptions.MiscError(str(e))
return (hawkey.chksum_type(chksum_type),
binascii.unhexlify(chksum_val))
return super(Package, self).chksum
@@ -330,10 +335,7 @@ def verifyLocalPkg(self):
if self._from_cmdline:
return True # local package always verifies against itself
(chksum_type, chksum) = self.returnIdSum()
- real_sum = dnf.yum.misc.checksum(chksum_type, self.localPkg(),
- datasize=self._size)
- if real_sum != chksum:
- logger.debug(_('%s: %s check failed: %s vs %s'),
- self, chksum_type, real_sum, chksum)
- return False
- return True
+ try:
+ return libdnf.utils.checksum_check(chksum_type, self.localPkg(), chksum)
+ except libdnf.error.Error as e:
+ raise dnf.exceptions.MiscError(str(e))
diff --git a/dnf/yum/misc.py b/dnf/yum/misc.py
index 3e3905feb8..af018a8a1b 100644
--- a/dnf/yum/misc.py
+++ b/dnf/yum/misc.py
@@ -22,7 +22,6 @@
from __future__ import print_function, absolute_import
from __future__ import unicode_literals
-from dnf.exceptions import MiscError
from dnf.pycomp import base64_decodebytes, basestring, unicode
from stat import *
import libdnf.utils
@@ -32,7 +31,6 @@
import dnf.i18n
import errno
import glob
-import hashlib
import io
import os
import os.path
@@ -41,7 +39,6 @@
import shutil
import tempfile
-_available_checksums = set(['md5', 'sha1', 'sha256', 'sha384', 'sha512'])
_default_checksums = ['sha256']
@@ -68,119 +65,9 @@ def re_full_search_needed(s):
return True
return False
-
-class Checksums(object):
- """ Generate checksum(s), on given pieces of data. Producing the
- Length and the result(s) when complete. """
-
- def __init__(self, checksums=None, ignore_missing=False, ignore_none=False):
- if checksums is None:
- checksums = _default_checksums
- self._sumalgos = []
- self._sumtypes = []
- self._len = 0
-
- done = set()
- for sumtype in checksums:
- if sumtype == 'sha':
- sumtype = 'sha1'
- if sumtype in done:
- continue
-
- if sumtype in _available_checksums:
- sumalgo = hashlib.new(sumtype)
- elif ignore_missing:
- continue
- else:
- raise MiscError('Error Checksumming, bad checksum type %s' %
- sumtype)
- done.add(sumtype)
- self._sumtypes.append(sumtype)
- self._sumalgos.append(sumalgo)
- if not done and not ignore_none:
- raise MiscError('Error Checksumming, no valid checksum type')
-
- def __len__(self):
- return self._len
-
- # Note that len(x) is assert limited to INT_MAX, which is 2GB on i686.
- length = property(fget=lambda self: self._len)
-
- def update(self, data):
- self._len += len(data)
- for sumalgo in self._sumalgos:
- data = data.encode('utf-8') if isinstance(data, unicode) else data
- sumalgo.update(data)
-
- def read(self, fo, size=2**16):
- data = fo.read(size)
- self.update(data)
- return data
-
- def hexdigests(self):
- ret = {}
- for sumtype, sumdata in zip(self._sumtypes, self._sumalgos):
- ret[sumtype] = sumdata.hexdigest()
- return ret
-
- def hexdigest(self, checksum=None):
- if checksum is None:
- if not self._sumtypes:
- return None
- checksum = self._sumtypes[0]
- if checksum == 'sha':
- checksum = 'sha1'
- return self.hexdigests()[checksum]
-
- def digests(self):
- ret = {}
- for sumtype, sumdata in zip(self._sumtypes, self._sumalgos):
- ret[sumtype] = sumdata.digest()
- return ret
-
- def digest(self, checksum=None):
- if checksum is None:
- if not self._sumtypes:
- return None
- checksum = self._sumtypes[0]
- if checksum == 'sha':
- checksum = 'sha1'
- return self.digests()[checksum]
-
def get_default_chksum_type():
return _default_checksums[0]
-def checksum(sumtype, file, CHUNK=2**16, datasize=None):
- """takes filename, hand back Checksum of it
- sumtype = md5 or sha/sha1/sha256/sha512 (note sha == sha1)
- filename = /path/to/file
- CHUNK=65536 by default"""
-
- # chunking brazenly lifted from Ryan Tomayko
-
- if isinstance(file, basestring):
- try:
- with open(file, 'rb', CHUNK) as fo:
- return checksum(sumtype, fo, CHUNK, datasize)
- except (IOError, OSError):
- raise MiscError('Error opening file for checksum: %s' % file)
-
- try:
- # assumes file is a file-like-object
- data = Checksums([sumtype])
- while data.read(file, CHUNK):
- if datasize is not None and data.length > datasize:
- break
-
- # This screws up the length, but that shouldn't matter. We only care
- # if this checksum == what we expect.
- if datasize is not None and datasize != data.length:
- return '!%u!%s' % (datasize, data.hexdigest(sumtype))
-
- return data.hexdigest(sumtype)
- except (IOError, OSError) as e:
- raise MiscError('Error reading file for checksum: %s' % file)
-
class GenericHolder(object):
"""Generic Holder class used to hold other objects of known types
It exists purely to be able to do object.somestuff, object.someotherstuff