filbranden / rpms / dnf

Forked from rpms/dnf 4 years ago
Clone

Blame SOURCES/0005-Change-synchronization-of-rpm-transaction-to-swdb-RhBug1737328.patch

1f580a
From f6ffdce83004acdb2325d966ad348dfe63fb55f8 Mon Sep 17 00:00:00 2001
1f580a
From: Jaroslav Mracek <jmracek@redhat.com>
1f580a
Date: Mon, 3 Jun 2019 19:26:55 +0200
1f580a
Subject: [PATCH 1/5] [transaction] Set an error for first tsi in unknown state
1f580a
 (RhBug:1737328)
1f580a
1f580a
It should prevent "TransactionItem state is not set" in case of multiple
1f580a
same NEVRASs and single with an error.
1f580a
1f580a
https://bugzilla.redhat.com/show_bug.cgi?id=1737328
1f580a
---
1f580a
 dnf/yum/rpmtrans.py | 4 ++++
1f580a
 1 file changed, 4 insertions(+)
1f580a
1f580a
diff --git a/dnf/yum/rpmtrans.py b/dnf/yum/rpmtrans.py
1f580a
index 6a60b5a08..50d8b81cb 100644
1f580a
--- a/dnf/yum/rpmtrans.py
1f580a
+++ b/dnf/yum/rpmtrans.py
1f580a
@@ -386,6 +386,10 @@ class RPMTransaction(object):
1f580a
         msg = "Error unpacking rpm package %s" % tsi.pkg
1f580a
         for display in self.displays:
1f580a
             display.error(msg)
1f580a
+        for tsi1 in transaction_list:
1f580a
+            if tsi1.state == libdnf.transaction.TransactionItemState_UNKNOWN:
1f580a
+                tsi1.state = libdnf.transaction.TransactionItemState_ERROR
1f580a
+                return
1f580a
         tsi.state = libdnf.transaction.TransactionItemState_ERROR
1f580a
 
1f580a
     def _scriptError(self, amount, total, key):
1f580a
-- 
1f580a
2.21.0
1f580a
1f580a
1f580a
From c07beb11b8f085a4ced74502c70d48b4db4e4099 Mon Sep 17 00:00:00 2001
1f580a
From: Daniel Mach <dmach@redhat.com>
1f580a
Date: Wed, 5 Jun 2019 08:29:03 +0200
1f580a
Subject: [PATCH 2/5] [history] Don't store failed transactions as succeeded.
1f580a
 (RhBug:1737328)
1f580a
1f580a
---
1f580a
 dnf/base.py       |  7 ++++---
1f580a
 dnf/db/history.py | 16 ++++++++++------
1f580a
 2 files changed, 14 insertions(+), 9 deletions(-)
1f580a
1f580a
diff --git a/dnf/base.py b/dnf/base.py
1f580a
index 7a388ecf4..add64036b 100644
1f580a
--- a/dnf/base.py
1f580a
+++ b/dnf/base.py
1f580a
@@ -987,7 +987,7 @@ class Base(object):
1f580a
             # particular element failed and if not, decide that is the
1f580a
             # case.
1f580a
             failed = [el for el in self._ts if el.Failed()]
1f580a
-            if len(failed) > 0:
1f580a
+            if failed:
1f580a
                 for te in failed:
1f580a
                     te_nevra = dnf.util._te_nevra(te)
1f580a
                     for tsi in self._transaction:
1f580a
@@ -996,6 +996,7 @@ class Base(object):
1f580a
 
1f580a
                 errstring = _('Errors occurred during transaction.')
1f580a
                 logger.debug(errstring)
1f580a
+                self.history.end(rpmdbv)
1f580a
             else:
1f580a
                 login = dnf.util.get_effective_login()
1f580a
                 msg = _("Failed to obtain the transaction lock "
1f580a
@@ -1006,7 +1007,7 @@ class Base(object):
1f580a
         else:
1f580a
             if self._record_history():
1f580a
                 herrors = [ucd(x) for x in errors]
1f580a
-                self.history.end(rpmdbv, 2, errors=herrors)
1f580a
+                self.history.end(rpmdbv)
1f580a
 
1f580a
             logger.critical(_("Transaction couldn't start:"))
1f580a
             for e in errors:
1f580a
@@ -1069,7 +1070,7 @@ class Base(object):
1f580a
             count = display_banner(tsi.pkg, count)
1f580a
 
1f580a
         rpmdbv = rpmdb_sack._rpmdb_version()
1f580a
-        self.history.end(rpmdbv, 0)
1f580a
+        self.history.end(rpmdbv)
1f580a
 
1f580a
         timer()
1f580a
         self._trans_success = True
1f580a
diff --git a/dnf/db/history.py b/dnf/db/history.py
1f580a
index dc9c53b70..7f22f97c1 100644
1f580a
--- a/dnf/db/history.py
1f580a
+++ b/dnf/db/history.py
1f580a
@@ -486,14 +486,18 @@ class SwdbInterface(object):
1f580a
             self.swdb.log_error(self._tid, error)
1f580a
     '''
1f580a
 
1f580a
-    # TODO: rename to end_transaction?
1f580a
-    def end(self, end_rpmdb_version="", return_code=0, errors=None):
1f580a
-        assert return_code or not errors
1f580a
-        # TODO: fix return_code
1f580a
-        return_code = not bool(return_code)
1f580a
+    def end(self, end_rpmdb_version="", return_code=None, errors=None):
1f580a
         if not hasattr(self, '_tid'):
1f580a
             return  # Failed at beg() time
1f580a
 
1f580a
+        if return_code is None:
1f580a
+            # return_code/state auto-detection
1f580a
+            return_code = libdnf.transaction.TransactionState_DONE
1f580a
+            for tsi in self.rpm:
1f580a
+                if tsi.state == libdnf.transaction.TransactionItemState_ERROR:
1f580a
+                    return_code = libdnf.transaction.TransactionState_ERROR
1f580a
+                    break
1f580a
+
1f580a
         for file_descriptor, line in self._output:
1f580a
             self.swdb.addConsoleOutputLine(file_descriptor, line)
1f580a
         self._output = []
1f580a
@@ -501,7 +505,7 @@ class SwdbInterface(object):
1f580a
         self.swdb.endTransaction(
1f580a
             int(time.time()),
1f580a
             str(end_rpmdb_version),
1f580a
-            bool(return_code)
1f580a
+            return_code,
1f580a
         )
1f580a
 
1f580a
         # Closing and cleanup is done in the close() method.
1f580a
-- 
1f580a
2.21.0
1f580a
1f580a
1f580a
From 5302f0fbf977e6447b5a00adabb34b1c01fe7e55 Mon Sep 17 00:00:00 2001
1f580a
From: Daniel Mach <dmach@redhat.com>
1f580a
Date: Wed, 5 Jun 2019 21:10:05 +0200
1f580a
Subject: [PATCH 3/5] [transaction] Add RPMCALLBACK_INST_{START,STOP} callback
1f580a
 handlers. (RhBug:1737328)
1f580a
1f580a
---
1f580a
 dnf/yum/rpmtrans.py | 23 ++++++++++++++++++-----
1f580a
 1 file changed, 18 insertions(+), 5 deletions(-)
1f580a
1f580a
diff --git a/dnf/yum/rpmtrans.py b/dnf/yum/rpmtrans.py
1f580a
index 50d8b81cb..0834a8c82 100644
1f580a
--- a/dnf/yum/rpmtrans.py
1f580a
+++ b/dnf/yum/rpmtrans.py
1f580a
@@ -262,6 +262,10 @@ class RPMTransaction(object):
1f580a
                 return self._instOpenFile(key)
1f580a
             elif what == rpm.RPMCALLBACK_INST_CLOSE_FILE:
1f580a
                 self._instCloseFile(key)
1f580a
+            elif what == rpm.RPMCALLBACK_INST_START:
1f580a
+                self._inst_start(key)
1f580a
+            elif what == rpm.RPMCALLBACK_INST_STOP:
1f580a
+                self._inst_stop(key)
1f580a
             elif what == rpm.RPMCALLBACK_INST_PROGRESS:
1f580a
                 self._instProgress(amount, total, key)
1f580a
             elif what == rpm.RPMCALLBACK_UNINST_START:
1f580a
@@ -321,12 +325,17 @@ class RPMTransaction(object):
1f580a
             return self.fd.fileno()
1f580a
 
1f580a
     def _instCloseFile(self, key):
1f580a
-        transaction_list = self._extract_cbkey(key)
1f580a
         self.fd.close()
1f580a
         self.fd = None
1f580a
 
1f580a
+    def _inst_start(self, key):
1f580a
+        pass
1f580a
+
1f580a
+    def _inst_stop(self, key):
1f580a
         if self.test or not self.trans_running:
1f580a
             return
1f580a
+
1f580a
+        transaction_list = self._extract_cbkey(key)
1f580a
         for tsi in transaction_list:
1f580a
             if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
1f580a
                 tsi.state = libdnf.transaction.TransactionItemState_DONE
1f580a
@@ -376,6 +385,10 @@ class RPMTransaction(object):
1f580a
 
1f580a
     def _cpioError(self, key):
1f580a
         transaction_list = self._extract_cbkey(key)
1f580a
+        for tsi in transaction_list:
1f580a
+            if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
1f580a
+                tsi.state = libdnf.transaction.TransactionItemState_ERROR
1f580a
+                break
1f580a
         msg = "Error in cpio payload of rpm package %s" % transaction_list[0].pkg
1f580a
         for display in self.displays:
1f580a
             display.error(msg)
1f580a
@@ -386,11 +399,11 @@ class RPMTransaction(object):
1f580a
         msg = "Error unpacking rpm package %s" % tsi.pkg
1f580a
         for display in self.displays:
1f580a
             display.error(msg)
1f580a
-        for tsi1 in transaction_list:
1f580a
-            if tsi1.state == libdnf.transaction.TransactionItemState_UNKNOWN:
1f580a
-                tsi1.state = libdnf.transaction.TransactionItemState_ERROR
1f580a
+
1f580a
+        for tsi in transaction_list:
1f580a
+           if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
1f580a
+                tsi.state = libdnf.transaction.TransactionItemState_ERROR
1f580a
                 return
1f580a
-        tsi.state = libdnf.transaction.TransactionItemState_ERROR
1f580a
 
1f580a
     def _scriptError(self, amount, total, key):
1f580a
         # "amount" carries the failed scriptlet tag,
1f580a
-- 
1f580a
2.21.0
1f580a
1f580a
1f580a
From a3cc8343ff418cca386a0a11f331ebe08e70bae6 Mon Sep 17 00:00:00 2001
1f580a
From: Jaroslav Mracek <jmracek@redhat.com>
1f580a
Date: Thu, 13 Jun 2019 12:02:07 +0200
1f580a
Subject: [PATCH 4/5] Change synchronization of rpm transaction to swdb
1f580a
 (RhBug:1737328)
1f580a
1f580a
Originally the state of transaction element was changed by callbacks,
1f580a
but this approach cannot ensure that all elements will be marked
1f580a
correctly in case of error when rpm could skip some of elements.
1f580a
1f580a
The new approach only synchronize the states of all elements in
1f580a
transaction after transaction is finished.
1f580a
1f580a
https://bugzilla.redhat.com/show_bug.cgi?id=1737328
1f580a
---
1f580a
 dnf/base.py         | 21 +++++----------------
1f580a
 dnf/util.py         | 36 ++++++++++++++++++++++++++++++++++++
1f580a
 dnf/yum/rpmtrans.py | 22 +++-------------------
1f580a
 3 files changed, 44 insertions(+), 35 deletions(-)
1f580a
1f580a
diff --git a/dnf/base.py b/dnf/base.py
1f580a
index add64036b..237ad397d 100644
1f580a
--- a/dnf/base.py
1f580a
+++ b/dnf/base.py
1f580a
@@ -978,6 +978,7 @@ class Base(object):
1f580a
                 os.nice(onice)
1f580a
             except:
1f580a
                 pass
1f580a
+        dnf.util._sync_rpm_trans_with_swdb(self._ts, self._transaction)
1f580a
 
1f580a
         if errors is None:
1f580a
             pass
1f580a
@@ -987,17 +988,7 @@ class Base(object):
1f580a
             # particular element failed and if not, decide that is the
1f580a
             # case.
1f580a
             failed = [el for el in self._ts if el.Failed()]
1f580a
-            if failed:
1f580a
-                for te in failed:
1f580a
-                    te_nevra = dnf.util._te_nevra(te)
1f580a
-                    for tsi in self._transaction:
1f580a
-                        if str(tsi) == te_nevra:
1f580a
-                            tsi.state = libdnf.transaction.TransactionItemState_ERROR
1f580a
-
1f580a
-                errstring = _('Errors occurred during transaction.')
1f580a
-                logger.debug(errstring)
1f580a
-                self.history.end(rpmdbv)
1f580a
-            else:
1f580a
+            if not failed:
1f580a
                 login = dnf.util.get_effective_login()
1f580a
                 msg = _("Failed to obtain the transaction lock "
1f580a
                         "(logged in as: %s).")
1f580a
@@ -1005,13 +996,11 @@ class Base(object):
1f580a
                 msg = _('Could not run transaction.')
1f580a
                 raise dnf.exceptions.Error(msg)
1f580a
         else:
1f580a
-            if self._record_history():
1f580a
-                herrors = [ucd(x) for x in errors]
1f580a
-                self.history.end(rpmdbv)
1f580a
-
1f580a
             logger.critical(_("Transaction couldn't start:"))
1f580a
             for e in errors:
1f580a
-                logger.critical(e[0])  # should this be 'to_unicoded'?
1f580a
+                logger.critical(ucd(e[0]))
1f580a
+            if self._record_history() and not self._ts.isTsFlagSet(rpm.RPMTRANS_FLAG_TEST):
1f580a
+                self.history.end(rpmdbv)
1f580a
             msg = _("Could not run transaction.")
1f580a
             raise dnf.exceptions.Error(msg)
1f580a
 
1f580a
diff --git a/dnf/util.py b/dnf/util.py
1f580a
index 2f1abe626..9d0288e5a 100644
1f580a
--- a/dnf/util.py
1f580a
+++ b/dnf/util.py
1f580a
@@ -396,6 +396,42 @@ def _te_nevra(te):
1f580a
     return nevra + te.V() + '-' + te.R() + '.' + te.A()
1f580a
 
1f580a
 
1f580a
+def _sync_rpm_trans_with_swdb(rpm_transaction, swdb_transaction):
1f580a
+    revert_actions = {libdnf.transaction.TransactionItemAction_DOWNGRADED,
1f580a
+                      libdnf.transaction.TransactionItemAction_OBSOLETED,
1f580a
+                      libdnf.transaction.TransactionItemAction_REMOVE,
1f580a
+                      libdnf.transaction.TransactionItemAction_UPGRADED,
1f580a
+                      libdnf.transaction.TransactionItemAction_REINSTALLED}
1f580a
+    cached_tsi = [tsi for tsi in swdb_transaction]
1f580a
+    el_not_found = False
1f580a
+    error = False
1f580a
+    for rpm_el in rpm_transaction:
1f580a
+        te_nevra = _te_nevra(rpm_el)
1f580a
+        tsi = rpm_el.Key()
1f580a
+        if tsi is None or not hasattr(tsi, "pkg"):
1f580a
+            for tsi_candidate in cached_tsi:
1f580a
+                if tsi_candidate.state != libdnf.transaction.TransactionItemState_UNKNOWN:
1f580a
+                    continue
1f580a
+                if tsi_candidate.action not in revert_actions:
1f580a
+                    continue
1f580a
+                if str(tsi_candidate) == te_nevra:
1f580a
+                    tsi = tsi_candidate
1f580a
+                    break
1f580a
+        if tsi is None or not hasattr(tsi, "pkg"):
1f580a
+            logger.critical(_("TransactionItem not found for key: {}").format(te_nevra))
1f580a
+            el_not_found = True
1f580a
+            continue
1f580a
+        if rpm_el.Failed():
1f580a
+            tsi.state = libdnf.transaction.TransactionItemState_ERROR
1f580a
+            error = True
1f580a
+        else:
1f580a
+            tsi.state = libdnf.transaction.TransactionItemState_DONE
1f580a
+    if error:
1f580a
+        logger.debug(_('Errors occurred during transaction.'))
1f580a
+
1f580a
+    return el_not_found
1f580a
+
1f580a
+
1f580a
 class tmpdir(object):
1f580a
     def __init__(self):
1f580a
         prefix = '%s-' % dnf.const.PREFIX
1f580a
diff --git a/dnf/yum/rpmtrans.py b/dnf/yum/rpmtrans.py
1f580a
index 0834a8c82..8a5bd4731 100644
1f580a
--- a/dnf/yum/rpmtrans.py
1f580a
+++ b/dnf/yum/rpmtrans.py
1f580a
@@ -336,10 +336,7 @@ class RPMTransaction(object):
1f580a
             return
1f580a
 
1f580a
         transaction_list = self._extract_cbkey(key)
1f580a
-        for tsi in transaction_list:
1f580a
-            if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
1f580a
-                tsi.state = libdnf.transaction.TransactionItemState_DONE
1f580a
-                break
1f580a
+        tsi = transaction_list[0]
1f580a
 
1f580a
         for display in self.displays:
1f580a
             display.filelog(tsi.pkg, tsi.action)
1f580a
@@ -370,10 +367,7 @@ class RPMTransaction(object):
1f580a
 
1f580a
     def _unInstStop(self, key):
1f580a
         transaction_list = self._extract_cbkey(key)
1f580a
-        for tsi in transaction_list:
1f580a
-            if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
1f580a
-                tsi.state = libdnf.transaction.TransactionItemState_DONE
1f580a
-                break
1f580a
+        tsi = transaction_list[0]
1f580a
 
1f580a
         for display in self.displays:
1f580a
             display.filelog(tsi.pkg, tsi.action)
1f580a
@@ -385,26 +379,16 @@ class RPMTransaction(object):
1f580a
 
1f580a
     def _cpioError(self, key):
1f580a
         transaction_list = self._extract_cbkey(key)
1f580a
-        for tsi in transaction_list:
1f580a
-            if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
1f580a
-                tsi.state = libdnf.transaction.TransactionItemState_ERROR
1f580a
-                break
1f580a
         msg = "Error in cpio payload of rpm package %s" % transaction_list[0].pkg
1f580a
         for display in self.displays:
1f580a
             display.error(msg)
1f580a
 
1f580a
     def _unpackError(self, key):
1f580a
         transaction_list = self._extract_cbkey(key)
1f580a
-        tsi = transaction_list[0]
1f580a
-        msg = "Error unpacking rpm package %s" % tsi.pkg
1f580a
+        msg = "Error unpacking rpm package %s" % transaction_list[0].pkg
1f580a
         for display in self.displays:
1f580a
             display.error(msg)
1f580a
 
1f580a
-        for tsi in transaction_list:
1f580a
-           if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
1f580a
-                tsi.state = libdnf.transaction.TransactionItemState_ERROR
1f580a
-                return
1f580a
-
1f580a
     def _scriptError(self, amount, total, key):
1f580a
         # "amount" carries the failed scriptlet tag,
1f580a
         # "total" carries fatal/non-fatal status
1f580a
-- 
1f580a
2.21.0
1f580a
1f580a
1f580a
From 91aa07fe7b7f01727f2eb5d231fdeb16bd82a872 Mon Sep 17 00:00:00 2001
1f580a
From: Jaroslav Mracek <jmracek@redhat.com>
1f580a
Date: Tue, 25 Jun 2019 11:44:37 +0200
1f580a
Subject: [PATCH 5/5] Add detailed debug login for swdb/rpm transaction
1f580a
 (RhBug:1737328)
1f580a
1f580a
The logging is required to investigate the issue with tsi.state unknown.
1f580a
The logging will be triggered only when transaction element
1f580a
synchronization failed.
1f580a
---
1f580a
 dnf/util.py | 25 +++++++++++++++++++++++--
1f580a
 1 file changed, 23 insertions(+), 2 deletions(-)
1f580a
1f580a
diff --git a/dnf/util.py b/dnf/util.py
1f580a
index 9d0288e5a..71f61ab52 100644
1f580a
--- a/dnf/util.py
1f580a
+++ b/dnf/util.py
1f580a
@@ -396,6 +396,23 @@ def _te_nevra(te):
1f580a
     return nevra + te.V() + '-' + te.R() + '.' + te.A()
1f580a
 
1f580a
 
1f580a
+def _log_rpm_trans_with_swdb(rpm_transaction, swdb_transaction):
1f580a
+    logger.debug("Logging transaction elements")
1f580a
+    for rpm_el in rpm_transaction:
1f580a
+        tsi = rpm_el.Key()
1f580a
+        tsi_state = None
1f580a
+        if tsi is not None:
1f580a
+            tsi_state = tsi.state
1f580a
+        msg = "RPM element: '{}', Key(): '{}', Key state: '{}', Failed() '{}': ".format(
1f580a
+            _te_nevra(rpm_el), tsi, tsi_state, rpm_el.Failed())
1f580a
+        logger.debug(msg)
1f580a
+    for tsi in swdb_transaction:
1f580a
+        msg = "SWDB element: '{}', State: '{}', Action: '{}', From repo: '{}', Reason: '{}', " \
1f580a
+              "Get reason: '{}'".format(str(tsi), tsi.state, tsi.action, tsi.from_repo, tsi.reason,
1f580a
+                                        tsi.get_reason())
1f580a
+        logger.debug(msg)
1f580a
+
1f580a
+
1f580a
 def _sync_rpm_trans_with_swdb(rpm_transaction, swdb_transaction):
1f580a
     revert_actions = {libdnf.transaction.TransactionItemAction_DOWNGRADED,
1f580a
                       libdnf.transaction.TransactionItemAction_OBSOLETED,
1f580a
@@ -426,10 +443,14 @@ def _sync_rpm_trans_with_swdb(rpm_transaction, swdb_transaction):
1f580a
             error = True
1f580a
         else:
1f580a
             tsi.state = libdnf.transaction.TransactionItemState_DONE
1f580a
+    for tsi in cached_tsi:
1f580a
+        if tsi.state == libdnf.transaction.TransactionItemState_UNKNOWN:
1f580a
+            logger.critical(_("TransactionSWDBItem not found for key: {}").format(str(tsi)))
1f580a
+            el_not_found = True
1f580a
     if error:
1f580a
         logger.debug(_('Errors occurred during transaction.'))
1f580a
-
1f580a
-    return el_not_found
1f580a
+    if el_not_found:
1f580a
+        _log_rpm_trans_with_swdb(rpm_transaction, cached_tsi)
1f580a
 
1f580a
 
1f580a
 class tmpdir(object):
1f580a
-- 
1f580a
2.21.0
1f580a