From 6677d22131d070415db4a845ed8893a713c770d9 Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: Feb 23 2021 06:13:09 +0000 Subject: import python-blivet-3.2.2-9.el8 --- diff --git a/SOURCES/0019-LVM-VDO-support.patch b/SOURCES/0019-LVM-VDO-support.patch new file mode 100644 index 0000000..c79d6c1 --- /dev/null +++ b/SOURCES/0019-LVM-VDO-support.patch @@ -0,0 +1,2027 @@ +From 18f05802f07f580ed31f38931b1103842397d598 Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Mon, 2 Nov 2020 14:19:52 +0100 +Subject: [PATCH 01/17] Fix type of LVM VDO logical volumes + +We should use "lvmvdolv" to make it similar to other "lvmXYZ" +types. +--- + blivet/devices/lvm.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/blivet/devices/lvm.py b/blivet/devices/lvm.py +index d9e24a33..9639256d 100644 +--- a/blivet/devices/lvm.py ++++ b/blivet/devices/lvm.py +@@ -1875,7 +1875,7 @@ def vg(self): + + @property + def type(self): +- return "vdolv" ++ return "lvmvdolv" + + @property + def resizable(self): + +From 7f4815e14075550f55f2afb44bfba461eacea1c4 Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Mon, 2 Nov 2020 14:21:33 +0100 +Subject: [PATCH 02/17] Add VDO pool data LV to internal LVs during populate + +--- + blivet/devices/lvm.py | 9 ++++++++- + blivet/populator/helpers/lvm.py | 2 +- + 2 files changed, 9 insertions(+), 2 deletions(-) + +diff --git a/blivet/devices/lvm.py b/blivet/devices/lvm.py +index 9639256d..d0957d6a 100644 +--- a/blivet/devices/lvm.py ++++ b/blivet/devices/lvm.py +@@ -1119,7 +1119,7 @@ class LVMInternalLVtype(Enum): + + @classmethod + def get_type(cls, lv_attr, lv_name): # pylint: disable=unused-argument +- attr_letters = {cls.data: ("T", "C"), ++ attr_letters = {cls.data: ("T", "C", "D"), + cls.meta: ("e",), + cls.log: ("l", "L"), + cls.image: ("i", "I"), +@@ -1824,6 +1824,13 @@ def _remove_log_vol(self, lv): + self._lvs.remove(lv) + self.vg._remove_log_vol(lv) + ++ @property ++ @util.requires_property("is_vdo_pool") ++ def _vdopool_data_lv(self): ++ if not self._internal_lvs: ++ return None ++ return self._internal_lvs[0] ++ + @property + @util.requires_property("is_vdo_pool") + def lvs(self): +diff --git a/blivet/populator/helpers/lvm.py b/blivet/populator/helpers/lvm.py +index ff8bf59f..b1626306 100644 +--- a/blivet/populator/helpers/lvm.py ++++ b/blivet/populator/helpers/lvm.py +@@ -211,7 +211,7 @@ def add_lv(lv): + origin = self._devicetree.get_device_by_name(origin_device_name) + + lv_kwargs["origin"] = origin +- elif lv_attr[0] in 'IrielTCo' and lv_name.endswith(']'): ++ elif lv_attr[0] in 'IrielTCoD' and lv_name.endswith(']'): + # an internal LV, add the an instance of the appropriate class + # to internal_lvs for later processing when non-internal LVs are + # processed + +From c164864955e371aef78b5020f28bf0c9d235ac7c Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Mon, 2 Nov 2020 14:22:12 +0100 +Subject: [PATCH 03/17] Add availability functions for LVM VDO + +VDO is currently available only on RHEL/CentOS so we need a +separate availability check for LVM VDO devices. +--- + blivet/devices/lvm.py | 6 ++++++ + blivet/tasks/availability.py | 8 ++++++++ + 2 files changed, 14 insertions(+) + +diff --git a/blivet/devices/lvm.py b/blivet/devices/lvm.py +index d0957d6a..ffc65dcd 100644 +--- a/blivet/devices/lvm.py ++++ b/blivet/devices/lvm.py +@@ -1790,6 +1790,9 @@ def populate_ksdata(self, data): + + + class LVMVDOPoolMixin(object): ++ ++ _external_dependencies = [availability.BLOCKDEV_LVM_PLUGIN, availability.BLOCKDEV_LVM_PLUGIN_VDO] ++ + def __init__(self): + self._lvs = [] + +@@ -1848,6 +1851,9 @@ def _create(self): + + + class LVMVDOLogicalVolumeMixin(object): ++ ++ _external_dependencies = [availability.BLOCKDEV_LVM_PLUGIN, availability.BLOCKDEV_LVM_PLUGIN_VDO] ++ + def __init__(self): + pass + +diff --git a/blivet/tasks/availability.py b/blivet/tasks/availability.py +index f3b76650..b107428e 100644 +--- a/blivet/tasks/availability.py ++++ b/blivet/tasks/availability.py +@@ -372,6 +372,13 @@ def available_resource(name): + blockdev.LVMTechMode.MODIFY)}) + BLOCKDEV_LVM_TECH = BlockDevMethod(BLOCKDEV_LVM) + ++BLOCKDEV_LVM_VDO = BlockDevTechInfo(plugin_name="lvm", ++ check_fn=blockdev.lvm_is_tech_avail, ++ technologies={blockdev.LVMTech.VDO: (blockdev.LVMTechMode.CREATE | ++ blockdev.LVMTechMode.REMOVE | ++ blockdev.LVMTechMode.QUERY)}) ++BLOCKDEV_LVM_TECH_VDO = BlockDevMethod(BLOCKDEV_LVM_VDO) ++ + # libblockdev mdraid plugin required technologies and modes + BLOCKDEV_MD_ALL_MODES = (blockdev.MDTechMode.CREATE | + blockdev.MDTechMode.DELETE | +@@ -410,6 +417,7 @@ def available_resource(name): + BLOCKDEV_DM_PLUGIN_RAID = blockdev_plugin("dm", BLOCKDEV_DM_TECH_RAID) + BLOCKDEV_LOOP_PLUGIN = blockdev_plugin("loop", BLOCKDEV_LOOP_TECH) + BLOCKDEV_LVM_PLUGIN = blockdev_plugin("lvm", BLOCKDEV_LVM_TECH) ++BLOCKDEV_LVM_PLUGIN_VDO = blockdev_plugin("lvm", BLOCKDEV_LVM_TECH_VDO) + BLOCKDEV_MDRAID_PLUGIN = blockdev_plugin("mdraid", BLOCKDEV_MD_TECH) + BLOCKDEV_MPATH_PLUGIN = blockdev_plugin("mpath", BLOCKDEV_MPATH_TECH) + BLOCKDEV_SWAP_PLUGIN = blockdev_plugin("swap", BLOCKDEV_SWAP_TECH) + +From d782620129d47a7b79b0e6b80455e6d93f8bcc88 Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Mon, 2 Nov 2020 14:27:55 +0100 +Subject: [PATCH 04/17] Read the LVM VDO pool current size from the internal + data LV + +The pool device mapper device size is always 512k when active. +--- + blivet/devices/lvm.py | 9 +++++++++ + 1 file changed, 9 insertions(+) + +diff --git a/blivet/devices/lvm.py b/blivet/devices/lvm.py +index ffc65dcd..73743fa8 100644 +--- a/blivet/devices/lvm.py ++++ b/blivet/devices/lvm.py +@@ -1845,6 +1845,15 @@ def direct(self): + """ Is this device directly accessible? """ + return False + ++ def read_current_size(self): ++ log_method_call(self, exists=self.exists, path=self.path, ++ sysfs_path=self.sysfs_path) ++ if self.size != Size(0): ++ return self.size ++ if self._vdopool_data_lv: ++ return self._vdopool_data_lv.read_current_size() ++ return Size(0) ++ + def _create(self): + """ Create the device. """ + raise NotImplementedError + +From 2da48ae84f4eac84e8cf998ee2402249a5a52626 Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Mon, 2 Nov 2020 14:29:43 +0100 +Subject: [PATCH 05/17] Add "vdo_lv" property to LVMVDOPoolMixin + +--- + blivet/devices/lvm.py | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/blivet/devices/lvm.py b/blivet/devices/lvm.py +index 73743fa8..2f93fa22 100644 +--- a/blivet/devices/lvm.py ++++ b/blivet/devices/lvm.py +@@ -1840,6 +1840,13 @@ def lvs(self): + """ A list of this VDO pool's LVs """ + return self._lvs[:] # we don't want folks changing our list + ++ @property ++ @util.requires_property("is_vdo_pool") ++ def vdo_lv(self): ++ if not self._lvs: ++ return None ++ return self._lvs[0] ++ + @property + def direct(self): + """ Is this device directly accessible? """ + +From bbfa2cbdc6cb85d405b895c66eb4867cea4218b4 Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Mon, 2 Nov 2020 14:30:37 +0100 +Subject: [PATCH 06/17] Add support for creating LVM VDO pools and LVM VDO + volumes + +The pool and the volume are created by one call but these can have +different properties (like size) and are in fact two block devices +when created, we also need to create two devices and add them to +the devicetree. The pool device must be always created first and +the _create function for the VDO volume is a no-op. +--- + blivet/devices/lvm.py | 63 +++++++++++++++++++++++++++++++++++++------ + 1 file changed, 55 insertions(+), 8 deletions(-) + +diff --git a/blivet/devices/lvm.py b/blivet/devices/lvm.py +index 2f93fa22..0802e2de 100644 +--- a/blivet/devices/lvm.py ++++ b/blivet/devices/lvm.py +@@ -311,7 +311,7 @@ def _add_log_vol(self, lv): + + # verify we have the space, then add it + # do not verify for growing vg (because of ks) +- if not lv.exists and not self.growable and not lv.is_thin_lv and lv.size > self.free_space: ++ if not lv.exists and not self.growable and not (lv.is_thin_lv or lv.is_vdo_lv) and lv.size > self.free_space: + raise errors.DeviceError("new lv is too large to fit in free space", self.name) + + log.debug("Adding %s/%s to %s", lv.name, lv.size, self.name) +@@ -639,7 +639,7 @@ def __init__(self, name, parents=None, size=None, uuid=None, seg_type=None, + percent=None, cache_request=None, pvs=None, from_lvs=None): + + if not exists: +- if seg_type not in [None, "linear", "thin", "thin-pool", "cache"] + lvm.raid_seg_types: ++ if seg_type not in [None, "linear", "thin", "thin-pool", "cache", "vdo-pool", "vdo"] + lvm.raid_seg_types: + raise ValueError("Invalid or unsupported segment type: %s" % seg_type) + if seg_type and seg_type in lvm.raid_seg_types and not pvs: + raise ValueError("List of PVs has to be given for every non-linear LV") +@@ -1793,7 +1793,11 @@ class LVMVDOPoolMixin(object): + + _external_dependencies = [availability.BLOCKDEV_LVM_PLUGIN, availability.BLOCKDEV_LVM_PLUGIN_VDO] + +- def __init__(self): ++ def __init__(self, compression=True, deduplication=True, index_memory=0, write_policy=None): ++ self.compression = compression ++ self.deduplication = deduplication ++ self.index_memory = index_memory ++ self.write_policy = write_policy + self._lvs = [] + + @property +@@ -1863,7 +1867,19 @@ def read_current_size(self): + + def _create(self): + """ Create the device. """ +- raise NotImplementedError ++ ++ if not self.vdo_lv: ++ raise errors.DeviceError("Cannot create new VDO pool without a VDO LV.") ++ ++ if self.write_policy: ++ write_policy = blockdev.lvm_get_vdo_write_policy_str(self.write_policy) ++ else: ++ write_policy = blockdev.LVMVDOWritePolicy.AUTO ++ ++ blockdev.lvm.vdo_pool_create(self.vg.name, self.vdo_lv.lvname, self.lvname, ++ self.size, self.vdo_lv.size, self.index_memory, ++ self.compression, self.deduplication, ++ write_policy) + + + class LVMVDOLogicalVolumeMixin(object): +@@ -1915,9 +1931,26 @@ def resizable(self): + def pool(self): + return self.parents[0] + ++ def _set_size(self, newsize): ++ if not isinstance(newsize, Size): ++ raise AttributeError("new size must of type Size") ++ ++ newsize = self.vg.align(newsize) ++ newsize = self.vg.align(util.numeric_type(newsize)) ++ # just make sure the size is set (no VG size/free space check needed for ++ # a VDO LV) ++ DMDevice._set_size(self, newsize) ++ ++ def _pre_create(self): ++ # skip LVMLogicalVolumeDevice's _pre_create() method as it checks for a ++ # free space in a VG which doesn't make sense for a VDO LV and causes a ++ # bug by limitting the VDO LV's size to VG free space which is nonsense ++ super(LVMLogicalVolumeBase, self)._pre_create() # pylint: disable=bad-super-call ++ + def _create(self): +- """ Create the device. """ +- raise NotImplementedError ++ # nothing to do here, VDO LV is created automatically together with ++ # the VDO pool ++ pass + + def _destroy(self): + # nothing to do here, VDO LV is destroyed automatically together with +@@ -1953,7 +1986,9 @@ def __init__(self, name, parents=None, size=None, uuid=None, seg_type=None, + fmt=None, exists=False, sysfs_path='', grow=None, maxsize=None, + percent=None, cache_request=None, pvs=None, + parent_lv=None, int_type=None, origin=None, vorigin=False, +- metadata_size=None, chunk_size=None, profile=None, from_lvs=None): ++ metadata_size=None, chunk_size=None, profile=None, from_lvs=None, ++ compression=False, deduplication=False, index_memory=0, ++ write_policy=None): + """ + :param name: the device name (generally a device node's basename) + :type name: str +@@ -2012,6 +2047,17 @@ def __init__(self, name, parents=None, size=None, uuid=None, seg_type=None, + :keyword from_lvs: LVs to create the new LV from (in the (data_lv, metadata_lv) order) + :type from_lvs: tuple of :class:`LVMLogicalVolumeDevice` + ++ For VDO pools only: ++ ++ :keyword compression: whether to enable compression on the VDO pool ++ :type compression: bool ++ :keyword dudplication: whether to enable dudplication on the VDO pool ++ :type dudplication: bool ++ :keyword index_memory: amount of index memory (in bytes) or 0 for default ++ :type index_memory: int ++ :keyword write_policy: write policy for the volume or None for default ++ :type write_policy: str ++ + """ + + if isinstance(parents, (list, ParentList)): +@@ -2032,7 +2078,8 @@ def __init__(self, name, parents=None, size=None, uuid=None, seg_type=None, + LVMLogicalVolumeBase.__init__(self, name, parents, size, uuid, seg_type, + fmt, exists, sysfs_path, grow, maxsize, + percent, cache_request, pvs, from_lvs) +- LVMVDOPoolMixin.__init__(self) ++ LVMVDOPoolMixin.__init__(self, compression, deduplication, index_memory, ++ write_policy) + LVMVDOLogicalVolumeMixin.__init__(self) + + LVMInternalLogicalVolumeMixin._init_check(self) + +From 2d1593b50dc6232e213b4df86dfbf5cf6d282dcd Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Mon, 2 Nov 2020 14:31:35 +0100 +Subject: [PATCH 07/17] Allow creating LVM VDO pools and volumes using + "blivet.new_lv" + +The steps to create the VDO devices would typically look like: + +pool = b.new_lv(vdo_pool=True, parents=[data], size=Size("8 GiB")) +vdolv = b.new_lv(vdo_lv=True, parents=[pool], size=Size("40 GiB")) +b.create_device(pool) +b.create_device(vdolv) +b.do_it() +--- + blivet/blivet.py | 18 ++++++++++++++---- + tests/devices_test/lvm_test.py | 31 +++++++++++++++++++++++++++++++ + 2 files changed, 45 insertions(+), 4 deletions(-) + +diff --git a/blivet/blivet.py b/blivet/blivet.py +index e7dbd37b..754eb152 100644 +--- a/blivet/blivet.py ++++ b/blivet/blivet.py +@@ -573,6 +573,10 @@ def new_lv(self, *args, **kwargs): + :type thin_pool: bool + :keyword thin_volume: whether to create a thin volume + :type thin_volume: bool ++ :keyword vdo_pool: whether to create a vdo pool ++ :type vdo_pool: bool ++ :keyword vdo_lv: whether to create a vdo lv ++ :type vdo_lv: bool + :returns: the new device + :rtype: :class:`~.devices.LVMLogicalVolumeDevice` + +@@ -589,8 +593,10 @@ def new_lv(self, *args, **kwargs): + """ + thin_volume = kwargs.pop("thin_volume", False) + thin_pool = kwargs.pop("thin_pool", False) ++ vdo_pool = kwargs.pop("vdo_pool", False) ++ vdo_lv = kwargs.pop("vdo_lv", False) + parent = kwargs.get("parents", [None])[0] +- if thin_volume and parent: ++ if (thin_volume or vdo_lv) and parent: + # kwargs["parents"] will contain the pool device, so... + vg = parent.vg + else: +@@ -600,6 +606,10 @@ def new_lv(self, *args, **kwargs): + kwargs["seg_type"] = "thin" + if thin_pool: + kwargs["seg_type"] = "thin-pool" ++ if vdo_pool: ++ kwargs["seg_type"] = "vdo-pool" ++ if vdo_lv: ++ kwargs["seg_type"] = "vdo" + + mountpoint = kwargs.pop("mountpoint", None) + if 'fmt_type' in kwargs: +@@ -625,7 +635,7 @@ def new_lv(self, *args, **kwargs): + swap = False + + prefix = "" +- if thin_pool: ++ if thin_pool or vdo_pool: + prefix = "pool" + + name = self.suggest_device_name(parent=vg, +@@ -636,10 +646,10 @@ def new_lv(self, *args, **kwargs): + if "%s-%s" % (vg.name, name) in self.names: + raise ValueError("name already in use") + +- if thin_pool or thin_volume: ++ if thin_pool or thin_volume or vdo_pool or vdo_lv: + cache_req = kwargs.pop("cache_request", None) + if cache_req: +- raise ValueError("Creating cached thin volumes and pools is not supported") ++ raise ValueError("Creating cached thin and VDO volumes and pools is not supported") + + return LVMLogicalVolumeDevice(name, *args, **kwargs) + +diff --git a/tests/devices_test/lvm_test.py b/tests/devices_test/lvm_test.py +index 204cb99a..493d3ba1 100644 +--- a/tests/devices_test/lvm_test.py ++++ b/tests/devices_test/lvm_test.py +@@ -689,3 +689,34 @@ def test_new_lv_from_non_existing_lvs(self): + with patch.object(pool, "_pre_create"): + pool.create() + self.assertTrue(lvm.thpool_convert.called) ++ ++ def test_new_vdo_pool(self): ++ b = blivet.Blivet() ++ pv = StorageDevice("pv1", fmt=blivet.formats.get_format("lvmpv"), ++ size=Size("10 GiB"), exists=True) ++ vg = LVMVolumeGroupDevice("testvg", parents=[pv], exists=True) ++ ++ for dev in (pv, vg): ++ b.devicetree._add_device(dev) ++ ++ # check that all the above devices are in the expected places ++ self.assertEqual(set(b.devices), {pv, vg}) ++ self.assertEqual(set(b.vgs), {vg}) ++ ++ self.assertEqual(vg.size, Size("10236 MiB")) ++ ++ vdopool = b.new_lv(name="vdopool", vdo_pool=True, ++ parents=[vg], compression=True, ++ deduplication=True, ++ size=blivet.size.Size("8 GiB")) ++ ++ vdolv = b.new_lv(name="vdolv", vdo_lv=True, ++ parents=[vdopool], ++ size=blivet.size.Size("40 GiB")) ++ ++ b.create_device(vdopool) ++ b.create_device(vdolv) ++ ++ self.assertEqual(vdopool.children[0], vdolv) ++ self.assertEqual(vdolv.parents[0], vdopool) ++ self.assertListEqual(vg.lvs, [vdopool, vdolv]) + +From 31ec429ad7bd0857a768e2dfebe1de088dafc144 Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Mon, 2 Nov 2020 14:32:47 +0100 +Subject: [PATCH 08/17] Add LVM VDO device factory + +--- + blivet/devicefactory.py | 100 +++++++++++++++++++++++++++- + tests/devicefactory_test.py | 128 +++++++++++++++++++++++++++++++++--- + 2 files changed, 218 insertions(+), 10 deletions(-) + +diff --git a/blivet/devicefactory.py b/blivet/devicefactory.py +index 9214ad54..c95037cc 100644 +--- a/blivet/devicefactory.py ++++ b/blivet/devicefactory.py +@@ -27,7 +27,7 @@ + from .devices import BTRFSDevice, DiskDevice + from .devices import LUKSDevice, LVMLogicalVolumeDevice + from .devices import PartitionDevice, MDRaidArrayDevice +-from .devices.lvm import DEFAULT_THPOOL_RESERVE ++from .devices.lvm import LVMVDOPoolMixin, DEFAULT_THPOOL_RESERVE + from .formats import get_format + from .devicelibs import btrfs + from .devicelibs import mdraid +@@ -58,6 +58,7 @@ + DEVICE_TYPE_BTRFS = 3 + DEVICE_TYPE_DISK = 4 + DEVICE_TYPE_LVM_THINP = 5 ++DEVICE_TYPE_LVM_VDO = 6 + + + def is_supported_device_type(device_type): +@@ -69,6 +70,9 @@ def is_supported_device_type(device_type): + :returns: True if this device type is supported + :rtype: bool + """ ++ if device_type == DEVICE_TYPE_LVM_VDO: ++ return not any(e for e in LVMVDOPoolMixin._external_dependencies if not e.available) ++ + devices = [] + if device_type == DEVICE_TYPE_BTRFS: + devices = [BTRFSDevice] +@@ -96,7 +100,7 @@ def get_supported_raid_levels(device_type): + pkg = None + if device_type == DEVICE_TYPE_BTRFS: + pkg = btrfs +- elif device_type in (DEVICE_TYPE_LVM, DEVICE_TYPE_LVM_THINP): ++ elif device_type in (DEVICE_TYPE_LVM, DEVICE_TYPE_LVM_THINP, DEVICE_TYPE_LVM_VDO): + pkg = lvm + elif device_type == DEVICE_TYPE_MD: + pkg = mdraid +@@ -116,6 +120,8 @@ def get_device_type(device): + "lvmlv": DEVICE_TYPE_LVM, + "lvmthinlv": DEVICE_TYPE_LVM_THINP, + "lvmthinpool": DEVICE_TYPE_LVM, ++ "lvmvdolv": DEVICE_TYPE_LVM_VDO, ++ "lvmvdopool": DEVICE_TYPE_LVM, + "btrfs subvolume": DEVICE_TYPE_BTRFS, + "btrfs volume": DEVICE_TYPE_BTRFS, + "mdarray": DEVICE_TYPE_MD} +@@ -136,6 +142,7 @@ def get_device_factory(blivet, device_type=DEVICE_TYPE_LVM, **kwargs): + DEVICE_TYPE_PARTITION: PartitionFactory, + DEVICE_TYPE_MD: MDFactory, + DEVICE_TYPE_LVM_THINP: LVMThinPFactory, ++ DEVICE_TYPE_LVM_VDO: LVMVDOFactory, + DEVICE_TYPE_DISK: DeviceFactory} + + factory_class = class_table[device_type] +@@ -1738,6 +1745,95 @@ def _get_new_device(self, *args, **kwargs): + return super(LVMThinPFactory, self)._get_new_device(*args, **kwargs) + + ++class LVMVDOFactory(LVMFactory): ++ ++ """ Factory for creating LVM VDO volumes. ++ ++ :keyword pool_name: name for the VDO pool, if not specified unique name will be generated ++ :type pool_name: str ++ :keyword virtual_size: size for the VDO volume, usually bigger than pool size, if not ++ specified physical size (pool size) will be used ++ :type size: :class:`~.size.Size` ++ :keyword compression: whether to enable compression (defaults to True) ++ :type compression: bool ++ :keyword deduplication: whether to enable deduplication (defaults to True) ++ :type deduplication: bool ++ """ ++ ++ def __init__(self, storage, **kwargs): ++ self.pool_name = kwargs.pop("pool_name", None) ++ self.virtual_size = kwargs.pop("virtual_size", None) ++ self.compression = kwargs.pop("compression", True) ++ self.deduplication = kwargs.pop("deduplication", True) ++ super(LVMVDOFactory, self).__init__(storage, **kwargs) ++ ++ def _get_new_pool(self, *args, **kwargs): ++ kwargs["vdo_pool"] = True ++ return super(LVMVDOFactory, self)._get_new_device(*args, **kwargs) ++ ++ def _set_device_size(self): ++ """ Set the size of the factory device. """ ++ super(LVMVDOFactory, self)._set_device_size() ++ ++ self.device.pool.size = self.size ++ self._reconfigure_container() ++ ++ if not self.virtual_size or self.virtual_size < self.size: ++ # virtual_size is not set or smaller than current size --> it should be same as the pool size ++ self.device.size = self.size ++ else: ++ self.device.size = self.virtual_size ++ ++ def _set_pool_name(self): ++ safe_new_name = self.storage.safe_device_name(self.pool_name) ++ if self.device.pool.name != safe_new_name: ++ if not safe_new_name: ++ log.error("not renaming '%s' to invalid name '%s'", ++ self.device.pool.name, self.pool_name) ++ return ++ if safe_new_name in self.storage.names: ++ log.error("not renaming '%s' to in-use name '%s'", ++ self.device.pool.name, safe_new_name) ++ return ++ ++ log.debug("renaming device '%s' to '%s'", ++ self.device.pool.name, safe_new_name) ++ self.device.pool.raw_device.name = safe_new_name ++ ++ def _set_name(self): ++ super(LVMVDOFactory, self)._set_name() ++ if self.pool_name: ++ self._set_pool_name() ++ ++ def _reconfigure_device(self): ++ super(LVMVDOFactory, self)._reconfigure_device() ++ ++ self.device.pool.compression = self.compression ++ self.device.pool.deduplication = self.deduplication ++ ++ # ++ # methods to configure the factory's device ++ # ++ def _get_new_device(self, *args, **kwargs): ++ """ Create and return the factory device as a StorageDevice. """ ++ pool = self._get_new_pool(name=self.pool_name, ++ size=self.size, ++ parents=[self.vg], ++ compression=self.compression, ++ deduplication=self.deduplication) ++ self.storage.create_device(pool) ++ ++ kwargs["parents"] = [pool] ++ kwargs["vdo_lv"] = True ++ ++ if self.virtual_size: ++ vdolv_kwargs = kwargs.copy() ++ vdolv_kwargs["size"] = self.virtual_size ++ else: ++ vdolv_kwargs = kwargs ++ return super(LVMVDOFactory, self)._get_new_device(*args, **vdolv_kwargs) ++ ++ + class MDFactory(DeviceFactory): + + """ Factory for creating MD RAID devices. """ +diff --git a/tests/devicefactory_test.py b/tests/devicefactory_test.py +index 08068779..7cdb51c5 100644 +--- a/tests/devicefactory_test.py ++++ b/tests/devicefactory_test.py +@@ -4,6 +4,9 @@ + from decimal import Decimal + import os + ++import test_compat # pylint: disable=unused-import ++from six.moves.mock import patch # pylint: disable=no-name-in-module,import-error ++ + import blivet + + from blivet import devicefactory +@@ -93,10 +96,12 @@ def _validate_factory_device(self, *args, **kwargs): + self.assertEqual(device.format.label, + kwargs.get('label')) + +- self.assertLessEqual(device.size, kwargs.get("size")) +- self.assertGreaterEqual(device.size, device.format.min_size) +- if device.format.max_size: +- self.assertLessEqual(device.size, device.format.max_size) ++ # sizes with VDO are special, we have a special check in LVMVDOFactoryTestCase._validate_factory_device ++ if device_type != devicefactory.DEVICE_TYPE_LVM_VDO: ++ self.assertLessEqual(device.size, kwargs.get("size")) ++ self.assertGreaterEqual(device.size, device.format.min_size) ++ if device.format.max_size: ++ self.assertLessEqual(device.size, device.format.max_size) + + self.assertEqual(device.encrypted, + kwargs.get("encrypted", False) or +@@ -115,7 +120,11 @@ def test_device_factory(self): + "mountpoint": '/factorytest'} + device = self._factory_device(device_type, **kwargs) + self._validate_factory_device(device, device_type, **kwargs) +- self.b.recursive_remove(device) ++ ++ if device.type == "lvmvdolv": ++ self.b.recursive_remove(device.pool) ++ else: ++ self.b.recursive_remove(device) + + if self.encryption_supported: + # Encrypt the leaf device +@@ -157,6 +166,12 @@ def test_device_factory(self): + device = self._factory_device(device_type, **kwargs) + self._validate_factory_device(device, device_type, **kwargs) + ++ # change size up ++ kwargs["device"] = device ++ kwargs["size"] = Size("900 MiB") ++ device = self._factory_device(device_type, **kwargs) ++ self._validate_factory_device(device, device_type, **kwargs) ++ + # Change LUKS version + kwargs["luks_version"] = "luks1" + device = self._factory_device(device_type, **kwargs) +@@ -179,7 +194,7 @@ def _get_size_delta(self, devices=None): + """ + return Size("1 MiB") + +- def test_get_free_disk_space(self): ++ def test_get_free_disk_space(self, *args): # pylint: disable=unused-argument + # get_free_disk_space should return the total free space on disks + kwargs = self._get_test_factory_args() + kwargs["size"] = Size("500 MiB") +@@ -206,7 +221,7 @@ def test_get_free_disk_space(self): + sum(d.size for d in self.b.disks) - device_space, + delta=self._get_size_delta(devices=[device])) + +- def test_normalize_size(self): ++ def test_normalize_size(self, *args): # pylint: disable=unused-argument + # _normalize_size should adjust target size to within the format limits + fstype = "ext2" + ext2 = get_format(fstype) +@@ -258,7 +273,7 @@ def test_default_factory_type(self): + factory = devicefactory.get_device_factory(self.b) + self.assertIsInstance(factory, devicefactory.LVMFactory) + +- def test_factory_defaults(self): ++ def test_factory_defaults(self, *args): # pylint: disable=unused-argument + ctor_kwargs = self._get_test_factory_args() + factory = devicefactory.get_device_factory(self.b, self.device_type, **ctor_kwargs) + for setting, value in factory._default_settings.items(): +@@ -522,6 +537,103 @@ def _get_size_delta(self, devices=None): + return delta + + ++class LVMVDOFactoryTestCase(LVMFactoryTestCase): ++ device_class = LVMLogicalVolumeDevice ++ device_type = devicefactory.DEVICE_TYPE_LVM_VDO ++ encryption_supported = False ++ ++ def _validate_factory_device(self, *args, **kwargs): ++ super(LVMVDOFactoryTestCase, self)._validate_factory_device(*args, ++ **kwargs) ++ device = args[0] ++ ++ if kwargs.get("encrypted", False): ++ vdolv = device.parents[0] ++ else: ++ vdolv = device ++ ++ self.assertTrue(hasattr(vdolv, "pool")) ++ ++ virtual_size = kwargs.get("virtual_size", 0) ++ if virtual_size: ++ self.assertEqual(vdolv.size, virtual_size) ++ else: ++ self.assertEqual(vdolv.size, vdolv.pool.size) ++ self.assertGreaterEqual(vdolv.size, vdolv.pool.size) ++ ++ compression = kwargs.get("compression", True) ++ self.assertEqual(vdolv.pool.compression, compression) ++ ++ deduplication = kwargs.get("deduplication", True) ++ self.assertEqual(vdolv.pool.deduplication, deduplication) ++ ++ pool_name = kwargs.get("pool_name", None) ++ if pool_name: ++ self.assertEqual(vdolv.pool.lvname, pool_name) ++ ++ return device ++ ++ @patch("blivet.formats.lvmpv.LVMPhysicalVolume.formattable", return_value=True) ++ @patch("blivet.formats.lvmpv.LVMPhysicalVolume.destroyable", return_value=True) ++ @patch("blivet.static_data.lvm_info.blockdev.lvm.lvs", return_value=[]) ++ @patch("blivet.devices.lvm.LVMVolumeGroupDevice.type_external_dependencies", return_value=set()) ++ @patch("blivet.devices.lvm.LVMLogicalVolumeBase.type_external_dependencies", return_value=set()) ++ def test_device_factory(self, *args): # pylint: disable=unused-argument,arguments-differ ++ device_type = self.device_type ++ kwargs = {"disks": self.b.disks, ++ "size": Size("400 MiB"), ++ "fstype": 'ext4', ++ "mountpoint": '/factorytest'} ++ device = self._factory_device(device_type, **kwargs) ++ self._validate_factory_device(device, device_type, **kwargs) ++ self.b.recursive_remove(device.pool) ++ ++ kwargs = {"disks": self.b.disks, ++ "size": Size("400 MiB"), ++ "fstype": 'ext4', ++ "mountpoint": '/factorytest', ++ "pool_name": "vdopool", ++ "deduplication": True, ++ "compression": True} ++ device = self._factory_device(device_type, **kwargs) ++ self._validate_factory_device(device, device_type, **kwargs) ++ ++ # change size without specifying virtual_size: both sizes should grow ++ kwargs["size"] = Size("600 MiB") ++ kwargs["device"] = device ++ device = self._factory_device(device_type, **kwargs) ++ self._validate_factory_device(device, device_type, **kwargs) ++ ++ # change virtual size ++ kwargs["virtual_size"] = Size("6 GiB") ++ kwargs["device"] = device ++ device = self._factory_device(device_type, **kwargs) ++ self._validate_factory_device(device, device_type, **kwargs) ++ ++ # change virtual size to smaller than size ++ kwargs["virtual_size"] = Size("500 GiB") ++ kwargs["device"] = device ++ device = self._factory_device(device_type, **kwargs) ++ self._validate_factory_device(device, device_type, **kwargs) ++ ++ # change deduplication and compression ++ kwargs["deduplication"] = False ++ kwargs["device"] = device ++ device = self._factory_device(device_type, **kwargs) ++ self._validate_factory_device(device, device_type, **kwargs) ++ ++ kwargs["compression"] = False ++ kwargs["device"] = device ++ device = self._factory_device(device_type, **kwargs) ++ self._validate_factory_device(device, device_type, **kwargs) ++ ++ # rename the pool ++ kwargs["pool_name"] = "vdopool2" ++ kwargs["device"] = device ++ device = self._factory_device(device_type, **kwargs) ++ self._validate_factory_device(device, device_type, **kwargs) ++ ++ + class MDFactoryTestCase(DeviceFactoryTestCase): + device_type = devicefactory.DEVICE_TYPE_MD + device_class = MDRaidArrayDevice + +From 22ba2b96111d5f153a3b55d3c56d84e597cf9a90 Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Mon, 2 Nov 2020 14:33:06 +0100 +Subject: [PATCH 09/17] Add VM test for LVM VDO + +--- + tests/vmtests/blivet_reset_vmtest.py | 15 +++++++++++++++ + tests/vmtests/runvmtests.py | 3 ++- + 2 files changed, 17 insertions(+), 1 deletion(-) + +diff --git a/tests/vmtests/blivet_reset_vmtest.py b/tests/vmtests/blivet_reset_vmtest.py +index 8743d51e..47fc84c4 100644 +--- a/tests/vmtests/blivet_reset_vmtest.py ++++ b/tests/vmtests/blivet_reset_vmtest.py +@@ -192,6 +192,21 @@ def setUp(self): + self.collect_expected_data() + + ++class LVMVDOTestCase(BlivetResetTestCase): ++ ++ def _set_up_storage(self): ++ if not devicefactory.is_supported_device_type(devicefactory.DEVICE_TYPE_LVM_VDO): ++ self.skipTest("VDO not supported, skipping") ++ ++ self.blivet.factory_device(devicefactory.DEVICE_TYPE_LVM_VDO, ++ size=Size("10 GiB"), ++ fstype="ext4", ++ disks=self.blivet.disks[:], ++ name="vdolv", ++ pool_name="vdopool", ++ virtual_size=Size("40 GiB")) ++ ++ + @unittest.skip("temporarily disabled due to issues with raids with metadata version 0.90") + class MDRaid0TestCase(BlivetResetTestCase): + +diff --git a/tests/vmtests/runvmtests.py b/tests/vmtests/runvmtests.py +index 88143d3a..6f20484f 100644 +--- a/tests/vmtests/runvmtests.py ++++ b/tests/vmtests/runvmtests.py +@@ -12,7 +12,8 @@ + "tests.vmtests.blivet_reset_vmtest.LVMThinSnapShotTestCase", + "tests.vmtests.blivet_reset_vmtest.LVMRaidTestCase", + "tests.vmtests.blivet_reset_vmtest.MDRaid0TestCase", +- "tests.vmtests.blivet_reset_vmtest.LVMOnMDTestCase"] ++ "tests.vmtests.blivet_reset_vmtest.LVMOnMDTestCase", ++ "tests.vmtests.blivet_reset_vmtest.LVMVDOTestCase"] + + SNAP_NAME = "snapshot" + + +From 52b37bb86e856f1ede71f7cceb7284a639d741f4 Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Thu, 19 Nov 2020 13:07:17 +0100 +Subject: [PATCH 10/17] Allow adding nodiscard option when running mkfs + +For filesystems that support it we might want to add some nodiscard +option to mkfs when creating format on devices like LVM VDO +volumes where discard is very slow and doesn't really makes sense +when running mkfs. +--- + blivet/formats/fs.py | 12 +++++- + blivet/tasks/fsmkfs.py | 59 +++++++++++++++++++++++++++--- + tests/formats_test/methods_test.py | 3 +- + 3 files changed, 66 insertions(+), 8 deletions(-) + +diff --git a/blivet/formats/fs.py b/blivet/formats/fs.py +index 4ba83e6d..e61e5b86 100644 +--- a/blivet/formats/fs.py ++++ b/blivet/formats/fs.py +@@ -132,6 +132,7 @@ def __init__(self, **kwargs): + self.mountopts = kwargs.get("mountopts", "") + self.label = kwargs.get("label") + self.fsprofile = kwargs.get("fsprofile") ++ self._mkfs_nodiscard = kwargs.get("nodiscard", False) + + self._user_mountopts = self.mountopts + +@@ -263,6 +264,14 @@ def label_format_ok(self, label): + label = property(lambda s: s._get_label(), lambda s, l: s._set_label(l), + doc="this filesystem's label") + ++ def can_nodiscard(self): ++ """Returns True if this filesystem supports nodiscard option during ++ creation, otherwise False. ++ ++ :rtype: bool ++ """ ++ return self._mkfs.can_nodiscard and self._mkfs.available ++ + def can_set_uuid(self): + """Returns True if this filesystem supports setting an UUID during + creation, otherwise False. +@@ -402,7 +411,8 @@ def _create(self, **kwargs): + try: + self._mkfs.do_task(options=kwargs.get("options"), + label=not self.relabels(), +- set_uuid=self.can_set_uuid()) ++ set_uuid=self.can_set_uuid(), ++ nodiscard=self.can_nodiscard()) + except FSWriteLabelError as e: + log.warning("Choosing not to apply label (%s) during creation of filesystem %s. Label format is unacceptable for this filesystem.", self.label, self.type) + except FSWriteUUIDError as e: +diff --git a/blivet/tasks/fsmkfs.py b/blivet/tasks/fsmkfs.py +index ad166aa0..c982f7e7 100644 +--- a/blivet/tasks/fsmkfs.py ++++ b/blivet/tasks/fsmkfs.py +@@ -37,6 +37,7 @@ class FSMkfsTask(fstask.FSTask): + + can_label = abc.abstractproperty(doc="whether this task labels") + can_set_uuid = abc.abstractproperty(doc="whether this task can set UUID") ++ can_nodiscard = abc.abstractproperty(doc="whether this task can set nodiscard option") + + + @add_metaclass(abc.ABCMeta) +@@ -48,6 +49,9 @@ class FSMkfs(task.BasicApplication, FSMkfsTask): + label_option = abc.abstractproperty( + doc="Option for setting a filesystem label.") + ++ nodiscard_option = abc.abstractproperty( ++ doc="Option for setting nodiscrad option for mkfs.") ++ + args = abc.abstractproperty(doc="options for creating filesystem") + + @abc.abstractmethod +@@ -80,6 +84,15 @@ def can_set_uuid(self): + """ + return self.get_uuid_args is not None + ++ @property ++ def can_nodiscard(self): ++ """Whether this task can set nodiscard option for a filesystem. ++ ++ :returns: True if nodiscard can be set ++ :rtype: bool ++ """ ++ return self.nodiscard_option is not None ++ + @property + def _label_options(self): + """ Any labeling options that a particular filesystem may use. +@@ -100,6 +113,23 @@ def _label_options(self): + else: + raise FSWriteLabelError("Choosing not to apply label (%s) during creation of filesystem %s. Label format is unacceptable for this filesystem." % (self.fs.label, self.fs.type)) + ++ @property ++ def _nodiscard_option(self): ++ """ Any nodiscard options that a particular filesystem may use. ++ ++ :returns: nodiscard options ++ :rtype: list of str ++ """ ++ # Do not know how to set nodiscard while formatting. ++ if self.nodiscard_option is None: ++ return [] ++ ++ # nodiscard option not requested ++ if not self.fs._mkfs_nodiscard: ++ return [] ++ ++ return self.nodiscard_option ++ + @property + def _uuid_options(self): + """Any UUID options that a particular filesystem may use. +@@ -119,7 +149,7 @@ def _uuid_options(self): + " is unacceptable for this filesystem." + % (self.fs.uuid, self.fs.type)) + +- def _format_options(self, options=None, label=False, set_uuid=False): ++ def _format_options(self, options=None, label=False, set_uuid=False, nodiscard=False): + """Get a list of format options to be used when creating the + filesystem. + +@@ -135,11 +165,12 @@ def _format_options(self, options=None, label=False, set_uuid=False): + + label_options = self._label_options if label else [] + uuid_options = self._uuid_options if set_uuid else [] ++ nodiscard_option = self._nodiscard_option if nodiscard else [] + create_options = shlex.split(self.fs.create_options or "") + return (options + self.args + label_options + uuid_options + +- create_options + [self.fs.device]) ++ nodiscard_option + create_options + [self.fs.device]) + +- def _mkfs_command(self, options, label, set_uuid): ++ def _mkfs_command(self, options, label, set_uuid, nodiscard): + """Return the command to make the filesystem. + + :param options: any special options +@@ -148,12 +179,14 @@ def _mkfs_command(self, options, label, set_uuid): + :type label: bool + :param set_uuid: whether to set an UUID + :type set_uuid: bool ++ :param nodiscard: whether to run mkfs with nodiscard option ++ :type nodiscard: bool + :returns: the mkfs command + :rtype: list of str + """ +- return [str(self.ext)] + self._format_options(options, label, set_uuid) ++ return [str(self.ext)] + self._format_options(options, label, set_uuid, nodiscard) + +- def do_task(self, options=None, label=False, set_uuid=False): ++ def do_task(self, options=None, label=False, set_uuid=False, nodiscard=False): + """Create the format on the device and label if possible and desired. + + :param options: any special options, may be None +@@ -168,7 +201,7 @@ def do_task(self, options=None, label=False, set_uuid=False): + raise FSError("\n".join(error_msgs)) + + options = options or [] +- cmd = self._mkfs_command(options, label, set_uuid) ++ cmd = self._mkfs_command(options, label, set_uuid, nodiscard) + try: + ret = util.run_program(cmd) + except OSError as e: +@@ -181,6 +214,7 @@ def do_task(self, options=None, label=False, set_uuid=False): + class BTRFSMkfs(FSMkfs): + ext = availability.MKFS_BTRFS_APP + label_option = None ++ nodiscard_option = ["--nodiscard"] + + def get_uuid_args(self, uuid): + return ["-U", uuid] +@@ -193,6 +227,7 @@ def args(self): + class Ext2FSMkfs(FSMkfs): + ext = availability.MKE2FS_APP + label_option = "-L" ++ nodiscard_option = ["-E", "nodiscard"] + + _opts = [] + +@@ -215,6 +250,7 @@ class Ext4FSMkfs(Ext3FSMkfs): + class FATFSMkfs(FSMkfs): + ext = availability.MKDOSFS_APP + label_option = "-n" ++ nodiscard_option = None + + def get_uuid_args(self, uuid): + return ["-i", uuid.replace('-', '')] +@@ -227,6 +263,7 @@ def args(self): + class GFS2Mkfs(FSMkfs): + ext = availability.MKFS_GFS2_APP + label_option = None ++ nodiscard_option = None + get_uuid_args = None + + @property +@@ -237,6 +274,7 @@ def args(self): + class HFSMkfs(FSMkfs): + ext = availability.HFORMAT_APP + label_option = "-l" ++ nodiscard_option = None + get_uuid_args = None + + @property +@@ -247,6 +285,7 @@ def args(self): + class HFSPlusMkfs(FSMkfs): + ext = availability.MKFS_HFSPLUS_APP + label_option = "-v" ++ nodiscard_option = None + get_uuid_args = None + + @property +@@ -257,6 +296,7 @@ def args(self): + class JFSMkfs(FSMkfs): + ext = availability.MKFS_JFS_APP + label_option = "-L" ++ nodiscard_option = None + get_uuid_args = None + + @property +@@ -267,6 +307,7 @@ def args(self): + class NTFSMkfs(FSMkfs): + ext = availability.MKNTFS_APP + label_option = "-L" ++ nodiscard_option = None + get_uuid_args = None + + @property +@@ -277,6 +318,7 @@ def args(self): + class ReiserFSMkfs(FSMkfs): + ext = availability.MKREISERFS_APP + label_option = "-l" ++ nodiscard_option = None + + def get_uuid_args(self, uuid): + return ["-u", uuid] +@@ -289,6 +331,7 @@ def args(self): + class XFSMkfs(FSMkfs): + ext = availability.MKFS_XFS_APP + label_option = "-L" ++ nodiscard_option = ["-K"] + + def get_uuid_args(self, uuid): + return ["-m", "uuid=" + uuid] +@@ -307,3 +350,7 @@ def can_label(self): + @property + def can_set_uuid(self): + return False ++ ++ @property ++ def can_nodiscard(self): ++ return False +diff --git a/tests/formats_test/methods_test.py b/tests/formats_test/methods_test.py +index 710fa1c5..b2674ea7 100644 +--- a/tests/formats_test/methods_test.py ++++ b/tests/formats_test/methods_test.py +@@ -307,7 +307,8 @@ def _test_create_backend(self): + self.format._mkfs.do_task.assert_called_with( + options=None, + label=not self.format.relabels(), +- set_uuid=self.format.can_set_uuid() ++ set_uuid=self.format.can_set_uuid(), ++ nodiscard=self.format.can_nodiscard() + ) + + def _test_setup_backend(self): + +From ac04f74fa9bc8ded3facd302ca74ec033009a0bd Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Thu, 19 Nov 2020 13:19:21 +0100 +Subject: [PATCH 11/17] Add nodiscard option by default when creating VDO + logical volumes + +User can override this by passing "nodiscard=False" to the LV +constructor, but we want nodiscard by default. +--- + blivet/blivet.py | 8 +++++++- + blivet/devicefactory.py | 6 ++++++ + tests/devicefactory_test.py | 7 +++++++ + 3 files changed, 20 insertions(+), 1 deletion(-) + +diff --git a/blivet/blivet.py b/blivet/blivet.py +index 754eb152..e4115691 100644 +--- a/blivet/blivet.py ++++ b/blivet/blivet.py +@@ -613,9 +613,15 @@ def new_lv(self, *args, **kwargs): + + mountpoint = kwargs.pop("mountpoint", None) + if 'fmt_type' in kwargs: ++ fmt_args = kwargs.pop("fmt_args", {}) ++ if vdo_lv and "nodiscard" not in fmt_args.keys(): ++ # we don't want to run discard on VDO LV during mkfs so if user don't ++ # tell us not to do it, we should add the nodiscard option to mkfs ++ fmt_args["nodiscard"] = True ++ + kwargs["fmt"] = get_format(kwargs.pop("fmt_type"), + mountpoint=mountpoint, +- **kwargs.pop("fmt_args", {})) ++ **fmt_args) + + name = kwargs.pop("name", None) + if name: +diff --git a/blivet/devicefactory.py b/blivet/devicefactory.py +index c95037cc..085f2fd6 100644 +--- a/blivet/devicefactory.py ++++ b/blivet/devicefactory.py +@@ -1811,6 +1811,12 @@ def _reconfigure_device(self): + self.device.pool.compression = self.compression + self.device.pool.deduplication = self.deduplication + ++ def _set_format(self): ++ super(LVMVDOFactory, self)._set_format() ++ ++ # preserve nodiscard mkfs option after changing filesystem ++ self.device.format._mkfs_nodiscard = True ++ + # + # methods to configure the factory's device + # +diff --git a/tests/devicefactory_test.py b/tests/devicefactory_test.py +index 7cdb51c5..4de1e05b 100644 +--- a/tests/devicefactory_test.py ++++ b/tests/devicefactory_test.py +@@ -571,6 +571,10 @@ def _validate_factory_device(self, *args, **kwargs): + if pool_name: + self.assertEqual(vdolv.pool.lvname, pool_name) + ++ # nodiscard should be always set for VDO LV format ++ if vdolv.format.type: ++ self.assertTrue(vdolv.format._mkfs_nodiscard) ++ + return device + + @patch("blivet.formats.lvmpv.LVMPhysicalVolume.formattable", return_value=True) +@@ -633,6 +637,9 @@ def test_device_factory(self, *args): # pylint: disable=unused-argument,argumen + device = self._factory_device(device_type, **kwargs) + self._validate_factory_device(device, device_type, **kwargs) + ++ # change fstype ++ kwargs["fstype"] = "xfs" ++ + + class MDFactoryTestCase(DeviceFactoryTestCase): + device_type = devicefactory.DEVICE_TYPE_MD + +From 43f25ce84729c321d1ff2bbba2f50489f6d736b4 Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Thu, 19 Nov 2020 13:31:40 +0100 +Subject: [PATCH 12/17] Add LVM VDO example + +--- + examples/lvm_vdo.py | 61 +++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 61 insertions(+) + create mode 100644 examples/lvm_vdo.py + +diff --git a/examples/lvm_vdo.py b/examples/lvm_vdo.py +new file mode 100644 +index 00000000..ad081642 +--- /dev/null ++++ b/examples/lvm_vdo.py +@@ -0,0 +1,61 @@ ++import os ++ ++import blivet ++from blivet.size import Size ++from blivet.util import set_up_logging, create_sparse_tempfile ++ ++set_up_logging() ++b = blivet.Blivet() # create an instance of Blivet (don't add system devices) ++ ++# create a disk image file on which to create new devices ++disk1_file = create_sparse_tempfile("disk1", Size("100GiB")) ++b.disk_images["disk1"] = disk1_file ++disk2_file = create_sparse_tempfile("disk2", Size("100GiB")) ++b.disk_images["disk2"] = disk2_file ++ ++b.reset() ++ ++try: ++ disk1 = b.devicetree.get_device_by_name("disk1") ++ disk2 = b.devicetree.get_device_by_name("disk2") ++ ++ b.initialize_disk(disk1) ++ b.initialize_disk(disk2) ++ ++ pv = b.new_partition(size=Size("50GiB"), fmt_type="lvmpv", parents=[disk1]) ++ b.create_device(pv) ++ pv2 = b.new_partition(size=Size("50GiB"), fmt_type="lvmpv", parents=[disk2]) ++ b.create_device(pv2) ++ ++ # allocate the partitions (decide where and on which disks they'll reside) ++ blivet.partitioning.do_partitioning(b) ++ ++ vg = b.new_vg(parents=[pv, pv2]) ++ b.create_device(vg) ++ ++ # create 80 GiB VDO pool ++ # there can be only one VDO LV on the pool and these are created together ++ # with one LVM call, we have 2 separate devices because there are two block ++ # devices in the end and it allows to control the different "physical" size of ++ # the pool and "logical" size of the VDO LV (which is usually bigger, accounting ++ # for the saved space with deduplication and/or compression) ++ pool = b.new_lv(size=Size("80GiB"), parents=[vg], name="vdopool", vdo_pool=True, ++ deduplication=True, compression=True) ++ b.create_device(pool) ++ ++ # create the VDO LV with 400 GiB "virtual size" and ext4 filesystem on the VDO ++ # pool ++ lv = b.new_lv(size=Size("400GiB"), parents=[pool], name="vdolv", vdo_lv=True, ++ fmt_type="ext4") ++ b.create_device(lv) ++ ++ print(b.devicetree) ++ ++ # write the new partitions to disk and format them as specified ++ b.do_it() ++ print(b.devicetree) ++ input("Check the state and hit ENTER to trigger cleanup") ++finally: ++ b.devicetree.teardown_disk_images() ++ os.unlink(disk1_file) ++ os.unlink(disk2_file) + +From c487a1e6023b54f5beea8d99ba2f5da5d80590ee Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Wed, 25 Nov 2020 13:30:15 +0100 +Subject: [PATCH 13/17] Add LVM VDO documentation + +--- + doc/lvmvdo.rst | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 86 insertions(+) + create mode 100644 doc/lvmvdo.rst + +diff --git a/doc/lvmvdo.rst b/doc/lvmvdo.rst +new file mode 100644 +index 00000000..3965abd3 +--- /dev/null ++++ b/doc/lvmvdo.rst +@@ -0,0 +1,86 @@ ++LVM VDO support ++=============== ++ ++Support for creating LVM VDO devices has been added in Blivet 3.4. ++ ++These devices are similar to LVM thinly provisioned volumes, but there are some special steps ++and limitations when creating these devices which this document describes. ++ ++LVM VDO in Blivet ++----------------- ++ ++LVM VDO devices are represented by two ``LVMLogicalVolumeDevice`` devices: ++ ++- VDO Pool logical volume with type 'lvmvdopool' ++- VDO logical volume with type 'lvmvdolv' which is the child of the VDO Pool device ++ ++Existing LVM VDO setup in Blivet: ++ ++ existing 20 GiB disk vdb (265) with existing msdos disklabel ++ existing 20 GiB partition vdb1 (275) with existing lvmpv ++ existing 20 GiB lvmvg data (284) ++ existing 10 GiB lvmvdopool data-vdopool (288) ++ existing 50 GiB lvmvdolv data-vdolv (295) ++ ++When creating LVM VDO setup using Blivet these two devices must be created together as these ++are created by a single LVM command. ++ ++It currently isn't possible to create additional VDO logical volumes in the pool. It is however ++possible to create multiple VDO pools in a single volume group. ++ ++Deduplication and compression are properties of the VDO pool. Size specified for the VDO pool ++volume will be used as the "physical" size for the pool and size specified for the VDO logical volume ++will be used as the "virtual" size for the VDO volume. ++ ++When creating format, it must be created on the VDO logical volume. For filesystems with discard ++support, no discard option will be automatically added when calling the ``mkfs`` command ++(e.g. ``-K`` for ``mkfs.xfs``). ++ ++Example for creating a *80 GiB* VDO pool with *400 GiB* VDO logical volume with an *ext4* format with ++both deduplication and compression enabled: ++ ++ pool = b.new_lv(size=Size("80GiB"), parents=[vg], name="vdopool", vdo_pool=True, ++ deduplication=True, compression=True) ++ b.create_device(pool) ++ ++ lv = b.new_lv(size=Size("400GiB"), parents=[pool], name="vdolv", vdo_lv=True, ++ fmt_type="ext4") ++ b.create_device(lv) ++ ++When removing existing LVM VDO devices, both devices must be removed from the devicetree and the VDO ++logical volume must be removed first (``recursive_remove`` can be used to automate these two steps). ++ ++Managing of existing LVM VDO devices is currently not supported. ++ ++ ++LVM VDO in Devicefactory ++------------------------ ++ ++For the top-down specified creation using device factories a new ``LVMVDOFactory`` factory has been ++added. Factory device in this case is the VDO logical volume and is again automatically created ++together with the VDO pool. ++ ++Example of creating a new LVM VDO setup using the ``devicefactory`` module: ++ ++ factory = blivet.devicefactory.LVMVDOFactory(b, size=Size("5 GiB"), virtual_size=Size("50 GiB"), ++ disks=disks, fstype="xfs", ++ container_name="data", ++ pool_name="myvdopool", ++ compression=True, deduplication=True) ++ factory.configure() ++ factory.device ++ ++ LVMLogicalVolumeDevice instance (0x7f14d17422b0) -- ++ name = data-00 status = False id = 528 ++ children = [] ++ parents = ['non-existent 5 GiB lvmvdopool data-myvdopool (519)'] ++ ... ++ ++``size`` in this case sets the pool (physical) size, the VDO logical volume size can be specified ++with ``virtual_size`` (if not specified it will be same as the pool size). Name for the VDO volume ++can be specified using the ``name`` keyword argument. ``pool_name`` argument is optional and ++a unique name will be generated if omitted. Both ``compression`` and ``deduplication`` default to ++``True`` (enabled) if not specified. ++ ++This factory can create only a single VDO logical volume in a single VDO pool but additional VDO pools ++can be added by repeating the steps to create the first one. + +From c6c776cf137b5c6ae454487df469e9a6dba8a5d1 Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Wed, 9 Dec 2020 14:06:27 +0100 +Subject: [PATCH 14/17] Set minimum size for LVM VDO pool devices + +--- + blivet/devicefactory.py | 3 +++ + blivet/devices/lvm.py | 26 ++++++++++++++++++++++++++ + tests/devicefactory_test.py | 29 ++++++++++++++++++++--------- + tests/devices_test/lvm_test.py | 6 ++++++ + 4 files changed, 55 insertions(+), 9 deletions(-) + +diff --git a/blivet/devicefactory.py b/blivet/devicefactory.py +index 085f2fd6..5e47eb9a 100644 +--- a/blivet/devicefactory.py ++++ b/blivet/devicefactory.py +@@ -277,6 +277,7 @@ class DeviceFactory(object): + "container_size": SIZE_POLICY_AUTO, + "container_raid_level": None, + "container_encrypted": None} ++ _device_min_size = Size(0) # no limit by default, limited only by filesystem size + + def __init__(self, storage, **kwargs): + """ +@@ -1760,6 +1761,8 @@ class LVMVDOFactory(LVMFactory): + :type deduplication: bool + """ + ++ _device_min_size = LVMVDOPoolMixin._min_size ++ + def __init__(self, storage, **kwargs): + self.pool_name = kwargs.pop("pool_name", None) + self.virtual_size = kwargs.pop("virtual_size", None) +diff --git a/blivet/devices/lvm.py b/blivet/devices/lvm.py +index 0802e2de..785fa2d2 100644 +--- a/blivet/devices/lvm.py ++++ b/blivet/devices/lvm.py +@@ -1792,6 +1792,7 @@ def populate_ksdata(self, data): + class LVMVDOPoolMixin(object): + + _external_dependencies = [availability.BLOCKDEV_LVM_PLUGIN, availability.BLOCKDEV_LVM_PLUGIN_VDO] ++ _min_size = Size("5 GiB") # 2.5 GiB for index and one 2 GiB slab rounded up to 5 GiB + + def __init__(self, compression=True, deduplication=True, index_memory=0, write_policy=None): + self.compression = compression +@@ -1800,6 +1801,9 @@ def __init__(self, compression=True, deduplication=True, index_memory=0, write_p + self.write_policy = write_policy + self._lvs = [] + ++ if not self.exists and self.size < self.min_size: ++ raise ValueError("Requested size %s is smaller than minimum %s" % (self.size, self.min_size)) ++ + @property + def is_vdo_pool(self): + return self.seg_type == "vdo-pool" +@@ -1856,6 +1860,23 @@ def direct(self): + """ Is this device directly accessible? """ + return False + ++ @property ++ @util.requires_property("is_vdo_pool") ++ def min_size(self): ++ if self.exists: ++ return self.current_size ++ ++ return self._min_size ++ ++ def _set_size(self, newsize): ++ if not isinstance(newsize, Size): ++ raise AttributeError("new size must of type Size") ++ ++ if newsize < self.min_size: ++ raise ValueError("Requested size %s is smaller than minimum %s" % (newsize, self.min_size)) ++ ++ DMDevice._set_size(self, newsize) ++ + def read_current_size(self): + log_method_call(self, exists=self.exists, path=self.path, + sysfs_path=self.sysfs_path) +@@ -2229,6 +2250,11 @@ def max_size(self): + max_format = self.format.max_size + return min(max_lv, max_format) if max_format else max_lv + ++ @property ++ @type_specific ++ def min_size(self): ++ return super(LVMLogicalVolumeDevice, self).min_size ++ + @property + @type_specific + def vg_space_used(self): +diff --git a/tests/devicefactory_test.py b/tests/devicefactory_test.py +index 4de1e05b..a1334cda 100644 +--- a/tests/devicefactory_test.py ++++ b/tests/devicefactory_test.py +@@ -49,13 +49,18 @@ class DeviceFactoryTestCase(unittest.TestCase): + encryption_supported = True + """ whether encryption of this device type is supported by blivet """ + ++ factory_class = None ++ """ devicefactory class used in this test case """ ++ ++ _disk_size = Size("2 GiB") ++ + def setUp(self): + if self.device_type is None: + raise unittest.SkipTest("abstract base class") + + self.b = blivet.Blivet() # don't populate it +- self.disk_files = [create_sparse_tempfile("factorytest", Size("2 GiB")), +- create_sparse_tempfile("factorytest", Size("2 GiB"))] ++ self.disk_files = [create_sparse_tempfile("factorytest", self._disk_size), ++ create_sparse_tempfile("factorytest", self._disk_size)] + for filename in self.disk_files: + disk = DiskFile(filename) + self.b.devicetree._add_device(disk) +@@ -197,7 +202,7 @@ def _get_size_delta(self, devices=None): + def test_get_free_disk_space(self, *args): # pylint: disable=unused-argument + # get_free_disk_space should return the total free space on disks + kwargs = self._get_test_factory_args() +- kwargs["size"] = Size("500 MiB") ++ kwargs["size"] = max(Size("500 MiB"), self.factory_class._device_min_size) + factory = devicefactory.get_device_factory(self.b, + self.device_type, + disks=self.b.disks, +@@ -285,7 +290,7 @@ def test_factory_defaults(self, *args): # pylint: disable=unused-argument + kwargs = self._get_test_factory_args() + kwargs.update({"disks": self.b.disks[:], + "fstype": "swap", +- "size": Size("2GiB"), ++ "size": max(Size("2GiB"), self.factory_class._device_min_size), + "label": "SWAP"}) + device = self._factory_device(self.device_type, **kwargs) + factory = devicefactory.get_device_factory(self.b, self.device_type, +@@ -302,6 +307,7 @@ def test_factory_defaults(self, *args): # pylint: disable=unused-argument + class PartitionFactoryTestCase(DeviceFactoryTestCase): + device_class = PartitionDevice + device_type = devicefactory.DEVICE_TYPE_PARTITION ++ factory_class = devicefactory.PartitionFactory + + def test_bug1178884(self): + # Test a change of format and size where old size is too large for the +@@ -330,6 +336,7 @@ def _get_size_delta(self, devices=None): + class LVMFactoryTestCase(DeviceFactoryTestCase): + device_class = LVMLogicalVolumeDevice + device_type = devicefactory.DEVICE_TYPE_LVM ++ factory_class = devicefactory.LVMFactory + + def _validate_factory_device(self, *args, **kwargs): + super(LVMFactoryTestCase, self)._validate_factory_device(*args, **kwargs) +@@ -510,6 +517,7 @@ class LVMThinPFactoryTestCase(LVMFactoryTestCase): + device_class = LVMLogicalVolumeDevice + device_type = devicefactory.DEVICE_TYPE_LVM_THINP + encryption_supported = False ++ factory_class = devicefactory.LVMThinPFactory + + def _validate_factory_device(self, *args, **kwargs): + super(LVMThinPFactoryTestCase, self)._validate_factory_device(*args, +@@ -541,6 +549,8 @@ class LVMVDOFactoryTestCase(LVMFactoryTestCase): + device_class = LVMLogicalVolumeDevice + device_type = devicefactory.DEVICE_TYPE_LVM_VDO + encryption_supported = False ++ _disk_size = Size("10 GiB") # we need bigger disks for VDO ++ factory_class = devicefactory.LVMVDOFactory + + def _validate_factory_device(self, *args, **kwargs): + super(LVMVDOFactoryTestCase, self)._validate_factory_device(*args, +@@ -585,7 +595,7 @@ def _validate_factory_device(self, *args, **kwargs): + def test_device_factory(self, *args): # pylint: disable=unused-argument,arguments-differ + device_type = self.device_type + kwargs = {"disks": self.b.disks, +- "size": Size("400 MiB"), ++ "size": Size("6 GiB"), + "fstype": 'ext4', + "mountpoint": '/factorytest'} + device = self._factory_device(device_type, **kwargs) +@@ -593,7 +603,7 @@ def test_device_factory(self, *args): # pylint: disable=unused-argument,argumen + self.b.recursive_remove(device.pool) + + kwargs = {"disks": self.b.disks, +- "size": Size("400 MiB"), ++ "size": Size("6 GiB"), + "fstype": 'ext4', + "mountpoint": '/factorytest', + "pool_name": "vdopool", +@@ -603,19 +613,19 @@ def test_device_factory(self, *args): # pylint: disable=unused-argument,argumen + self._validate_factory_device(device, device_type, **kwargs) + + # change size without specifying virtual_size: both sizes should grow +- kwargs["size"] = Size("600 MiB") ++ kwargs["size"] = Size("8 GiB") + kwargs["device"] = device + device = self._factory_device(device_type, **kwargs) + self._validate_factory_device(device, device_type, **kwargs) + + # change virtual size +- kwargs["virtual_size"] = Size("6 GiB") ++ kwargs["virtual_size"] = Size("40 GiB") + kwargs["device"] = device + device = self._factory_device(device_type, **kwargs) + self._validate_factory_device(device, device_type, **kwargs) + + # change virtual size to smaller than size +- kwargs["virtual_size"] = Size("500 GiB") ++ kwargs["virtual_size"] = Size("10 GiB") + kwargs["device"] = device + device = self._factory_device(device_type, **kwargs) + self._validate_factory_device(device, device_type, **kwargs) +@@ -644,6 +654,7 @@ def test_device_factory(self, *args): # pylint: disable=unused-argument,argumen + class MDFactoryTestCase(DeviceFactoryTestCase): + device_type = devicefactory.DEVICE_TYPE_MD + device_class = MDRaidArrayDevice ++ factory_class = devicefactory.MDFactory + + def test_device_factory(self): + # RAID0 across two disks +diff --git a/tests/devices_test/lvm_test.py b/tests/devices_test/lvm_test.py +index 493d3ba1..78b140ba 100644 +--- a/tests/devices_test/lvm_test.py ++++ b/tests/devices_test/lvm_test.py +@@ -705,6 +705,12 @@ def test_new_vdo_pool(self): + + self.assertEqual(vg.size, Size("10236 MiB")) + ++ with self.assertRaises(ValueError): ++ vdopool = b.new_lv(name="vdopool", vdo_pool=True, ++ parents=[vg], compression=True, ++ deduplication=True, ++ size=blivet.size.Size("1 GiB")) ++ + vdopool = b.new_lv(name="vdopool", vdo_pool=True, + parents=[vg], compression=True, + deduplication=True, + +From 197f2877709e702c101ada6b9a055a88f09320c8 Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Fri, 11 Dec 2020 14:20:48 +0100 +Subject: [PATCH 15/17] Use better description for libblockdev plugins in + tasks.availability + +The old names were quite confusing when showing that "lvm" is +missing when in fact libblockdev LVM plugin is missing. Also with +LVM VDO we need to be able to tell the difference between missing +LVM plugin and missing LVM VDO support. +--- + blivet/tasks/availability.py | 26 +++++++++++++------------- + 1 file changed, 13 insertions(+), 13 deletions(-) + +diff --git a/blivet/tasks/availability.py b/blivet/tasks/availability.py +index b107428e..52418685 100644 +--- a/blivet/tasks/availability.py ++++ b/blivet/tasks/availability.py +@@ -236,13 +236,13 @@ def availability_errors(self, resource): + :returns: [] if the name of the plugin is loaded + :rtype: list of str + """ +- if resource.name not in blockdev.get_available_plugin_names(): # pylint: disable=no-value-for-parameter +- return ["libblockdev plugin %s not loaded" % resource.name] ++ if self._tech_info.plugin_name not in blockdev.get_available_plugin_names(): # pylint: disable=no-value-for-parameter ++ return ["libblockdev plugin %s not loaded" % self._tech_info.plugin_name] + else: + tech_missing = self._check_technologies() + if tech_missing: + return ["libblockdev plugin %s is loaded but some required " +- "technologies are not available:\n%s" % (resource.name, tech_missing)] ++ "technologies are not available:\n%s" % (self._tech_info.plugin_name, tech_missing)] + else: + return [] + +@@ -411,16 +411,16 @@ def available_resource(name): + # we can't just check if the plugin is loaded, we also need to make sure + # that all technologies required by us our supported (some may be missing + # due to missing dependencies) +-BLOCKDEV_BTRFS_PLUGIN = blockdev_plugin("btrfs", BLOCKDEV_BTRFS_TECH) +-BLOCKDEV_CRYPTO_PLUGIN = blockdev_plugin("crypto", BLOCKDEV_CRYPTO_TECH) +-BLOCKDEV_DM_PLUGIN = blockdev_plugin("dm", BLOCKDEV_DM_TECH) +-BLOCKDEV_DM_PLUGIN_RAID = blockdev_plugin("dm", BLOCKDEV_DM_TECH_RAID) +-BLOCKDEV_LOOP_PLUGIN = blockdev_plugin("loop", BLOCKDEV_LOOP_TECH) +-BLOCKDEV_LVM_PLUGIN = blockdev_plugin("lvm", BLOCKDEV_LVM_TECH) +-BLOCKDEV_LVM_PLUGIN_VDO = blockdev_plugin("lvm", BLOCKDEV_LVM_TECH_VDO) +-BLOCKDEV_MDRAID_PLUGIN = blockdev_plugin("mdraid", BLOCKDEV_MD_TECH) +-BLOCKDEV_MPATH_PLUGIN = blockdev_plugin("mpath", BLOCKDEV_MPATH_TECH) +-BLOCKDEV_SWAP_PLUGIN = blockdev_plugin("swap", BLOCKDEV_SWAP_TECH) ++BLOCKDEV_BTRFS_PLUGIN = blockdev_plugin("libblockdev btrfs plugin", BLOCKDEV_BTRFS_TECH) ++BLOCKDEV_CRYPTO_PLUGIN = blockdev_plugin("libblockdev crypto plugin", BLOCKDEV_CRYPTO_TECH) ++BLOCKDEV_DM_PLUGIN = blockdev_plugin("libblockdev dm plugin", BLOCKDEV_DM_TECH) ++BLOCKDEV_DM_PLUGIN_RAID = blockdev_plugin("libblockdev dm plugin (raid technology)", BLOCKDEV_DM_TECH_RAID) ++BLOCKDEV_LOOP_PLUGIN = blockdev_plugin("libblockdev loop plugin", BLOCKDEV_LOOP_TECH) ++BLOCKDEV_LVM_PLUGIN = blockdev_plugin("libblockdev lvm plugin", BLOCKDEV_LVM_TECH) ++BLOCKDEV_LVM_PLUGIN_VDO = blockdev_plugin("libblockdev lvm plugin (vdo technology)", BLOCKDEV_LVM_TECH_VDO) ++BLOCKDEV_MDRAID_PLUGIN = blockdev_plugin("libblockdev mdraid plugin", BLOCKDEV_MD_TECH) ++BLOCKDEV_MPATH_PLUGIN = blockdev_plugin("libblockdev mpath plugin", BLOCKDEV_MPATH_TECH) ++BLOCKDEV_SWAP_PLUGIN = blockdev_plugin("libblockdev swap plugin", BLOCKDEV_SWAP_TECH) + + # applications with versions + # we need e2fsprogs newer than 1.41 and we are checking the version by running + +From 5fc047b48b0de18fa249f102d2a7163ac2d6e6a6 Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Fri, 11 Dec 2020 14:24:18 +0100 +Subject: [PATCH 16/17] Fix external dependencies for LVM VDO devices + +The external and unavailable dependencies code is mostly supposed +to work with just class objects and not instances, which is problem +for LVM devices where the LVMLogicalVolumeDevice can't depend on +LVM VDO and special LVM VDO device mixin classes don't inherit +from the Device class so they are missing some availability +functions. +This fix adds the neccessary functions to LVM VDO mixin classes to +make sure both "unavailable_type_dependencies" and +"type_external_dependencies" work with LVMVDOLogicalVolumeMixin +and LVMVDOPoolMixin. When working with an LVMLogicalVolumeDevice +instance its dependencies are correctly set based on type of the +logical volume. +--- + blivet/devicefactory.py | 7 +-- + blivet/devices/lvm.py | 31 ++++++++++ + tests/action_test.py | 7 +++ + tests/devicefactory_test.py | 32 ++++++++++ + tests/devices_test/lvm_test.py | 106 +++++++++++++++++++++++++++++++++ + 5 files changed, 179 insertions(+), 4 deletions(-) + +diff --git a/blivet/devicefactory.py b/blivet/devicefactory.py +index 5e47eb9a..b29a107a 100644 +--- a/blivet/devicefactory.py ++++ b/blivet/devicefactory.py +@@ -27,7 +27,7 @@ + from .devices import BTRFSDevice, DiskDevice + from .devices import LUKSDevice, LVMLogicalVolumeDevice + from .devices import PartitionDevice, MDRaidArrayDevice +-from .devices.lvm import LVMVDOPoolMixin, DEFAULT_THPOOL_RESERVE ++from .devices.lvm import LVMVDOPoolMixin, LVMVDOLogicalVolumeMixin, DEFAULT_THPOOL_RESERVE + from .formats import get_format + from .devicelibs import btrfs + from .devicelibs import mdraid +@@ -70,9 +70,6 @@ def is_supported_device_type(device_type): + :returns: True if this device type is supported + :rtype: bool + """ +- if device_type == DEVICE_TYPE_LVM_VDO: +- return not any(e for e in LVMVDOPoolMixin._external_dependencies if not e.available) +- + devices = [] + if device_type == DEVICE_TYPE_BTRFS: + devices = [BTRFSDevice] +@@ -84,6 +81,8 @@ def is_supported_device_type(device_type): + devices = [PartitionDevice] + elif device_type == DEVICE_TYPE_MD: + devices = [MDRaidArrayDevice] ++ elif device_type == DEVICE_TYPE_LVM_VDO: ++ devices = [LVMLogicalVolumeDevice, LVMVDOPoolMixin, LVMVDOLogicalVolumeMixin] + + return not any(c.unavailable_type_dependencies() for c in devices) + +diff --git a/blivet/devices/lvm.py b/blivet/devices/lvm.py +index 785fa2d2..ac900bf3 100644 +--- a/blivet/devices/lvm.py ++++ b/blivet/devices/lvm.py +@@ -1804,6 +1804,17 @@ def __init__(self, compression=True, deduplication=True, index_memory=0, write_p + if not self.exists and self.size < self.min_size: + raise ValueError("Requested size %s is smaller than minimum %s" % (self.size, self.min_size)) + ++ # these two methods are defined in Device but LVMVDOPoolMixin doesn't inherit from ++ # it and we can't have this code in LVMLogicalVolumeDevice because we need to be able ++ # to get dependencies without creating instance of the class ++ @classmethod ++ def type_external_dependencies(cls): ++ return set(d for d in cls._external_dependencies) | LVMLogicalVolumeDevice.type_external_dependencies() ++ ++ @classmethod ++ def unavailable_type_dependencies(cls): ++ return set(e for e in cls.type_external_dependencies() if not e.available) ++ + @property + def is_vdo_pool(self): + return self.seg_type == "vdo-pool" +@@ -1926,6 +1937,17 @@ def _check_parents(self): + if not container or not isinstance(container, LVMLogicalVolumeDevice) or not container.is_vdo_pool: + raise ValueError("constructor requires a vdo-pool LV") + ++ # these two methods are defined in Device but LVMVDOLogicalVolumeMixin doesn't inherit ++ # from it and we can't have this code in LVMLogicalVolumeDevice because we need to be ++ # able to get dependencies without creating instance of the class ++ @classmethod ++ def type_external_dependencies(cls): ++ return set(d for d in cls._external_dependencies) | LVMLogicalVolumeDevice.type_external_dependencies() ++ ++ @classmethod ++ def unavailable_type_dependencies(cls): ++ return set(e for e in cls.type_external_dependencies() if not e.available) ++ + @property + def vg_space_used(self): + return Size(0) # the pool's size is already accounted for in the vg +@@ -2217,6 +2239,15 @@ def _convert_from_lvs(self): + """Convert the LVs to create this LV from into its internal LVs""" + raise ValueError("Cannot create a new LV of type '%s' from other LVs" % self.seg_type) + ++ @property ++ def external_dependencies(self): ++ deps = super(LVMLogicalVolumeBase, self).external_dependencies ++ if self.is_vdo_pool: ++ deps.update(LVMVDOPoolMixin.type_external_dependencies()) ++ if self.is_vdo_lv: ++ deps.update(LVMVDOLogicalVolumeMixin.type_external_dependencies()) ++ return deps ++ + @property + @type_specific + def vg(self): +diff --git a/tests/action_test.py b/tests/action_test.py +index 77176f46..38a2e872 100644 +--- a/tests/action_test.py ++++ b/tests/action_test.py +@@ -18,6 +18,8 @@ + from blivet.devices import MDRaidArrayDevice + from blivet.devices import LVMVolumeGroupDevice + from blivet.devices import LVMLogicalVolumeDevice ++from blivet.devices.lvm import LVMVDOPoolMixin ++from blivet.devices.lvm import LVMVDOLogicalVolumeMixin + + # format classes + from blivet.formats.fs import Ext2FS +@@ -1252,6 +1254,11 @@ def test_lv_from_lvs_actions(self): + self.assertEqual(set(self.storage.lvs), {pool}) + self.assertEqual(set(pool._internal_lvs), {lv1, lv2}) + ++ ++@unittest.skipUnless(not any(x.unavailable_type_dependencies() for x in DEVICE_CLASSES + [LVMVDOPoolMixin, LVMVDOLogicalVolumeMixin]), "some unsupported device classes required for this test") ++@unittest.skipUnless(all(x().utils_available for x in FORMAT_CLASSES), "some unsupported format classes required for this test") ++class DeviceActionLVMVDOTestCase(DeviceActionTestCase): ++ + def test_lvm_vdo_destroy(self): + self.destroy_all_devices() + sdc = self.storage.devicetree.get_device_by_name("sdc") +diff --git a/tests/devicefactory_test.py b/tests/devicefactory_test.py +index a1334cda..e4210ead 100644 +--- a/tests/devicefactory_test.py ++++ b/tests/devicefactory_test.py +@@ -592,6 +592,8 @@ def _validate_factory_device(self, *args, **kwargs): + @patch("blivet.static_data.lvm_info.blockdev.lvm.lvs", return_value=[]) + @patch("blivet.devices.lvm.LVMVolumeGroupDevice.type_external_dependencies", return_value=set()) + @patch("blivet.devices.lvm.LVMLogicalVolumeBase.type_external_dependencies", return_value=set()) ++ @patch("blivet.devices.lvm.LVMVDOPoolMixin.type_external_dependencies", return_value=set()) ++ @patch("blivet.devices.lvm.LVMVDOLogicalVolumeMixin.type_external_dependencies", return_value=set()) + def test_device_factory(self, *args): # pylint: disable=unused-argument,arguments-differ + device_type = self.device_type + kwargs = {"disks": self.b.disks, +@@ -650,6 +652,36 @@ def test_device_factory(self, *args): # pylint: disable=unused-argument,argumen + # change fstype + kwargs["fstype"] = "xfs" + ++ @patch("blivet.formats.lvmpv.LVMPhysicalVolume.formattable", return_value=True) ++ @patch("blivet.formats.lvmpv.LVMPhysicalVolume.destroyable", return_value=True) ++ @patch("blivet.static_data.lvm_info.blockdev.lvm.lvs", return_value=[]) ++ @patch("blivet.devices.lvm.LVMVolumeGroupDevice.type_external_dependencies", return_value=set()) ++ @patch("blivet.devices.lvm.LVMLogicalVolumeBase.type_external_dependencies", return_value=set()) ++ @patch("blivet.devices.lvm.LVMVDOPoolMixin.type_external_dependencies", return_value=set()) ++ @patch("blivet.devices.lvm.LVMVDOLogicalVolumeMixin.type_external_dependencies", return_value=set()) ++ def test_factory_defaults(self, *args): # pylint: disable=unused-argument ++ super(LVMVDOFactoryTestCase, self).test_factory_defaults() ++ ++ @patch("blivet.formats.lvmpv.LVMPhysicalVolume.formattable", return_value=True) ++ @patch("blivet.formats.lvmpv.LVMPhysicalVolume.destroyable", return_value=True) ++ @patch("blivet.static_data.lvm_info.blockdev.lvm.lvs", return_value=[]) ++ @patch("blivet.devices.lvm.LVMVolumeGroupDevice.type_external_dependencies", return_value=set()) ++ @patch("blivet.devices.lvm.LVMLogicalVolumeBase.type_external_dependencies", return_value=set()) ++ @patch("blivet.devices.lvm.LVMVDOPoolMixin.type_external_dependencies", return_value=set()) ++ @patch("blivet.devices.lvm.LVMVDOLogicalVolumeMixin.type_external_dependencies", return_value=set()) ++ def test_get_free_disk_space(self, *args): ++ super(LVMVDOFactoryTestCase, self).test_get_free_disk_space() ++ ++ @patch("blivet.formats.lvmpv.LVMPhysicalVolume.formattable", return_value=True) ++ @patch("blivet.formats.lvmpv.LVMPhysicalVolume.destroyable", return_value=True) ++ @patch("blivet.static_data.lvm_info.blockdev.lvm.lvs", return_value=[]) ++ @patch("blivet.devices.lvm.LVMVolumeGroupDevice.type_external_dependencies", return_value=set()) ++ @patch("blivet.devices.lvm.LVMLogicalVolumeBase.type_external_dependencies", return_value=set()) ++ @patch("blivet.devices.lvm.LVMVDOPoolMixin.type_external_dependencies", return_value=set()) ++ @patch("blivet.devices.lvm.LVMVDOLogicalVolumeMixin.type_external_dependencies", return_value=set()) ++ def test_normalize_size(self, *args): # pylint: disable=unused-argument ++ super(LVMVDOFactoryTestCase, self).test_normalize_size() ++ + + class MDFactoryTestCase(DeviceFactoryTestCase): + device_type = devicefactory.DEVICE_TYPE_MD +diff --git a/tests/devices_test/lvm_test.py b/tests/devices_test/lvm_test.py +index 78b140ba..d938144d 100644 +--- a/tests/devices_test/lvm_test.py ++++ b/tests/devices_test/lvm_test.py +@@ -10,10 +10,13 @@ + from blivet.devices import StorageDevice + from blivet.devices import LVMLogicalVolumeDevice + from blivet.devices import LVMVolumeGroupDevice ++from blivet.devices.lvm import LVMVDOPoolMixin ++from blivet.devices.lvm import LVMVDOLogicalVolumeMixin + from blivet.devices.lvm import LVMCacheRequest + from blivet.devices.lvm import LVPVSpec, LVMInternalLVtype + from blivet.size import Size + from blivet.devicelibs import raid ++from blivet import devicefactory + from blivet import errors + + DEVICE_CLASSES = [ +@@ -690,6 +693,10 @@ def test_new_lv_from_non_existing_lvs(self): + pool.create() + self.assertTrue(lvm.thpool_convert.called) + ++ ++@unittest.skipUnless(not any(x.unavailable_type_dependencies() for x in DEVICE_CLASSES + [LVMVDOPoolMixin, LVMVDOLogicalVolumeMixin]), "some unsupported device classes required for this test") ++class BlivetNewLVMVDODeviceTest(unittest.TestCase): ++ + def test_new_vdo_pool(self): + b = blivet.Blivet() + pv = StorageDevice("pv1", fmt=blivet.formats.get_format("lvmpv"), +@@ -726,3 +733,102 @@ def test_new_vdo_pool(self): + self.assertEqual(vdopool.children[0], vdolv) + self.assertEqual(vdolv.parents[0], vdopool) + self.assertListEqual(vg.lvs, [vdopool, vdolv]) ++ ++ ++@unittest.skipUnless(not any(x.unavailable_type_dependencies() for x in DEVICE_CLASSES), "some unsupported device classes required for this test") ++class BlivetLVMVDODependenciesTest(unittest.TestCase): ++ def test_vdo_dependencies(self): ++ blivet.tasks.availability.CACHE_AVAILABILITY = False ++ ++ b = blivet.Blivet() ++ pv = StorageDevice("pv1", fmt=blivet.formats.get_format("lvmpv"), ++ size=Size("10 GiB"), exists=True) ++ vg = LVMVolumeGroupDevice("testvg", parents=[pv], exists=True) ++ ++ for dev in (pv, vg): ++ b.devicetree._add_device(dev) ++ ++ # check that all the above devices are in the expected places ++ self.assertEqual(set(b.devices), {pv, vg}) ++ self.assertEqual(set(b.vgs), {vg}) ++ ++ self.assertEqual(vg.size, Size("10236 MiB")) ++ ++ vdopool = b.new_lv(name="vdopool", vdo_pool=True, ++ parents=[vg], compression=True, ++ deduplication=True, ++ size=blivet.size.Size("8 GiB")) ++ ++ vdolv = b.new_lv(name="vdolv", vdo_lv=True, ++ parents=[vdopool], ++ size=blivet.size.Size("40 GiB")) ++ ++ # Dependencies check: for VDO types these should be combination of "normal" ++ # LVM dependencies (LVM libblockdev plugin + kpartx and DM plugin from DMDevice) ++ # and LVM VDO technology from the LVM plugin ++ lvm_vdo_dependencies = ["kpartx", ++ "libblockdev dm plugin", ++ "libblockdev lvm plugin", ++ "libblockdev lvm plugin (vdo technology)"] ++ pool_deps = [d.name for d in vdopool.external_dependencies] ++ six.assertCountEqual(self, pool_deps, lvm_vdo_dependencies) ++ ++ vdolv_deps = [d.name for d in vdolv.external_dependencies] ++ six.assertCountEqual(self, vdolv_deps, lvm_vdo_dependencies) ++ ++ # same dependencies should be returned when checking with class not instance ++ pool_type_deps = [d.name for d in LVMVDOPoolMixin.type_external_dependencies()] ++ six.assertCountEqual(self, pool_type_deps, lvm_vdo_dependencies) ++ ++ vdolv_type_deps = [d.name for d in LVMVDOLogicalVolumeMixin.type_external_dependencies()] ++ six.assertCountEqual(self, vdolv_type_deps, lvm_vdo_dependencies) ++ ++ # just to be sure LVM VDO specific code didn't break "normal" LVs ++ normallv = b.new_lv(name="lvol0", ++ parents=[vg], ++ size=blivet.size.Size("1 GiB")) ++ ++ normalvl_deps = [d.name for d in normallv.external_dependencies] ++ six.assertCountEqual(self, normalvl_deps, ["kpartx", ++ "libblockdev dm plugin", ++ "libblockdev lvm plugin"]) ++ ++ with patch("blivet.devices.lvm.LVMVDOPoolMixin._external_dependencies", ++ new=[blivet.tasks.availability.unavailable_resource("VDO unavailability test")]): ++ with patch("blivet.devices.lvm.LVMVDOLogicalVolumeMixin._external_dependencies", ++ new=[blivet.tasks.availability.unavailable_resource("VDO unavailability test")]): ++ ++ pool_deps = [d.name for d in vdopool.unavailable_dependencies] ++ self.assertEqual(pool_deps, ["VDO unavailability test"]) ++ ++ vdolv_deps = [d.name for d in vdolv.unavailable_dependencies] ++ self.assertEqual(vdolv_deps, ["VDO unavailability test"]) ++ ++ # same dependencies should be returned when checking with class not instance ++ pool_type_deps = [d.name for d in LVMVDOPoolMixin.unavailable_type_dependencies()] ++ six.assertCountEqual(self, pool_type_deps, ["VDO unavailability test"]) ++ ++ vdolv_type_deps = [d.name for d in LVMVDOLogicalVolumeMixin.unavailable_type_dependencies()] ++ six.assertCountEqual(self, vdolv_type_deps, ["VDO unavailability test"]) ++ ++ normallv_deps = [d.name for d in normallv.unavailable_dependencies] ++ self.assertEqual(normallv_deps, []) ++ ++ with self.assertRaises(errors.DependencyError): ++ b.create_device(vdopool) ++ b.create_device(vdolv) ++ ++ b.create_device(normallv) ++ ++ def test_vdo_dependencies_devicefactory(self): ++ with patch("blivet.devices.lvm.LVMVDOPoolMixin._external_dependencies", ++ new=[blivet.tasks.availability.unavailable_resource("VDO unavailability test")]): ++ with patch("blivet.devices.lvm.LVMVDOLogicalVolumeMixin._external_dependencies", ++ new=[blivet.tasks.availability.unavailable_resource("VDO unavailability test")]): ++ ++ # shouldn't affect "normal" LVM ++ lvm_supported = devicefactory.is_supported_device_type(devicefactory.DEVICE_TYPE_LVM) ++ self.assertTrue(lvm_supported) ++ ++ vdo_supported = devicefactory.is_supported_device_type(devicefactory.DEVICE_TYPE_LVM_VDO) ++ self.assertFalse(vdo_supported) + +From c7fb125ec552ee5070f8180f92fe5545709192ff Mon Sep 17 00:00:00 2001 +From: Vojtech Trefny +Date: Fri, 11 Dec 2020 15:02:05 +0100 +Subject: [PATCH 17/17] Bump required libblockdev version to 2.24 + +LVM VDO support was added in 2.24. +--- + python-blivet.spec | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/python-blivet.spec b/python-blivet.spec +index ffd4210e..58cad0b2 100644 +--- a/python-blivet.spec ++++ b/python-blivet.spec +@@ -36,7 +36,7 @@ Source1: http://github.com/storaged-project/blivet/archive/%{realname}-%{realver + %global partedver 1.8.1 + %global pypartedver 3.10.4 + %global utillinuxver 2.15.1 +-%global libblockdevver 2.19 ++%global libblockdevver 2.24 + %global libbytesizever 0.3 + %global pyudevver 0.18 + diff --git a/SPECS/python-blivet.spec b/SPECS/python-blivet.spec index dd28979..a85e8d6 100644 --- a/SPECS/python-blivet.spec +++ b/SPECS/python-blivet.spec @@ -23,7 +23,7 @@ Version: 3.2.2 #%%global prerelease .b2 # prerelease, if defined, should be something like .a1, .b1, .b2.dev1, or .c2 -Release: 8%{?prerelease}%{?dist} +Release: 9%{?prerelease}%{?dist} Epoch: 1 License: LGPLv2+ Group: System Environment/Libraries @@ -49,6 +49,7 @@ Patch14: 0015-Fix-possible-UnicodeDecodeError-when-reading-model-f.patch Patch15: 0016-Basic-LVM-VDO-support.patch Patch16: 0017-Let-parted-fix-fixable-issues-with-partition-table.patch Patch17: 0018-Fix-possible-UnicodeDecodeError-when-reading-sysfs-a.patch +Patch18: 0019-LVM-VDO-support.patch # Versions of required components (done so we make sure the buildrequires # match the requires versions of things). @@ -210,6 +211,10 @@ configuration. %endif %changelog +* Tue Feb 9 2021 Vojtech Trefny - 3.2.2-9 +- LVM VDO support + Resolves: rhbz#1509337 + * Mon Jan 11 2021 Vojtech Trefny - 3.2.2-8 - Let parted fix fixable issues with partition table Resolves: rhbz#1846869