diff --git a/SOURCES/0044-v2v-vcenter-Implement-cookie-scripts.patch b/SOURCES/0044-v2v-vcenter-Implement-cookie-scripts.patch new file mode 100644 index 0000000..f71fde4 --- /dev/null +++ b/SOURCES/0044-v2v-vcenter-Implement-cookie-scripts.patch @@ -0,0 +1,275 @@ +From cc9a507e2372b5b6408964f9c31a3bd526aabf7c Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 23 Sep 2020 09:56:27 +0100 +Subject: [PATCH] v2v: vcenter: Implement cookie scripts. + +For conversions[*] which take longer than 30 minutes it can happen +that the HTTPS authorization cookie that we fetched from VMware when +we first connect expires. This can especially happen when there are +multiple disks, because we may not "touch" (therefore autorenew) the +second disk while we are doing the long conversion. This can lead to +failures, some of which are silent: again if there are multiple disks, +fstrim of the non-system disks can fail silently resulting in the copy +step taking a very long time. + +The solution to this is to use the new nbdkit-curl-plugin +cookie-script feature which allows nbdkit to automatically renew the +cookie as required. + +During the conversion or copying steps you may see the cookie being +autorenewed: + + nbdkit: curl[3]: debug: curl: running cookie-script + nbdkit: curl[3]: debug: cookie-script returned cookies + +This removes the ?user and ?password parameters from Nbdkit_sources.- +create_curl because they are no longer needed after this change. +Note for future: if we need to add them back, we must prevent both +user and cookie_script parameters from being used at the same time, +because simply having the user parameter will try basic +authentication, overriding the cookie, which will either fail (no +password) or run very slowly. + +This change requires nbdkit >= 1.22 which is checked at runtime only +if this feature is used. + +[*] Note here I mean conversions not the total runtime of virt-v2v. +When doing the copy the cookie does not expire because it is +continuously auto-renewed by VMware as we continuously access the disk +(this works differently from systems like Docker where the cookie is +only valid from the absolute time when it is first created). This +change also implements the cookie-script logic for copying. + +(cherry picked from commit 2b9a11743b74ef3716b66a7e395108a26382e331) + +Notes for cherry pick to RHEL 8.6: + +We no longer need the session_cookie field inside virt-v2v since it is +replaced by the cookie script. However it is still needed by +virt-v2v-copy-to-local. (This utility is removed upstream and in RHEL +9, but we need to keep it around at least for appearances in RHEL 8.) + +So when cherry picking I had to retain the get_session_cookie function +which required also keeping fetch_headers_and_url as it was (not +dropping headers). +--- + v2v/nbdkit_sources.ml | 34 ++++++++++++----- + v2v/nbdkit_sources.mli | 5 +-- + v2v/parse_libvirt_xml.ml | 3 +- + v2v/vCenter.ml | 80 +++++++++++++++++++++++++++++++--------- + 4 files changed, 90 insertions(+), 32 deletions(-) + +diff --git a/v2v/nbdkit_sources.ml b/v2v/nbdkit_sources.ml +index 7c177e35..16af5f5c 100644 +--- a/v2v/nbdkit_sources.ml ++++ b/v2v/nbdkit_sources.ml +@@ -26,7 +26,6 @@ open Types + open Utils + + let nbdkit_min_version = (1, 12, 0) +-let nbdkit_min_version_string = "1.12.0" + + type password = + | NoPassword (* no password option at all *) +@@ -38,11 +37,16 @@ let error_unless_nbdkit_working () = + if not (Nbdkit.is_installed ()) then + error (f_"nbdkit is not installed or not working") + +-let error_unless_nbdkit_min_version config = ++let error_unless_nbdkit_version_ge config min_version = + let version = Nbdkit.version config in +- if version < nbdkit_min_version then +- error (f_"nbdkit is too old. nbdkit >= %s is required.") +- nbdkit_min_version_string ++ if version < min_version then ( ++ let min_major, min_minor, min_release = min_version in ++ error (f_"nbdkit is too old. nbdkit >= %d.%d.%d is required.") ++ min_major min_minor min_release ++ ) ++ ++let error_unless_nbdkit_min_version config = ++ error_unless_nbdkit_version_ge config nbdkit_min_version + + let error_unless_nbdkit_plugin_exists plugin = + if not (Nbdkit.probe_plugin plugin) then +@@ -297,23 +301,35 @@ let create_ssh ?bandwidth ~password ?port ~server ?user path = + common_create ?bandwidth password "ssh" (get_args ()) + + (* Create an nbdkit module specialized for reading from Curl sources. *) +-let create_curl ?bandwidth ?cookie ~password ?(sslverify=true) ?user url = ++let create_curl ?bandwidth ?cookie_script ?cookie_script_renew ++ ?(sslverify=true) url = + error_unless_nbdkit_plugin_exists "curl"; + ++ (* The cookie* parameters require nbdkit 1.22, so check that early. *) ++ if cookie_script <> None || cookie_script_renew <> None then ( ++ let config = Nbdkit.config () in ++ error_unless_nbdkit_version_ge config (1, 22, 0) ++ ); ++ + let add_arg, get_args = + let args = ref [] in + let add_arg (k, v) = List.push_front (k, v) args in + let get_args () = List.rev !args in + add_arg, get_args in + +- Option.may (fun s -> add_arg ("user", s)) user; + (* https://bugzilla.redhat.com/show_bug.cgi?id=1146007#c10 *) + add_arg ("timeout", "2000"); +- Option.may (fun s -> add_arg ("cookie", s)) cookie; ++ Option.may (fun s -> add_arg ("cookie-script", s)) cookie_script; ++ Option.may (fun i -> add_arg ("cookie-script-renew", string_of_int i)) ++ cookie_script_renew; + if not sslverify then add_arg ("sslverify", "false"); + add_arg ("url", url); + +- common_create ?bandwidth password "curl" (get_args ()) ++ (* For lots of extra debugging, uncomment one or both lines. *) ++ (*add_arg ("--debug", "curl.verbose=1");*) ++ (*add_arg ("--debug", "curl.scripts=1");*) ++ ++ common_create ?bandwidth NoPassword "curl" (get_args ()) + + let run cmd = + let sock, _ = Nbdkit.run_unix cmd in +diff --git a/v2v/nbdkit_sources.mli b/v2v/nbdkit_sources.mli +index 94810ea6..922642df 100644 +--- a/v2v/nbdkit_sources.mli ++++ b/v2v/nbdkit_sources.mli +@@ -60,10 +60,9 @@ val create_ssh : ?bandwidth:Types.bandwidth -> + Note this doesn't run nbdkit yet, it just creates the object. *) + + val create_curl : ?bandwidth:Types.bandwidth -> +- ?cookie:string -> +- password:password -> ++ ?cookie_script:string -> ++ ?cookie_script_renew:int -> + ?sslverify:bool -> +- ?user:string -> + string -> Nbdkit.cmd + (** Create a nbdkit object using the Curl plugin. The required + string parameter is the URL. +diff --git a/v2v/parse_libvirt_xml.ml b/v2v/parse_libvirt_xml.ml +index 0b136839..fffc5a24 100644 +--- a/v2v/parse_libvirt_xml.ml ++++ b/v2v/parse_libvirt_xml.ml +@@ -319,8 +319,7 @@ let parse_libvirt_xml ?bandwidth ?conn xml = + | _, Some port -> + invalid_arg "invalid port number in libvirt XML" in + sprintf "%s://%s%s%s" driver host port (uri_quote path) in +- let nbdkit = Nbdkit_sources.create_curl ?bandwidth ~password:NoPassword +- url in ++ let nbdkit = Nbdkit_sources.create_curl ?bandwidth url in + let qemu_uri = Nbdkit_sources.run nbdkit in + add_disk qemu_uri format controller P_dont_rewrite + | Some protocol, _, _ -> +diff --git a/v2v/vCenter.ml b/v2v/vCenter.ml +index 4c128b0c..ead03364 100644 +--- a/v2v/vCenter.ml ++++ b/v2v/vCenter.ml +@@ -46,11 +46,12 @@ let rec map_source ?bandwidth ?password_file dcPath uri server path = + (* XXX only works if the query string is not URI-quoted *) + String.find query "no_verify=1" = -1 in + ++ (* Check the URL exists and authentication info is correct. *) + let https_url = + let https_url = get_https_url dcPath uri server path in +- (* Check the URL exists. *) +- let status, _, _ = ++ let status, _, dump_response = + fetch_headers_from_url password_file uri sslverify https_url in ++ + (* If a disk is actually a snapshot image it will have '-00000n' + * appended to its name, e.g.: + * [yellow:storage1] RHEL4-X/RHEL4-X-000003.vmdk +@@ -58,28 +59,71 @@ let rec map_source ?bandwidth ?password_file dcPath uri server path = + * a 404 and the vmdk name looks like it might be a snapshot, try + * again without the snapshot suffix. + *) +- if status = "404" && PCRE.matches snapshot_re path then ( +- let path = PCRE.sub 1 ^ PCRE.sub 2 in +- get_https_url dcPath uri server path +- ) +- else +- (* Note that other non-200 status errors will be handled +- * in get_session_cookie below, so we don't have to worry +- * about them here. +- *) +- https_url in ++ let https_url, status, dump_response = ++ if status = "404" && PCRE.matches snapshot_re path then ( ++ let path = PCRE.sub 1 ^ PCRE.sub 2 in ++ let https_url = get_https_url dcPath uri server path in ++ let status, _, dump_response = ++ fetch_headers_from_url password_file uri sslverify https_url in ++ https_url, status, dump_response ++ ) ++ else (https_url, status, dump_response) in ++ ++ if status = "401" then ( ++ dump_response stderr; ++ if uri.uri_user <> None then ++ error (f_"vcenter: incorrect username or password") ++ else ++ error (f_"vcenter: incorrect username or password. You might need to specify the username in the URI like this: [vpx|esx|..]://USERNAME@[etc]") ++ ); ++ ++ if status = "404" then ( ++ dump_response stderr; ++ error (f_"vcenter: URL not found: %s") https_url ++ ); ++ ++ if status <> "200" then ( ++ dump_response stderr; ++ error (f_"vcenter: invalid response from server: %s") status ++ ); ++ ++ https_url in + + let session_cookie = + get_session_cookie password_file uri sslverify https_url in + +- let password = +- match password_file with +- | None -> Nbdkit_sources.NoPassword +- | Some password_file -> Nbdkit_sources.PasswordFile password_file in ++ (* Write a cookie script to retrieve the session cookie. ++ * See nbdkit-curl-plugin(1) "Example: VMware ESXi cookies" ++ *) ++ let cookie_script, chan = ++ Filename.open_temp_file ~perms:0o700 "v2vcs" ".sh" in ++ unlink_on_exit cookie_script; ++ let fpf fs = fprintf chan fs in ++ fpf "#!/bin/sh -\n"; ++ fpf "\n"; ++ fpf "curl --head -s"; ++ if not sslverify then fpf " --insecure"; ++ (match uri.uri_user, password_file with ++ | None, None -> () ++ | Some user, None -> fpf " -u %s" (quote user) ++ | None, Some password_file -> ++ fpf " -u \"$LOGNAME\":\"$(cat %s)\"" (quote password_file) ++ | Some user, Some password_file -> ++ fpf " -u %s:\"$(cat %s)\"" (quote user) (quote password_file) ++ ); ++ fpf " %s" (quote https_url); ++ fpf " |\n"; ++ fpf "\tsed -ne %s\n" (quote "{ s/^Set-Cookie: \\([^;]*\\);.*/\\1/ip }"); ++ close_out chan; ++ ++ (* VMware authentication expires after 30 minutes so we must renew ++ * after < 30 minutes. ++ *) ++ let cookie_script_renew = 25*60 in + + let nbdkit = +- Nbdkit_sources.create_curl ?bandwidth ?cookie:session_cookie ~password ~sslverify +- ?user:uri.uri_user https_url in ++ Nbdkit_sources.create_curl ?bandwidth ~cookie_script ~cookie_script_renew ++ ~sslverify https_url in + let qemu_uri = Nbdkit_sources.run nbdkit in + + (* Return the struct. *) +-- +2.18.4 + diff --git a/SOURCES/copy-patches.sh b/SOURCES/copy-patches.sh index 9ee98b6..5abf976 100755 --- a/SOURCES/copy-patches.sh +++ b/SOURCES/copy-patches.sh @@ -6,7 +6,7 @@ set -e # directory. Use it like this: # ./copy-patches.sh -rhel_version=av-8.5.0 +rhel_version=8.6.0 # Check we're in the right directory. if [ ! -f virt-v2v.spec ]; then diff --git a/SPECS/virt-v2v.spec b/SPECS/virt-v2v.spec index c4518b8..aa2ae46 100644 --- a/SPECS/virt-v2v.spec +++ b/SPECS/virt-v2v.spec @@ -10,7 +10,7 @@ Name: virt-v2v Epoch: 1 Version: 1.42.0 -Release: 15%{?dist} +Release: 16%{?dist} Summary: Convert a virtual machine to run on KVM License: GPLv2+ @@ -31,7 +31,7 @@ Source2: libguestfs.keyring ExclusiveArch: x86_64 # RHEL 8 git repository is: -# https://github.com/libguestfs/virt-v2v/tree/rhel-av-8.5.0 +# https://github.com/libguestfs/virt-v2v/tree/rhel-8.6.0 # Use 'copy-patches.sh' to copy the patches from the git repo # to the current directory. @@ -79,6 +79,7 @@ Patch0040: 0040-RHEL-8-docs-Fix-version-of-virt-v2v-which-added-UEFI.patch Patch0041: 0041-v2v-Increase-Linux-minimum-root-filesystem-to-100-MB.patch Patch0042: 0042-v2v-rhv-upload-plugin-Fix-waiting-for-finalize.patch Patch0043: 0043-v2v-windows-Do-not-fix-NTFS-heads-in-Windows-Vista-a.patch +Patch0044: 0044-v2v-vcenter-Implement-cookie-scripts.patch # Patches which apply to the common/ submodule. # These have to be hand-modified. @@ -322,6 +323,10 @@ rm $RPM_BUILD_ROOT%{_mandir}/man1/virt-v2v-test-harness.1* %changelog +* Fri Oct 29 2021 Richard W.M. Jones - 1:1.42.0-16 +- Implement cookie scripts for more reliable vCenter/HTTPS transfers + resolves: rhbz#2018173 + * Wed Aug 18 2021 Richard W.M. Jones - 1:1.42.0-15 - v2v: windows: Do not fix NTFS heads in Windows Vista and later resolves: rhbz#1995000