Blame SOURCES/0044-v2v-vcenter-Implement-cookie-scripts.patch

62f9b7
From cc9a507e2372b5b6408964f9c31a3bd526aabf7c Mon Sep 17 00:00:00 2001
62f9b7
From: "Richard W.M. Jones" <rjones@redhat.com>
62f9b7
Date: Wed, 23 Sep 2020 09:56:27 +0100
62f9b7
Subject: [PATCH] v2v: vcenter: Implement cookie scripts.
62f9b7
62f9b7
For conversions[*] which take longer than 30 minutes it can happen
62f9b7
that the HTTPS authorization cookie that we fetched from VMware when
62f9b7
we first connect expires.  This can especially happen when there are
62f9b7
multiple disks, because we may not "touch" (therefore autorenew) the
62f9b7
second disk while we are doing the long conversion.  This can lead to
62f9b7
failures, some of which are silent: again if there are multiple disks,
62f9b7
fstrim of the non-system disks can fail silently resulting in the copy
62f9b7
step taking a very long time.
62f9b7
62f9b7
The solution to this is to use the new nbdkit-curl-plugin
62f9b7
cookie-script feature which allows nbdkit to automatically renew the
62f9b7
cookie as required.
62f9b7
62f9b7
During the conversion or copying steps you may see the cookie being
62f9b7
autorenewed:
62f9b7
62f9b7
  nbdkit: curl[3]: debug: curl: running cookie-script
62f9b7
  nbdkit: curl[3]: debug: cookie-script returned cookies
62f9b7
62f9b7
This removes the ?user and ?password parameters from Nbdkit_sources.-
62f9b7
create_curl because they are no longer needed after this change.
62f9b7
Note for future: if we need to add them back, we must prevent both
62f9b7
user and cookie_script parameters from being used at the same time,
62f9b7
because simply having the user parameter will try basic
62f9b7
authentication, overriding the cookie, which will either fail (no
62f9b7
password) or run very slowly.
62f9b7
62f9b7
This change requires nbdkit >= 1.22 which is checked at runtime only
62f9b7
if this feature is used.
62f9b7
62f9b7
[*] Note here I mean conversions not the total runtime of virt-v2v.
62f9b7
When doing the copy the cookie does not expire because it is
62f9b7
continuously auto-renewed by VMware as we continuously access the disk
62f9b7
(this works differently from systems like Docker where the cookie is
62f9b7
only valid from the absolute time when it is first created).  This
62f9b7
change also implements the cookie-script logic for copying.
62f9b7
62f9b7
(cherry picked from commit 2b9a11743b74ef3716b66a7e395108a26382e331)
62f9b7
62f9b7
Notes for cherry pick to RHEL 8.6:
62f9b7
62f9b7
We no longer need the session_cookie field inside virt-v2v since it is
62f9b7
replaced by the cookie script.  However it is still needed by
62f9b7
virt-v2v-copy-to-local.  (This utility is removed upstream and in RHEL
62f9b7
9, but we need to keep it around at least for appearances in RHEL 8.)
62f9b7
62f9b7
So when cherry picking I had to retain the get_session_cookie function
62f9b7
which required also keeping fetch_headers_and_url as it was (not
62f9b7
dropping headers).
62f9b7
---
62f9b7
 v2v/nbdkit_sources.ml    | 34 ++++++++++++-----
62f9b7
 v2v/nbdkit_sources.mli   |  5 +--
62f9b7
 v2v/parse_libvirt_xml.ml |  3 +-
62f9b7
 v2v/vCenter.ml           | 80 +++++++++++++++++++++++++++++++---------
62f9b7
 4 files changed, 90 insertions(+), 32 deletions(-)
62f9b7
62f9b7
diff --git a/v2v/nbdkit_sources.ml b/v2v/nbdkit_sources.ml
62f9b7
index 7c177e35..16af5f5c 100644
62f9b7
--- a/v2v/nbdkit_sources.ml
62f9b7
+++ b/v2v/nbdkit_sources.ml
62f9b7
@@ -26,7 +26,6 @@ open Types
62f9b7
 open Utils
62f9b7
 
62f9b7
 let nbdkit_min_version = (1, 12, 0)
62f9b7
-let nbdkit_min_version_string = "1.12.0"
62f9b7
 
62f9b7
 type password =
62f9b7
 | NoPassword                    (* no password option at all *)
62f9b7
@@ -38,11 +37,16 @@ let error_unless_nbdkit_working () =
62f9b7
   if not (Nbdkit.is_installed ()) then
62f9b7
     error (f_"nbdkit is not installed or not working")
62f9b7
 
62f9b7
-let error_unless_nbdkit_min_version config =
62f9b7
+let error_unless_nbdkit_version_ge config min_version =
62f9b7
   let version = Nbdkit.version config in
62f9b7
-  if version < nbdkit_min_version then
62f9b7
-    error (f_"nbdkit is too old.  nbdkit >= %s is required.")
62f9b7
-          nbdkit_min_version_string
62f9b7
+  if version < min_version then (
62f9b7
+    let min_major, min_minor, min_release = min_version in
62f9b7
+    error (f_"nbdkit is too old.  nbdkit >= %d.%d.%d is required.")
62f9b7
+          min_major min_minor min_release
62f9b7
+  )
62f9b7
+
62f9b7
+let error_unless_nbdkit_min_version config =
62f9b7
+  error_unless_nbdkit_version_ge config nbdkit_min_version
62f9b7
 
62f9b7
 let error_unless_nbdkit_plugin_exists plugin =
62f9b7
   if not (Nbdkit.probe_plugin plugin) then
62f9b7
@@ -297,23 +301,35 @@ let create_ssh ?bandwidth ~password ?port ~server ?user path =
62f9b7
   common_create ?bandwidth password "ssh" (get_args ())
62f9b7
 
62f9b7
 (* Create an nbdkit module specialized for reading from Curl sources. *)
62f9b7
-let create_curl ?bandwidth ?cookie ~password ?(sslverify=true) ?user url =
62f9b7
+let create_curl ?bandwidth ?cookie_script ?cookie_script_renew
62f9b7
+                ?(sslverify=true) url =
62f9b7
   error_unless_nbdkit_plugin_exists "curl";
62f9b7
 
62f9b7
+  (* The cookie* parameters require nbdkit 1.22, so check that early. *)
62f9b7
+  if cookie_script <> None || cookie_script_renew <> None then (
62f9b7
+    let config = Nbdkit.config () in
62f9b7
+    error_unless_nbdkit_version_ge config (1, 22, 0)
62f9b7
+  );
62f9b7
+
62f9b7
   let add_arg, get_args =
62f9b7
     let args = ref [] in
62f9b7
     let add_arg (k, v) = List.push_front (k, v) args in
62f9b7
     let get_args () = List.rev !args in
62f9b7
     add_arg, get_args in
62f9b7
 
62f9b7
-  Option.may (fun s -> add_arg ("user", s)) user;
62f9b7
   (* https://bugzilla.redhat.com/show_bug.cgi?id=1146007#c10 *)
62f9b7
   add_arg ("timeout", "2000");
62f9b7
-  Option.may (fun s -> add_arg ("cookie", s)) cookie;
62f9b7
+  Option.may (fun s -> add_arg ("cookie-script", s)) cookie_script;
62f9b7
+  Option.may (fun i -> add_arg ("cookie-script-renew", string_of_int i))
62f9b7
+             cookie_script_renew;
62f9b7
   if not sslverify then add_arg ("sslverify", "false");
62f9b7
   add_arg ("url", url);
62f9b7
 
62f9b7
-  common_create ?bandwidth password "curl" (get_args ())
62f9b7
+  (* For lots of extra debugging, uncomment one or both lines. *)
62f9b7
+  (*add_arg ("--debug", "curl.verbose=1");*)
62f9b7
+  (*add_arg ("--debug", "curl.scripts=1");*)
62f9b7
+
62f9b7
+  common_create ?bandwidth NoPassword "curl" (get_args ())
62f9b7
 
62f9b7
 let run cmd =
62f9b7
   let sock, _ = Nbdkit.run_unix cmd in
62f9b7
diff --git a/v2v/nbdkit_sources.mli b/v2v/nbdkit_sources.mli
62f9b7
index 94810ea6..922642df 100644
62f9b7
--- a/v2v/nbdkit_sources.mli
62f9b7
+++ b/v2v/nbdkit_sources.mli
62f9b7
@@ -60,10 +60,9 @@ val create_ssh : ?bandwidth:Types.bandwidth ->
62f9b7
     Note this doesn't run nbdkit yet, it just creates the object. *)
62f9b7
 
62f9b7
 val create_curl : ?bandwidth:Types.bandwidth ->
62f9b7
-                  ?cookie:string ->
62f9b7
-                  password:password ->
62f9b7
+                  ?cookie_script:string ->
62f9b7
+                  ?cookie_script_renew:int ->
62f9b7
                   ?sslverify:bool ->
62f9b7
-                  ?user:string ->
62f9b7
                   string -> Nbdkit.cmd
62f9b7
 (** Create a nbdkit object using the Curl plugin.  The required
62f9b7
     string parameter is the URL.
62f9b7
diff --git a/v2v/parse_libvirt_xml.ml b/v2v/parse_libvirt_xml.ml
62f9b7
index 0b136839..fffc5a24 100644
62f9b7
--- a/v2v/parse_libvirt_xml.ml
62f9b7
+++ b/v2v/parse_libvirt_xml.ml
62f9b7
@@ -319,8 +319,7 @@ let parse_libvirt_xml ?bandwidth ?conn xml =
62f9b7
                | _, Some port ->
62f9b7
                   invalid_arg "invalid port number in libvirt XML" in
62f9b7
              sprintf "%s://%s%s%s" driver host port (uri_quote path) in
62f9b7
-           let nbdkit = Nbdkit_sources.create_curl ?bandwidth ~password:NoPassword
62f9b7
-                                           url in
62f9b7
+           let nbdkit = Nbdkit_sources.create_curl ?bandwidth url in
62f9b7
            let qemu_uri = Nbdkit_sources.run nbdkit in
62f9b7
            add_disk qemu_uri format controller P_dont_rewrite
62f9b7
         | Some protocol, _, _ ->
62f9b7
diff --git a/v2v/vCenter.ml b/v2v/vCenter.ml
62f9b7
index 4c128b0c..ead03364 100644
62f9b7
--- a/v2v/vCenter.ml
62f9b7
+++ b/v2v/vCenter.ml
62f9b7
@@ -46,11 +46,12 @@ let rec map_source ?bandwidth ?password_file dcPath uri server path =
62f9b7
        (* XXX only works if the query string is not URI-quoted *)
62f9b7
        String.find query "no_verify=1" = -1 in
62f9b7
 
62f9b7
+  (* Check the URL exists and authentication info is correct. *)
62f9b7
   let https_url =
62f9b7
     let https_url = get_https_url dcPath uri server path in
62f9b7
-    (* Check the URL exists. *)
62f9b7
-    let status, _, _ =
62f9b7
+    let status, _, dump_response =
62f9b7
       fetch_headers_from_url password_file uri sslverify https_url in
62f9b7
+
62f9b7
     (* If a disk is actually a snapshot image it will have '-00000n'
62f9b7
      * appended to its name, e.g.:
62f9b7
      *   [yellow:storage1] RHEL4-X/RHEL4-X-000003.vmdk
62f9b7
@@ -58,28 +59,71 @@ let rec map_source ?bandwidth ?password_file dcPath uri server path =
62f9b7
      * a 404 and the vmdk name looks like it might be a snapshot, try
62f9b7
      * again without the snapshot suffix.
62f9b7
      *)
62f9b7
-    if status = "404" && PCRE.matches snapshot_re path then (
62f9b7
-      let path = PCRE.sub 1 ^ PCRE.sub 2 in
62f9b7
-      get_https_url dcPath uri server path
62f9b7
-    )
62f9b7
-    else
62f9b7
-      (* Note that other non-200 status errors will be handled
62f9b7
-       * in get_session_cookie below, so we don't have to worry
62f9b7
-       * about them here.
62f9b7
-       *)
62f9b7
-      https_url in
62f9b7
+    let https_url, status, dump_response =
62f9b7
+      if status = "404" && PCRE.matches snapshot_re path then (
62f9b7
+        let path = PCRE.sub 1 ^ PCRE.sub 2 in
62f9b7
+        let https_url = get_https_url dcPath uri server path in
62f9b7
+        let status, _, dump_response =
62f9b7
+          fetch_headers_from_url password_file uri sslverify https_url in
62f9b7
+        https_url, status, dump_response
62f9b7
+      )
62f9b7
+      else (https_url, status, dump_response) in
62f9b7
+
62f9b7
+    if status = "401" then (
62f9b7
+      dump_response stderr;
62f9b7
+      if uri.uri_user <> None then
62f9b7
+        error (f_"vcenter: incorrect username or password")
62f9b7
+      else
62f9b7
+        error (f_"vcenter: incorrect username or password.  You might need to specify the username in the URI like this: [vpx|esx|..]://USERNAME@[etc]")
62f9b7
+    );
62f9b7
+
62f9b7
+    if status = "404" then (
62f9b7
+      dump_response stderr;
62f9b7
+      error (f_"vcenter: URL not found: %s") https_url
62f9b7
+    );
62f9b7
+
62f9b7
+    if status <> "200" then (
62f9b7
+      dump_response stderr;
62f9b7
+      error (f_"vcenter: invalid response from server: %s") status
62f9b7
+    );
62f9b7
+
62f9b7
+    https_url in
62f9b7
 
62f9b7
   let session_cookie =
62f9b7
     get_session_cookie password_file uri sslverify https_url in
62f9b7
 
62f9b7
-  let password =
62f9b7
-    match password_file with
62f9b7
-    | None -> Nbdkit_sources.NoPassword
62f9b7
-    | Some password_file -> Nbdkit_sources.PasswordFile password_file in
62f9b7
+  (* Write a cookie script to retrieve the session cookie.
62f9b7
+   * See nbdkit-curl-plugin(1) "Example: VMware ESXi cookies"
62f9b7
+   *)
62f9b7
+  let cookie_script, chan =
62f9b7
+    Filename.open_temp_file ~perms:0o700 "v2vcs" ".sh" in
62f9b7
+  unlink_on_exit cookie_script;
62f9b7
+  let fpf fs = fprintf chan fs in
62f9b7
+  fpf "#!/bin/sh -\n";
62f9b7
+  fpf "\n";
62f9b7
+  fpf "curl --head -s";
62f9b7
+  if not sslverify then fpf " --insecure";
62f9b7
+  (match uri.uri_user, password_file with
62f9b7
+   | None, None -> ()
62f9b7
+   | Some user, None -> fpf " -u %s" (quote user)
62f9b7
+   | None, Some password_file ->
62f9b7
+      fpf " -u \"$LOGNAME\":\"$(cat %s)\"" (quote password_file)
62f9b7
+   | Some user, Some password_file ->
62f9b7
+      fpf " -u %s:\"$(cat %s)\"" (quote user) (quote password_file)
62f9b7
+  );
62f9b7
+  fpf " %s" (quote https_url);
62f9b7
+  fpf " |\n";
62f9b7
+  fpf "\tsed -ne %s\n" (quote "{ s/^Set-Cookie: \\([^;]*\\);.*/\\1/ip }");
62f9b7
+  close_out chan;
62f9b7
+
62f9b7
+  (* VMware authentication expires after 30 minutes so we must renew
62f9b7
+   * after < 30 minutes.
62f9b7
+   *)
62f9b7
+  let cookie_script_renew = 25*60 in
62f9b7
 
62f9b7
   let nbdkit =
62f9b7
-    Nbdkit_sources.create_curl ?bandwidth ?cookie:session_cookie ~password ~sslverify
62f9b7
-                       ?user:uri.uri_user https_url in
62f9b7
+    Nbdkit_sources.create_curl ?bandwidth ~cookie_script ~cookie_script_renew
62f9b7
+                               ~sslverify https_url in
62f9b7
   let qemu_uri = Nbdkit_sources.run nbdkit in
62f9b7
 
62f9b7
   (* Return the struct. *)
62f9b7
-- 
0602f3
2.31.1
62f9b7