Blame SOURCES/0024-v2v-Allow-large-temporary-directory-to-be-set-on-a-g.patch

7ed5e3
From 186c237ac1cb6f6830cfe2d08dfdcfdbdffab264 Mon Sep 17 00:00:00 2001
7ed5e3
From: "Richard W.M. Jones" <rjones@redhat.com>
7ed5e3
Date: Mon, 6 Apr 2020 10:19:12 +0100
7ed5e3
Subject: [PATCH] v2v: Allow large temporary directory to be set on a global
7ed5e3
 basis.
7ed5e3
7ed5e3
Previously we placed large files in g#get_cachedir () (usually
7ed5e3
/var/tmp).  However the problem is this ties the libguestfs appliance
7ed5e3
and the virt-v2v overlay files to the same location.
7ed5e3
7ed5e3
When virt-v2v is run in a container, or any other situation where
7ed5e3
local storage is limited, it's helpful to be able to put the overlay
7ed5e3
files on an externally mounted PVC, which might be using NFS and
7ed5e3
shared between containers.  But putting the libguestfs appliance on
7ed5e3
NFS in a shared location is certainly not recommended.
7ed5e3
7ed5e3
This allows the two locations to be set separately:
7ed5e3
7ed5e3
  VIRT_V2V_TMPDIR - location of large temporary files, can use NFS
7ed5e3
                    and may be shared
7ed5e3
7ed5e3
  LIBGUESTFS_CACHEDIR - location of libguestfs appliance
7ed5e3
7ed5e3
Another motivation for this patch is to allow more reliable cleanup of
7ed5e3
large temporary files by an external process, as described in the
7ed5e3
updated documentation.
7ed5e3
7ed5e3
Small temporary files are placed in $TMPDIR (usually /tmp).  I cleaned
7ed5e3
up some existing code which used /var/tmp for small temporaries.
7ed5e3
7ed5e3
(cherry picked from commit 717b808bc5cb632778973eb000600e87eaf5c31a)
7ed5e3
---
7ed5e3
 docs/virt-v2v.pod        | 27 +++++++++++++++++++--------
7ed5e3
 v2v/input_ova.ml         |  4 ++--
7ed5e3
 v2v/input_vmx.ml         |  3 +--
7ed5e3
 v2v/output_glance.ml     |  3 +--
7ed5e3
 v2v/output_null.ml       |  3 +--
7ed5e3
 v2v/output_rhv_upload.ml | 29 ++++++++++++++++-------------
7ed5e3
 v2v/parse_ova.ml         |  6 ++----
7ed5e3
 v2v/python_script.ml     | 12 +++---------
7ed5e3
 v2v/python_script.mli    |  5 +----
7ed5e3
 v2v/utils.ml             |  6 +++++-
7ed5e3
 v2v/utils.mli            |  5 +++++
7ed5e3
 v2v/v2v.ml               | 14 ++++++--------
7ed5e3
 12 files changed, 62 insertions(+), 55 deletions(-)
7ed5e3
7ed5e3
diff --git a/docs/virt-v2v.pod b/docs/virt-v2v.pod
7ed5e3
index 6f9f323e..af69d633 100644
7ed5e3
--- a/docs/virt-v2v.pod
7ed5e3
+++ b/docs/virt-v2v.pod
7ed5e3
@@ -1172,8 +1172,8 @@ possible.
7ed5e3
 =head3 Disk space
7ed5e3
 
7ed5e3
 Virt-v2v places potentially large temporary files in
7ed5e3
-C<$LIBGUESTFS_CACHEDIR> (which is F if you don't set it).
7ed5e3
-Using tmpfs is a bad idea.
7ed5e3
+C<$VIRT_V2V_TMPDIR> (usually F, see also
7ed5e3
+L</ENVIRONMENT VARIBLES> below).  Using tmpfs is a bad idea.
7ed5e3
 
7ed5e3
 For each guest disk, an overlay is stored temporarily.  This stores
7ed5e3
 the changes made during conversion, and is used as a cache.  The
7ed5e3
@@ -1186,12 +1186,12 @@ and output methods may use disk space, as outlined in the table below.
7ed5e3
 =item I<-i ova>
7ed5e3
 
7ed5e3
 This temporarily places a full copy of the uncompressed source disks
7ed5e3
-in C<$LIBGUESTFS_CACHEDIR> (or F).
7ed5e3
+in C<$VIRT_V2V_TMPDIR> (or F).
7ed5e3
 
7ed5e3
 =item I<-o glance>
7ed5e3
 
7ed5e3
 This temporarily places a full copy of the output disks in
7ed5e3
-C<$LIBGUESTFS_CACHEDIR> (or F).
7ed5e3
+C<$VIRT_V2V_TMPDIR> (or F).
7ed5e3
 
7ed5e3
 =item I<-o local>
7ed5e3
 
7ed5e3
@@ -1311,7 +1311,7 @@ have at least 100 available inodes.
7ed5e3
 =head3 Minimum free space check in the host
7ed5e3
 
7ed5e3
 You must have sufficient free space in the host directory used to
7ed5e3
-store temporary overlays.  To find out
7ed5e3
+store large temporary overlays.  To find out
7ed5e3
 which directory this is, use:
7ed5e3
 
7ed5e3
  $ df -h "`guestfish get-cachedir`"
7ed5e3
@@ -1319,9 +1319,12 @@ which directory this is, use:
7ed5e3
  /dev/mapper/root   50G   40G  6.8G  86% /
7ed5e3
 
7ed5e3
 and look under the C<Avail> column.  Virt-v2v will refuse to do the
7ed5e3
-conversion at all unless at least 1GB is available there.
7ed5e3
+conversion at all unless at least 1GB is available there.  You can
7ed5e3
+change the directory that virt-v2v uses by setting
7ed5e3
+C<$VIRT_V2V_TMPDIR>.
7ed5e3
 
7ed5e3
-See also L</Resource requirements> above.
7ed5e3
+See also L</Resource requirements> above and L</ENVIRONMENT VARIABLES>
7ed5e3
+below.
7ed5e3
 
7ed5e3
 =head2 Running virt-v2v as root or non-root
7ed5e3
 
7ed5e3
@@ -1496,10 +1499,18 @@ conversion.
7ed5e3
 
7ed5e3
 =over 4
7ed5e3
 
7ed5e3
+=item C<VIRT_V2V_TMPDIR>
7ed5e3
+
7ed5e3
 =item C<LIBGUESTFS_CACHEDIR>
7ed5e3
 
7ed5e3
 Location of the temporary directory used for the potentially large
7ed5e3
-temporary overlay file.  If not set, F is used.
7ed5e3
+temporary overlay file.  If neither environment variable is set then
7ed5e3
+F is used.
7ed5e3
+
7ed5e3
+To reliably ensure large temporary files are cleaned up (for example
7ed5e3
+in case virt-v2v crashes) you should create a randomly named directory
7ed5e3
+under F, set C<VIRT_V2V_TMPDIR> to point to this directory,
7ed5e3
+then when virt-v2v exits remove the directory.
7ed5e3
 
7ed5e3
 See the L</Disk space> section above.
7ed5e3
 
7ed5e3
diff --git a/v2v/input_ova.ml b/v2v/input_ova.ml
7ed5e3
index 5d3bece1..d78a5ce8 100644
7ed5e3
--- a/v2v/input_ova.ml
7ed5e3
+++ b/v2v/input_ova.ml
7ed5e3
@@ -132,8 +132,8 @@ class input_ova ova = object
7ed5e3
            (* The spec allows the file to be gzip-compressed, in
7ed5e3
             * which case we must uncompress it into a temporary.
7ed5e3
             *)
7ed5e3
-           let temp_dir = (open_guestfs ())#get_cachedir () in
7ed5e3
-           let new_filename = Filename.temp_file ~temp_dir "ova" ".vmdk" in
7ed5e3
+           let new_filename =
7ed5e3
+             Filename.temp_file ~temp_dir:Utils.large_tmpdir "ova" ".vmdk" in
7ed5e3
            unlink_on_exit new_filename;
7ed5e3
            let cmd =
7ed5e3
              sprintf "zcat %s > %s" (quote filename) (quote new_filename) in
7ed5e3
diff --git a/v2v/input_vmx.ml b/v2v/input_vmx.ml
7ed5e3
index f1d143e9..7a7647e5 100644
7ed5e3
--- a/v2v/input_vmx.ml
7ed5e3
+++ b/v2v/input_vmx.ml
7ed5e3
@@ -389,8 +389,7 @@ and find_nics vmx =
7ed5e3
 
7ed5e3
 class input_vmx input_password input_transport arg =
7ed5e3
   let tmpdir =
7ed5e3
-    let base_dir = (open_guestfs ())#get_cachedir () in
7ed5e3
-    let t = Mkdtemp.temp_dir ~base_dir "vmx." in
7ed5e3
+    let t = Mkdtemp.temp_dir "vmx." in
7ed5e3
     rmdir_on_exit t;
7ed5e3
     t in
7ed5e3
 object
7ed5e3
diff --git a/v2v/output_glance.ml b/v2v/output_glance.ml
7ed5e3
index 0a9e9181..e8facd0a 100644
7ed5e3
--- a/v2v/output_glance.ml
7ed5e3
+++ b/v2v/output_glance.ml
7ed5e3
@@ -33,8 +33,7 @@ class output_glance () =
7ed5e3
    * to write to a temporary file.  XXX
7ed5e3
    *)
7ed5e3
   let tmpdir =
7ed5e3
-    let base_dir = (open_guestfs ())#get_cachedir () in
7ed5e3
-    let t = Mkdtemp.temp_dir ~base_dir "glance." in
7ed5e3
+    let t = Mkdtemp.temp_dir ~base_dir:large_tmpdir "glance." in
7ed5e3
     rmdir_on_exit t;
7ed5e3
     t in
7ed5e3
 object
7ed5e3
diff --git a/v2v/output_null.ml b/v2v/output_null.ml
7ed5e3
index 3528da50..edb749ea 100644
7ed5e3
--- a/v2v/output_null.ml
7ed5e3
+++ b/v2v/output_null.ml
7ed5e3
@@ -75,8 +75,7 @@ class output_null =
7ed5e3
    * the null-co device w/ a JSON URL.
7ed5e3
    *)
7ed5e3
   let tmpdir =
7ed5e3
-    let base_dir = (open_guestfs ())#get_cachedir () in
7ed5e3
-    let t = Mkdtemp.temp_dir ~base_dir "null." in
7ed5e3
+    let t = Mkdtemp.temp_dir ~base_dir:large_tmpdir "null." in
7ed5e3
     rmdir_on_exit t;
7ed5e3
     t in
7ed5e3
 object
7ed5e3
diff --git a/v2v/output_rhv_upload.ml b/v2v/output_rhv_upload.ml
7ed5e3
index 81896e53..913992d9 100644
7ed5e3
--- a/v2v/output_rhv_upload.ml
7ed5e3
+++ b/v2v/output_rhv_upload.ml
7ed5e3
@@ -148,25 +148,28 @@ class output_rhv_upload output_alloc output_conn
7ed5e3
                         rhv_options =
7ed5e3
   (* Create a temporary directory which will be deleted on exit. *)
7ed5e3
   let tmpdir =
7ed5e3
-    let base_dir = (open_guestfs ())#get_cachedir () in
7ed5e3
-    let t = Mkdtemp.temp_dir ~base_dir "rhvupload." in
7ed5e3
+    let t = Mkdtemp.temp_dir "rhvupload." in
7ed5e3
     rmdir_on_exit t;
7ed5e3
     t in
7ed5e3
 
7ed5e3
   let diskid_file_of_id id = tmpdir // sprintf "diskid.%d" id in
7ed5e3
 
7ed5e3
   (* Create Python scripts for precheck, vmcheck, plugin and create VM. *)
7ed5e3
-  let py_create = Python_script.create ~tmpdir in
7ed5e3
-  let precheck_script = py_create ~name:"rhv-upload-precheck.py"
7ed5e3
-                        Output_rhv_upload_precheck_source.code in
7ed5e3
-  let vmcheck_script = py_create ~name:"rhv-upload-vmcheck.py"
7ed5e3
-                       Output_rhv_upload_vmcheck_source.code in
7ed5e3
-  let plugin_script = py_create ~name:"rhv-upload-plugin.py"
7ed5e3
-                      Output_rhv_upload_plugin_source.code in
7ed5e3
-  let createvm_script = py_create ~name:"rhv-upload-createvm.py"
7ed5e3
-                        Output_rhv_upload_createvm_source.code in
7ed5e3
-  let deletedisks_script = py_create ~name:"rhv-upload-deletedisks.py"
7ed5e3
-                           Output_rhv_upload_deletedisks_source.code in
7ed5e3
+  let precheck_script =
7ed5e3
+    Python_script.create ~name:"rhv-upload-precheck.py"
7ed5e3
+      Output_rhv_upload_precheck_source.code in
7ed5e3
+  let vmcheck_script =
7ed5e3
+    Python_script.create ~name:"rhv-upload-vmcheck.py"
7ed5e3
+      Output_rhv_upload_vmcheck_source.code in
7ed5e3
+  let plugin_script =
7ed5e3
+    Python_script.create ~name:"rhv-upload-plugin.py"
7ed5e3
+      Output_rhv_upload_plugin_source.code in
7ed5e3
+  let createvm_script =
7ed5e3
+    Python_script.create ~name:"rhv-upload-createvm.py"
7ed5e3
+      Output_rhv_upload_createvm_source.code in
7ed5e3
+  let deletedisks_script =
7ed5e3
+    Python_script.create ~name:"rhv-upload-deletedisks.py"
7ed5e3
+      Output_rhv_upload_deletedisks_source.code in
7ed5e3
 
7ed5e3
   (* JSON parameters which are invariant between disks. *)
7ed5e3
   let json_params = [
7ed5e3
diff --git a/v2v/parse_ova.ml b/v2v/parse_ova.ml
7ed5e3
index 0b939ac4..568ac5fa 100644
7ed5e3
--- a/v2v/parse_ova.ml
7ed5e3
+++ b/v2v/parse_ova.ml
7ed5e3
@@ -71,8 +71,7 @@ let rec parse_ova ova =
7ed5e3
     if is_directory ova then ova, Directory
7ed5e3
     else (
7ed5e3
       let tmpdir =
7ed5e3
-        let base_dir = (open_guestfs ())#get_cachedir () in
7ed5e3
-        let t = Mkdtemp.temp_dir ~base_dir "ova." in
7ed5e3
+        let t = Mkdtemp.temp_dir ~base_dir:large_tmpdir "ova." in
7ed5e3
         rmdir_on_exit t;
7ed5e3
         t in
7ed5e3
 
7ed5e3
@@ -221,8 +220,7 @@ and uncompress_head format file =
7ed5e3
  *)
7ed5e3
 and uncompressed_type format file =
7ed5e3
   let head, headlen = uncompress_head format file in
7ed5e3
-  let tmpfile, chan =
7ed5e3
-    Filename.open_temp_file "ova.file." "" in
7ed5e3
+  let tmpfile, chan = Filename.open_temp_file "ova.file." "" in
7ed5e3
   output chan head 0 headlen;
7ed5e3
   close_out chan;
7ed5e3
   let ret = detect_file_type tmpfile in
7ed5e3
diff --git a/v2v/python_script.ml b/v2v/python_script.ml
7ed5e3
index b1ea8f9d..212c8e1b 100644
7ed5e3
--- a/v2v/python_script.ml
7ed5e3
+++ b/v2v/python_script.ml
7ed5e3
@@ -31,15 +31,9 @@ type script = {
7ed5e3
   path : string;                (* Path to script. *)
7ed5e3
 }
7ed5e3
 
7ed5e3
-let create ?(name = "script.py") ?tmpdir code =
7ed5e3
-  let tmpdir =
7ed5e3
-    match tmpdir with
7ed5e3
-    | None ->
7ed5e3
-      let base_dir = (open_guestfs ())#get_cachedir () in
7ed5e3
-      let t = Mkdtemp.temp_dir ~base_dir "v2v." in
7ed5e3
-      rmdir_on_exit t;
7ed5e3
-      t
7ed5e3
-    | Some dir -> dir in
7ed5e3
+let create ?(name = "script.py") code =
7ed5e3
+  let tmpdir = Mkdtemp.temp_dir "v2v." in
7ed5e3
+  rmdir_on_exit tmpdir;
7ed5e3
   let path = tmpdir // name in
7ed5e3
   with_open_out path (fun chan -> output_string chan code);
7ed5e3
   { tmpdir; path }
7ed5e3
diff --git a/v2v/python_script.mli b/v2v/python_script.mli
7ed5e3
index 6bf77e34..fdf73514 100644
7ed5e3
--- a/v2v/python_script.mli
7ed5e3
+++ b/v2v/python_script.mli
7ed5e3
@@ -20,14 +20,11 @@
7ed5e3
 
7ed5e3
 type script
7ed5e3
 
7ed5e3
-val create : ?name:string -> ?tmpdir:string -> string -> script
7ed5e3
+val create : ?name:string -> string -> script
7ed5e3
 (** Create a Python script object.
7ed5e3
 
7ed5e3
     The optional parameter [?name] is a hint for the name of the script.
7ed5e3
 
7ed5e3
-    The optional parameter [?tmpdir] is the temporary directory to use
7ed5e3
-    (instead of creating a new one).
7ed5e3
-
7ed5e3
     The parameter is the Python code.  Usually this is
7ed5e3
     [Some_source.code] where [some_source.ml] is generated from
7ed5e3
     the Python file by [v2v/embed.sh] (see also [v2v/Makefile.am]). *)
7ed5e3
diff --git a/v2v/utils.ml b/v2v/utils.ml
7ed5e3
index c2940582..a6c359f0 100644
7ed5e3
--- a/v2v/utils.ml
7ed5e3
+++ b/v2v/utils.ml
7ed5e3
@@ -24,6 +24,10 @@ open Std_utils
7ed5e3
 open Tools_utils
7ed5e3
 open Common_gettext.Gettext
7ed5e3
 
7ed5e3
+let large_tmpdir =
7ed5e3
+  try Sys.getenv "VIRT_V2V_TMPDIR"
7ed5e3
+  with Not_found -> (open_guestfs ())#get_cachedir ()
7ed5e3
+
7ed5e3
 (* Is SELinux enabled and enforcing on the host? *)
7ed5e3
 let have_selinux =
7ed5e3
   0 = Sys.command "getenforce 2>/dev/null | grep -isq Enforcing"
7ed5e3
@@ -114,6 +118,7 @@ let qemu_img_supports_offset_and_size () =
7ed5e3
    * file that has an offset and size.
7ed5e3
    *)
7ed5e3
   let tmp = Filename.temp_file "v2vqemuimgtst" ".img" in
7ed5e3
+  unlink_on_exit tmp;
7ed5e3
   Unix.truncate tmp 1024;
7ed5e3
 
7ed5e3
   let json = [
7ed5e3
@@ -133,7 +138,6 @@ let qemu_img_supports_offset_and_size () =
7ed5e3
             (if verbose () then "" else " 2>&1") in
7ed5e3
   debug "%s" cmd;
7ed5e3
   let r = 0 = Sys.command cmd in
7ed5e3
-  Unix.unlink tmp;
7ed5e3
   debug "qemu-img supports \"offset\" and \"size\" in json URLs: %b" r;
7ed5e3
   r
7ed5e3
 
7ed5e3
diff --git a/v2v/utils.mli b/v2v/utils.mli
7ed5e3
index 937e2b9b..d86ca507 100644
7ed5e3
--- a/v2v/utils.mli
7ed5e3
+++ b/v2v/utils.mli
7ed5e3
@@ -18,6 +18,11 @@
7ed5e3
 
7ed5e3
 (** Utilities used in virt-v2v only. *)
7ed5e3
 
7ed5e3
+val large_tmpdir : string
7ed5e3
+(** [VIRT_V2V_TMPDIR] or [/var/tmp].  Create all large temporary files
7ed5e3
+    such as overlays in this directory.  Small temporary files can
7ed5e3
+    use the default behaviour eg. of {!Filename.temp_file} *)
7ed5e3
+
7ed5e3
 val have_selinux : bool
7ed5e3
 (** True if SELinux is enabled and enforcing on the host. *)
7ed5e3
 
7ed5e3
diff --git a/v2v/v2v.ml b/v2v/v2v.ml
7ed5e3
index 73edff2c..a58ff433 100644
7ed5e3
--- a/v2v/v2v.ml
7ed5e3
+++ b/v2v/v2v.ml
7ed5e3
@@ -264,8 +264,6 @@ and set_source_networks_and_bridges cmdline source =
7ed5e3
   let nics = List.map (Networks.map cmdline.network_map) source.s_nics in
7ed5e3
   { source with s_nics = nics }
7ed5e3
 
7ed5e3
-and overlay_dir = (open_guestfs ())#get_cachedir ()
7ed5e3
-
7ed5e3
 (* Conversion can fail or hang if there is insufficient free space in
7ed5e3
  * the temporary directory used to store overlays on the host
7ed5e3
  * (RHBZ#1316479).  Although only a few hundred MB is actually
7ed5e3
@@ -273,12 +271,12 @@ and overlay_dir = (open_guestfs ())#get_cachedir ()
7ed5e3
  * guestfs appliance which is also stored here.
7ed5e3
  *)
7ed5e3
 and check_host_free_space () =
7ed5e3
-  let free_space = StatVFS.free_space (StatVFS.statvfs overlay_dir) in
7ed5e3
-  debug "check_host_free_space: overlay_dir=%s free_space=%Ld"
7ed5e3
-        overlay_dir free_space;
7ed5e3
+  let free_space = StatVFS.free_space (StatVFS.statvfs large_tmpdir) in
7ed5e3
+  debug "check_host_free_space: large_tmpdir=%s free_space=%Ld"
7ed5e3
+        large_tmpdir free_space;
7ed5e3
   if free_space < 1_073_741_824L then
7ed5e3
     error (f_"insufficient free space in the conversion server temporary directory %s (%s).\n\nEither free up space in that directory, or set the LIBGUESTFS_CACHEDIR environment variable to point to another directory with more than 1GB of free space.\n\nSee also the virt-v2v(1) manual, section \"Minimum free space check in the host\".")
7ed5e3
-          overlay_dir (human_size free_space)
7ed5e3
+          large_tmpdir (human_size free_space)
7ed5e3
 
7ed5e3
 (* Create a qcow2 v3 overlay to protect the source image(s). *)
7ed5e3
 and create_overlays source_disks =
7ed5e3
@@ -286,7 +284,7 @@ and create_overlays source_disks =
7ed5e3
   List.mapi (
7ed5e3
     fun i ({ s_qemu_uri = qemu_uri; s_format = format } as source) ->
7ed5e3
       let overlay_file =
7ed5e3
-        Filename.temp_file ~temp_dir:overlay_dir "v2vovl" ".qcow2" in
7ed5e3
+        Filename.temp_file ~temp_dir:large_tmpdir "v2vovl" ".qcow2" in
7ed5e3
       unlink_on_exit overlay_file;
7ed5e3
 
7ed5e3
       (* There is a specific reason to use the newer qcow2 variant:
7ed5e3
@@ -823,7 +821,7 @@ and preserve_overlays overlays src_name =
7ed5e3
   List.iter (
7ed5e3
     fun ov ->
7ed5e3
       let saved_filename =
7ed5e3
-        sprintf "%s/%s-%s.qcow2" overlay_dir src_name ov.ov_sd in
7ed5e3
+        sprintf "%s/%s-%s.qcow2" large_tmpdir src_name ov.ov_sd in
7ed5e3
       rename ov.ov_overlay_file saved_filename;
7ed5e3
       info (f_"Overlay saved as %s [--debug-overlays]") saved_filename
7ed5e3
   ) overlays
7ed5e3
-- 
e21fe6
2.31.1
7ed5e3