From 93eb72e70c87bda26197351d20240dcb592b08de Mon Sep 17 00:00:00 2001 From: Andrii Sultanov Date: Thu, 1 May 2025 10:24:41 +0100 Subject: [PATCH 1/3] {export,import}_raw_vdi: add qcow as supported format This patch allows to pass "qcow2" as a supported format when calling VDI export and import. Qcow_tool_wrapper is added as a helper that calls the Python script for export (conversion from raw to qcow2 stream) and the OCaml Qcow_stream library for import (conversion from qcow2 stream to raw). Signed-off-by: Guillaume Signed-off-by: Andrii Sultanov --- dune-project | 1 + ocaml/xapi/dune | 1 + ocaml/xapi/export_raw_vdi.ml | 20 +++++++----- ocaml/xapi/import_raw_vdi.ml | 27 ++++++++++++++++ ocaml/xapi/importexport.ml | 16 ++++++++-- ocaml/xapi/qcow_tool_wrapper.ml | 56 +++++++++++++++++++++++++++++++++ opam/xapi.opam | 1 + 7 files changed, 113 insertions(+), 9 deletions(-) create mode 100644 ocaml/xapi/qcow_tool_wrapper.ml diff --git a/dune-project b/dune-project index 002f1c481f1..94e9700e1aa 100644 --- a/dune-project +++ b/dune-project @@ -402,6 +402,7 @@ (= :version))) ; the public library is only used for testing integers ipaddr + qcow-stream logs magic-mime mirage-crypto diff --git a/ocaml/xapi/dune b/ocaml/xapi/dune index 88213955afc..76d31dbb56f 100644 --- a/ocaml/xapi/dune +++ b/ocaml/xapi/dune @@ -164,6 +164,7 @@ psq ptime ptime.clock.os + qcow-stream rpclib.core rpclib.json rpclib.xml diff --git a/ocaml/xapi/export_raw_vdi.ml b/ocaml/xapi/export_raw_vdi.ml index cea32fb5533..4a54283cc2b 100644 --- a/ocaml/xapi/export_raw_vdi.ml +++ b/ocaml/xapi/export_raw_vdi.ml @@ -47,12 +47,18 @@ let localhost_handler rpc session_id vdi (req : Http.Request.t) let copy base_path path size = try debug "Copying VDI contents..." ; - Vhd_tool_wrapper.send ?relative_to:base_path - (Vhd_tool_wrapper.update_task_progress __context) - "none" - (Importexport.Format.to_string format) - s path size "" ; - debug "Copying VDI complete." + match format with + | Qcow -> + Qcow_tool_wrapper.send + (Qcow_tool_wrapper.update_task_progress __context) + s path size + | Vhd | Tar | Raw -> + Vhd_tool_wrapper.send ?relative_to:base_path + (Vhd_tool_wrapper.update_task_progress __context) + "none" + (Importexport.Format.to_string format) + s path size "" ; + debug "Copying VDI complete." with Unix.Unix_error (Unix.EIO, _, _) -> raise (Api_errors.Server_error @@ -73,7 +79,7 @@ let localhost_handler rpc session_id vdi (req : Http.Request.t) in Http_svr.headers s headers ; match format with - | Raw | Vhd -> + | Raw | Vhd | Qcow -> let size = Db.VDI.get_virtual_size ~__context ~self:vdi in if format = Vhd && size > Constants.max_vhd_size then raise diff --git a/ocaml/xapi/import_raw_vdi.ml b/ocaml/xapi/import_raw_vdi.ml index 565c29e7d8e..8eacfe0a786 100644 --- a/ocaml/xapi/import_raw_vdi.ml +++ b/ocaml/xapi/import_raw_vdi.ml @@ -106,6 +106,10 @@ let localhost_handler rpc session_id vdi_opt (req : Request.t) ) ) | None -> + (* FIXME: Currently, when importing an image with a virtual + size that's bigger than the VDI's virtual size, we fail in + an unhelpful manner on some write. + We could instead parse the header first and fail early. *) let vdi = match ( vdi_opt @@ -122,6 +126,22 @@ let localhost_handler rpc session_id vdi_opt (req : Request.t) ~virtual_size:length ~_type:`user ~sharable:false ~read_only:false ~other_config:[] ~xenstore_data:[] ~sm_config:[] ~tags:[] + | None, Importexport.Format.Qcow, _, _ -> + error + "Importing a QCOW2 directly into an SR not yet \ + supported" ; + raise + (HandleError + ( Api_errors.Server_error + ( Api_errors.internal_error + , [ + "Importing a QCOW2 directly into an SR not \ + yet supported" + ] + ) + , Http.http_400_badrequest ~version:"1.0" () + ) + ) | None, Importexport.Format.Vhd, _, _ -> error "Importing a VHD directly into an SR not yet supported" ; @@ -158,6 +178,13 @@ let localhost_handler rpc session_id vdi_opt (req : Request.t) in Http_svr.headers s headers ; ( match format with + | Qcow -> + Sm_fs_ops.with_block_attached_device __context rpc + session_id vdi `RW (fun path -> + Qcow_tool_wrapper.receive + (Qcow_tool_wrapper.update_task_progress __context) + s path + ) | Raw | Vhd -> let prezeroed = not diff --git a/ocaml/xapi/importexport.ml b/ocaml/xapi/importexport.ml index a210bda04d6..6ba6769b7ef 100644 --- a/ocaml/xapi/importexport.ml +++ b/ocaml/xapi/importexport.ml @@ -430,9 +430,17 @@ let sr_of_req ~__context (req : Http.Request.t) = None module Format = struct - type t = Raw | Vhd | Tar + type t = Raw | Vhd | Tar | Qcow - let to_string = function Raw -> "raw" | Vhd -> "vhd" | Tar -> "tar" + let to_string = function + | Raw -> + "raw" + | Vhd -> + "vhd" + | Tar -> + "tar" + | Qcow -> + "qcow2" let of_string x = match String.lowercase_ascii x with @@ -442,6 +450,8 @@ module Format = struct Some Vhd | "tar" -> Some Tar + | "qcow2" -> + Some Qcow | _ -> None @@ -457,6 +467,8 @@ module Format = struct "application/vhd" | Tar -> "application/x-tar" + | Qcow -> + "application/x-qemu-disk" let _key = "format" diff --git a/ocaml/xapi/qcow_tool_wrapper.ml b/ocaml/xapi/qcow_tool_wrapper.ml new file mode 100644 index 00000000000..d22c35b6a02 --- /dev/null +++ b/ocaml/xapi/qcow_tool_wrapper.ml @@ -0,0 +1,56 @@ +(* + * Copyright (C) 2025 Vates. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + *) + +module D = Debug.Make (struct let name = "qcow_tool_wrapper" end) + +open D + +let run_qcow_tool (_progress_cb : int -> unit) (args : string list) + (ufd : Unix.file_descr) = + let qcow_tool = !Xapi_globs.qcow_to_stdout in + info "Executing %s %s" qcow_tool (String.concat " " args) ; + let open Forkhelpers in + match + with_logfile_fd "qcow-tool" (fun log_fd -> + let pid = + safe_close_and_exec None (Some ufd) (Some log_fd) [] qcow_tool args + in + let _, status = waitpid pid in + if status <> Unix.WEXITED 0 then ( + error "qcow-tool failed, returning VDI_IO_ERROR" ; + raise + (Api_errors.Server_error + (Api_errors.vdi_io_error, ["Device I/O errors"]) + ) + ) + ) + with + | Success (out, _) -> + debug "qcow-tool successful export (%s)" out + | Failure (out, _e) -> + error "qcow-tool output: %s" out ; + raise (Api_errors.Server_error (Api_errors.vdi_io_error, [out])) + +let update_task_progress (__context : Context.t) (x : int) = + TaskHelper.set_progress ~__context (float_of_int x /. 100.) + +let receive (progress_cb : int -> unit) (unix_fd : Unix.file_descr) + (path : string) = + debug "Calling Qcow_stream.stream_decode (output_path = '%s')" path ; + Qcow_stream.stream_decode ~progress_cb unix_fd path + +let send (progress_cb : int -> unit) (unix_fd : Unix.file_descr) (path : string) + (_size : Int64.t) = + let args = [path] in + run_qcow_tool progress_cb args unix_fd diff --git a/opam/xapi.opam b/opam/xapi.opam index 06380ac4f8a..e61306f1573 100644 --- a/opam/xapi.opam +++ b/opam/xapi.opam @@ -34,6 +34,7 @@ depends: [ "http-lib" {with-test & = version} "integers" "ipaddr" + "qcow-stream" "logs" "magic-mime" "mirage-crypto" From aebce6565348e32e8b707bcccac7adb44ba0893c Mon Sep 17 00:00:00 2001 From: Andrii Sultanov Date: Wed, 7 May 2025 13:26:15 +0100 Subject: [PATCH 2/3] tests/unix_select: Add Lwt_engine__fun to the list of expected functions calling select With Qcow_stream being called directly from Qcow_tool_wrapper, Lwt runtime is now directly called from xapi. Signed-off-by: Andrii Sultanov --- ocaml/tests/unix_select.gawk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocaml/tests/unix_select.gawk b/ocaml/tests/unix_select.gawk index 2decd404495..1496eba53cb 100644 --- a/ocaml/tests/unix_select.gawk +++ b/ocaml/tests/unix_select.gawk @@ -35,7 +35,7 @@ END { for (idx in CALLS[SYM]) { caller = CALLS[SYM][idx]; print "--" - if (caller ~ /caml(Thread|Unix__fun_).*/) { + if (caller ~ /caml(Thread|Unix__fun_|Lwt_engine__fun_).*/) { # direct calls from these functions to unix_select are expected print caller "[expected]" } else { From 3ad070bb2a6626e14dcd76fcee74e8c3e1115dda Mon Sep 17 00:00:00 2001 From: Andrii Sultanov Date: Thu, 15 May 2025 14:17:18 +0100 Subject: [PATCH 3/3] export_raw_vdi: Add support for differential QCOW2 export with base Also add an mli file for qcow_tool_wrapper Signed-off-by: Andrii Sultanov --- ocaml/xapi/export_raw_vdi.ml | 2 +- ocaml/xapi/qcow_tool_wrapper.ml | 8 +++++--- ocaml/xapi/qcow_tool_wrapper.mli | 25 +++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 ocaml/xapi/qcow_tool_wrapper.mli diff --git a/ocaml/xapi/export_raw_vdi.ml b/ocaml/xapi/export_raw_vdi.ml index 4a54283cc2b..df3d778d579 100644 --- a/ocaml/xapi/export_raw_vdi.ml +++ b/ocaml/xapi/export_raw_vdi.ml @@ -49,7 +49,7 @@ let localhost_handler rpc session_id vdi (req : Http.Request.t) debug "Copying VDI contents..." ; match format with | Qcow -> - Qcow_tool_wrapper.send + Qcow_tool_wrapper.send ?relative_to:base_path (Qcow_tool_wrapper.update_task_progress __context) s path size | Vhd | Tar | Raw -> diff --git a/ocaml/xapi/qcow_tool_wrapper.ml b/ocaml/xapi/qcow_tool_wrapper.ml index d22c35b6a02..9c152898d25 100644 --- a/ocaml/xapi/qcow_tool_wrapper.ml +++ b/ocaml/xapi/qcow_tool_wrapper.ml @@ -50,7 +50,9 @@ let receive (progress_cb : int -> unit) (unix_fd : Unix.file_descr) debug "Calling Qcow_stream.stream_decode (output_path = '%s')" path ; Qcow_stream.stream_decode ~progress_cb unix_fd path -let send (progress_cb : int -> unit) (unix_fd : Unix.file_descr) (path : string) - (_size : Int64.t) = - let args = [path] in +let send ?relative_to (progress_cb : int -> unit) (unix_fd : Unix.file_descr) + (path : string) (_size : Int64.t) = + let args = + [path] @ match relative_to with None -> [] | Some vdi -> ["--diff"; vdi] + in run_qcow_tool progress_cb args unix_fd diff --git a/ocaml/xapi/qcow_tool_wrapper.mli b/ocaml/xapi/qcow_tool_wrapper.mli new file mode 100644 index 00000000000..51c3c626567 --- /dev/null +++ b/ocaml/xapi/qcow_tool_wrapper.mli @@ -0,0 +1,25 @@ +(* + * Copyright (C) 2025 Vates. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + *) + +val update_task_progress : Context.t -> int -> unit + +val receive : (int -> unit) -> Unix.file_descr -> string -> unit + +val send : + ?relative_to:string + -> (int -> unit) + -> Unix.file_descr + -> string + -> int64 + -> unit