From fc9cb5a1f389febe09c784f76661925b666d2b09 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Tue, 7 Feb 2023 15:06:15 +0000 Subject: [PATCH 1/3] CA-374989: add default values for removed fields This means that if at some point some records are fetched that do not contain these fields, they can be used as default instead of failing to load the records. Signed-off-by: Pau Ruiz Safont --- ocaml/idl/datamodel.ml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/ocaml/idl/datamodel.ml b/ocaml/idl/datamodel.ml index 4cd6494d528..99eabe4321d 100644 --- a/ocaml/idl/datamodel.ml +++ b/ocaml/idl/datamodel.ml @@ -459,16 +459,16 @@ end let iobandwidth = let msg = "Disabled and replaced by RRDs" in [ - field ~persist:false ~qualifier:DynamicRO ~ty:Float "read_kbs" - "Read bandwidth (KiB/s)" + field ~persist:false ~qualifier:DynamicRO ~ty:Float + ~default_value:(Some (VFloat 0.)) "read_kbs" "Read bandwidth (KiB/s)" ~lifecycle: [ (Published, rel_rio, "") ; (Deprecated, rel_tampa, "Dummy transition") ; (Removed, rel_tampa, msg) ] - ; field ~persist:false ~qualifier:DynamicRO ~ty:Float "write_kbs" - "Write bandwidth (KiB/s)" + ; field ~persist:false ~qualifier:DynamicRO ~ty:Float + ~default_value:(Some (VFloat 0.)) "write_kbs" "Write bandwidth (KiB/s)" ~lifecycle: [ (Published, rel_rio, "") @@ -922,7 +922,7 @@ module Host_metrics = struct [ field ~qualifier:DynamicRO "total" "Total host memory (bytes)" ~doc_tags:[Memory] - ; field "free" "Free host memory (bytes)" + ; field "free" "Free host memory (bytes)" ~default_value:(Some (VInt 0L)) ~lifecycle: [ (Published, rel_rio, "") @@ -2670,6 +2670,7 @@ module VIF = struct @ [namespace ~name:"qos" ~contents:(qos "VIF") ()] @ [ field ~qualifier:DynamicRO ~ty:(Ref _vif_metrics) + ~default_value:(Some (VRef null_ref)) ~lifecycle: [ (Published, rel_rio, "") @@ -4697,6 +4698,7 @@ module VBD = struct @ [namespace ~name:"qos" ~contents:(qos "VBD") ()] @ [ field ~qualifier:DynamicRO ~ty:(Ref _vbd_metrics) + ~default_value:(Some (VRef null_ref)) ~lifecycle: [ (Published, rel_rio, "") @@ -4731,6 +4733,7 @@ module VBD_metrics = struct uid _vbd_metrics ; namespace ~name:"io" ~contents:iobandwidth () ; field ~qualifier:DynamicRO ~ty:DateTime + ~default_value:(Some (VDateTime Date.never)) ~lifecycle: [ (Published, rel_rio, "") @@ -5064,6 +5067,7 @@ module VM_metrics = struct ~ty:(Map (Int, Float)) ~persist:false "utilisation" "Utilisation for all of guest's current VCPUs" + ~default_value:(Some (VMap [])) ~lifecycle: [ (Published, rel_rio, "") @@ -5175,6 +5179,7 @@ module VM_guest_metrics = struct "Logically equivalent to PV_drivers_detected" ; field ~qualifier:DynamicRO ~ty:(Map (String, String)) + ~default_value:(Some (VMap [])) ~lifecycle: [ (Published, rel_rio, "free/used/total") @@ -5189,6 +5194,7 @@ module VM_guest_metrics = struct memory_internal_free RRD data-sources instead." ; field ~qualifier:DynamicRO ~ty:(Map (String, String)) + ~default_value:(Some (VMap [])) ~lifecycle: [ (Published, rel_rio, "Disk configuration/free space") From 9608b74224f59c8e6c2b59a039a40dc609618a1c Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Tue, 7 Feb 2023 14:11:42 +0000 Subject: [PATCH 2/3] CA-374989: Revert "CP-40357: Purge all removed fields from the database and clients" This change made cross-pool migrations from earlier releases to the newest release to break. This was due to the removal of the fields in the record objects. In particular the metrics field in the VDI records, which are being retrieved to later collect the name of it's network's bridge. This reverts commit bd8b6508c9f1210d704e3c7f2160df2d6c5d689a. Signed-off-by: Pau Ruiz Safont --- ocaml/idl/datamodel_schema.ml | 3 -- ocaml/idl/datamodel_utils.ml | 22 +++--------- ocaml/idl/json_backend/gen_json.ml | 2 +- ocaml/idl/markdown_backend.ml | 2 +- ocaml/idl/ocaml_backend/gen_api.ml | 6 ++-- ocaml/idl/ocaml_backend/gen_client.ml | 2 +- ocaml/idl/ocaml_backend/gen_db_actions.ml | 13 +++---- ocaml/idl/ocaml_backend/gen_db_check.ml | 5 ++- ocaml/idl/ocaml_backend/gen_test.ml | 2 +- ocaml/sdk-gen/csharp/gen_csharp_binding.ml | 3 +- ocaml/sdk-gen/powershell/common_functions.ml | 2 +- .../powershell/gen_powershell_binding.ml | 8 ++--- ocaml/tests/common/test_common.ml | 36 ++++++++++--------- ocaml/xapi-cli-server/cli_operations.ml | 15 ++++---- ocaml/xapi-cli-server/records.ml | 35 ++++++++++++++++++ ocaml/xapi/create_misc.ml | 28 +++++++++------ ocaml/xapi/db_gc_util.ml | 8 ++++- ocaml/xapi/dbsync.ml | 4 +-- ocaml/xapi/dbsync_master.ml | 4 +-- ocaml/xapi/eventgen.ml | 2 +- ocaml/xapi/export.ml | 2 ++ ocaml/xapi/import.ml | 2 ++ ocaml/xapi/monitor_dbcalls_cache.ml | 7 +++- ocaml/xapi/monitor_master.ml | 1 + ocaml/xapi/monitor_mem_host.ml | 17 +++++---- ocaml/xapi/vm_evacuation.ml | 10 +++--- ocaml/xapi/xapi_guest_agent.ml | 4 +-- ocaml/xapi/xapi_host.ml | 3 +- ocaml/xapi/xapi_pif.ml | 4 +-- ocaml/xapi/xapi_vbd.ml | 7 +++- ocaml/xapi/xapi_vbd_helpers.ml | 12 ++++++- ocaml/xapi/xapi_vif_helpers.ml | 21 +++++++++-- ocaml/xapi/xapi_vm.ml | 27 +++++++------- ocaml/xapi/xapi_vm.mli | 1 + ocaml/xapi/xapi_vm_clone.ml | 6 ++-- ocaml/xapi/xapi_vm_helpers.ml | 25 +++++++++++-- 36 files changed, 225 insertions(+), 126 deletions(-) diff --git a/ocaml/idl/datamodel_schema.ml b/ocaml/idl/datamodel_schema.ml index fa17239f329..132d109cb1e 100644 --- a/ocaml/idl/datamodel_schema.ml +++ b/ocaml/idl/datamodel_schema.ml @@ -20,9 +20,6 @@ let of_datamodel () = match fs with | [] -> acc - | Datamodel_types.(Field {lifecycle= {state; _}; _}) :: fs - when state = Removed_s -> - flatten_fields fs acc | Datamodel_types.Field f :: fs -> flatten_fields fs (f :: acc) | Datamodel_types.Namespace (_, internal_fs) :: fs -> diff --git a/ocaml/idl/datamodel_utils.ml b/ocaml/idl/datamodel_utils.ml index 651a7677f69..6245f927ba5 100644 --- a/ocaml/idl/datamodel_utils.ml +++ b/ocaml/idl/datamodel_utils.ml @@ -152,7 +152,7 @@ module Relations = struct end (** Compute a flat list of fields from a datamodel object *) -let all_fields_of_obj (x : obj) : field list = +let fields_of_obj (x : obj) : field list = let rec of_contents = function | Namespace (_, xs) -> List.concat (List.map of_contents xs) @@ -161,26 +161,14 @@ let all_fields_of_obj (x : obj) : field list = in List.concat (List.map of_contents x.contents) -(* Like [fields_of_obj], but without Removed fields*) -let active_fields_of_obj x = - let rec of_contents = function - | Namespace (_, xs) -> - List.concat (List.map of_contents xs) - | Field x when x.lifecycle.state <> Removed_s -> - [x] - | _ -> - [] - in - List.concat (List.map of_contents x.contents) - (* True if an object has a label (and therefore should have a get_by_name_label message *) let obj_has_get_by_name_label x = - let all_fields = all_fields_of_obj x in + let all_fields = fields_of_obj x in List.filter (fun fld -> fld.full_name = ["name"; "label"]) all_fields <> [] (* True if an object has tags (and therefore should have a get_tags message *) let obj_has_get_tags x = - let all_fields = all_fields_of_obj x in + let all_fields = fields_of_obj x in List.filter (fun fld -> fld.full_name = ["tags"]) all_fields <> [] (** XXX: unfortunately we don't mark which parameters of a message refer to the self; @@ -198,7 +186,7 @@ let find_self_parameter (msg : message) = ) let plural name = - if String.ends_with ~suffix:"metrics" name then + if Xstringext.String.endswith "metrics" name then name ^ " instances" else name ^ "s" @@ -493,7 +481,7 @@ let all_new_messages_of_field obj fld = implicit ones. NB this list requires filtering before being used for (eg) a client *) let messages_of_obj (x : obj) document_order : message list = - let all_fields = all_fields_of_obj x in + let all_fields = fields_of_obj x in let self = self_of_obj x in (* Generate appropriate get/set/add/remove messages for each field. diff --git a/ocaml/idl/json_backend/gen_json.ml b/ocaml/idl/json_backend/gen_json.ml index 29212893b73..569fd18a3d8 100644 --- a/ocaml/idl/json_backend/gen_json.ml +++ b/ocaml/idl/json_backend/gen_json.ml @@ -240,7 +240,7 @@ end = struct let ctor_fields = List.filter (function {qualifier= StaticRO | RW; _} -> true | _ -> false) - (all_fields_of_obj obj) + (fields_of_obj obj) |> List.map (fun f -> String.concat "_" f.full_name ^ if f.default_value = None then "*" else "" diff --git a/ocaml/idl/markdown_backend.ml b/ocaml/idl/markdown_backend.ml index b5635c6d7f4..5dc93a0963b 100644 --- a/ocaml/idl/markdown_backend.ml +++ b/ocaml/idl/markdown_backend.ml @@ -284,7 +284,7 @@ let print_field_table_of_obj printer ~is_class_deprecated ~is_class_removed x = ) in x - |> Datamodel_utils.all_fields_of_obj + |> Datamodel_utils.fields_of_obj |> List.sort (fun x y -> compare_case_ins (Datamodel_utils.wire_name_of_field x) diff --git a/ocaml/idl/ocaml_backend/gen_api.ml b/ocaml/idl/ocaml_backend/gen_api.ml index 1581850f953..01c49bdbe88 100644 --- a/ocaml/idl/ocaml_backend/gen_api.ml +++ b/ocaml/idl/ocaml_backend/gen_api.ml @@ -120,8 +120,7 @@ let gen_record_type ~with_module highapi tys = | DT.Record record :: t -> let obj_name = OU.ocaml_of_record_name record in let all_fields = - DU.active_fields_of_obj - (Dm_api.get_obj_by_name highapi ~objname:record) + DU.fields_of_obj (Dm_api.get_obj_by_name highapi ~objname:record) in let field fld = OU.ocaml_of_record_field (obj_name :: fld.DT.full_name) @@ -297,8 +296,7 @@ let toposort_types highapi types = true | DT.Record record -> let all_fields = - DU.active_fields_of_obj - (Dm_api.get_obj_by_name highapi ~objname:record) + DU.fields_of_obj (Dm_api.get_obj_by_name highapi ~objname:record) in List.exists (fun fld -> references name fld.DT.ty) all_fields | DT.Option ty -> diff --git a/ocaml/idl/ocaml_backend/gen_client.ml b/ocaml/idl/ocaml_backend/gen_client.ml index 4271a0a1661..651c703fd38 100644 --- a/ocaml/idl/ocaml_backend/gen_client.ml +++ b/ocaml/idl/ocaml_backend/gen_client.ml @@ -89,7 +89,7 @@ let client_api ~sync api = let ctor_fields (obj : obj) = List.filter (function {DT.qualifier= DT.StaticRO | DT.RW; _} -> true | _ -> false) - (DU.active_fields_of_obj obj) + (DU.fields_of_obj obj) (* Compute a message parameter list from a message suitable for the client (only!) *) let args_of_message ?(expand_record = true) (obj : obj) diff --git a/ocaml/idl/ocaml_backend/gen_db_actions.ml b/ocaml/idl/ocaml_backend/gen_db_actions.ml index 7d4816e60ef..b149a86a4df 100644 --- a/ocaml/idl/ocaml_backend/gen_db_actions.ml +++ b/ocaml/idl/ocaml_backend/gen_db_actions.ml @@ -177,7 +177,7 @@ let args_of_message (obj : obj) ({msg_tag= tag; _} as msg) = if x <> obj.DT.name then failwith "args_of_message" ; (* Client constructor takes all object fields regardless of qualifier but excluding Set(Ref _) types *) - let fields = DU.active_fields_of_obj obj in + let fields = DU.fields_of_obj obj in let fields = List.filter field_in_this_table fields in List.map Client.param_of_field fields | _ -> @@ -324,9 +324,7 @@ let db_action api : O.Module.t = String.concat "\n" mk_rec in let get_record_aux_fn (obj : obj) = - let record_fields = - List.filter client_side_field (DU.active_fields_of_obj obj) - in + let record_fields = List.filter client_side_field (DU.fields_of_obj obj) in O.Let.make ~name:"get_record'" ~params: [ @@ -338,7 +336,7 @@ let db_action api : O.Module.t = () in let get_record_internal_aux_fn (obj : obj) = - let record_fields = DU.active_fields_of_obj obj in + let record_fields = DU.fields_of_obj obj in O.Let.make ~name:"get_record_internal'" ~params: [ @@ -449,9 +447,8 @@ let db_action api : O.Module.t = (Escaping.escape_obj obj.DT.name) Client._self | FromObject Make -> - let fields = - List.filter field_in_this_table (DU.active_fields_of_obj obj) - in + let fields = List.filter field_in_this_table (DU.fields_of_obj obj) in + (* let fields = db_fields_of_obj obj in *) let kvs = List.map (fun fld -> diff --git a/ocaml/idl/ocaml_backend/gen_db_check.ml b/ocaml/idl/ocaml_backend/gen_db_check.ml index 93dcaec398c..dfa877de011 100644 --- a/ocaml/idl/ocaml_backend/gen_db_check.ml +++ b/ocaml/idl/ocaml_backend/gen_db_check.ml @@ -19,6 +19,7 @@ module OU = Ocaml_utils module Client = Gen_client module DT = Datamodel_types module DU = Datamodel_utils +module DM = Datamodel open DT (* Names of the modules we're going to use to perform the db operations *) @@ -65,9 +66,7 @@ let db_check api : O.Module.t = let check_refs (obj : obj) = (* List all the fields of the object which are references AND stored in this table *) - let fields = - List.filter field_in_this_table (DU.active_fields_of_obj obj) - in + let fields = List.filter field_in_this_table (DU.fields_of_obj obj) in let fields = List.filter (function {DT.ty= Ref _; _} -> true | _ -> false) fields in diff --git a/ocaml/idl/ocaml_backend/gen_test.ml b/ocaml/idl/ocaml_backend/gen_test.ml index ab930c6b8f5..d9824961db0 100644 --- a/ocaml/idl/ocaml_backend/gen_test.ml +++ b/ocaml/idl/ocaml_backend/gen_test.ml @@ -55,7 +55,7 @@ let rec gen_test_type highapi ty = and gen_record_type highapi record = let obj_name = OU.ocaml_of_record_name record in let all_fields = - DU.active_fields_of_obj (Dm_api.get_obj_by_name highapi ~objname:record) + DU.fields_of_obj (Dm_api.get_obj_by_name highapi ~objname:record) in let field fld = OU.ocaml_of_record_field (obj_name :: fld.DT.full_name) in let map_fields fn = diff --git a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml index 62812d94e61..cf07d9622c8 100644 --- a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml +++ b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml @@ -755,8 +755,7 @@ and gen_save_changes_to_field out_chan exposed_class_name fr = and ctor_call classname = let fields = - Datamodel_utils.active_fields_of_obj - (Dm_api.get_obj_by_name api ~objname:classname) + Datamodel_utils.fields_of_obj (Dm_api.get_obj_by_name api ~objname:classname) in let fields2 = List.filter diff --git a/ocaml/sdk-gen/powershell/common_functions.ml b/ocaml/sdk-gen/powershell/common_functions.ml index c8c971aa595..d81f9f687db 100644 --- a/ocaml/sdk-gen/powershell/common_functions.ml +++ b/ocaml/sdk-gen/powershell/common_functions.ml @@ -199,7 +199,7 @@ and cut_msg_name message_name fn_type = (* True if an object has a uuid (and therefore should have a get_by_uuid message *) and has_uuid x = - let all_fields = DU.active_fields_of_obj x in + let all_fields = DU.fields_of_obj x in List.filter (fun fld -> fld.full_name = ["uuid"]) all_fields <> [] and has_name x = DU.obj_has_get_by_name_label x diff --git a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml index 795f74beb89..da89c34d043 100644 --- a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml +++ b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml @@ -497,7 +497,7 @@ and print_params_constructor message obj classname = \ #endregion\n" (qualified_class_name classname) ( if is_real_constructor message then - gen_fields (DU.active_fields_of_obj obj) + gen_fields (DU.fields_of_obj obj) else gen_constructor_params message.msg_params ) @@ -589,7 +589,7 @@ and gen_make_record obj classname = \ Record = new %s(HashTable);\n\ \ }\n" (qualified_class_name classname) - (gen_record_fields (DU.active_fields_of_obj obj)) + (gen_record_fields (DU.fields_of_obj obj)) (qualified_class_name classname) and gen_record_fields fields = @@ -642,8 +642,8 @@ and gen_make_fields message obj = \ else if (HashTable != null)\n\ \ {%s\n\ \ }" - (explode_record_fields message (DU.active_fields_of_obj obj)) - (explode_hashtable_fields message (DU.active_fields_of_obj obj)) + (explode_record_fields message (DU.fields_of_obj obj)) + (explode_hashtable_fields message (DU.fields_of_obj obj)) and explode_record_fields message fields = let print_map tl hd = diff --git a/ocaml/tests/common/test_common.ml b/ocaml/tests/common/test_common.ml index 27a87e58a1e..82f9b2aaa00 100644 --- a/ocaml/tests/common/test_common.ml +++ b/ocaml/tests/common/test_common.ml @@ -138,13 +138,14 @@ let make_vm ~__context ?(name_label = "name_label") ?(pCI_bus = "") ?(other_config = []) ?(xenstore_data = []) ?(recommendations = "") ?(ha_always_run = false) ?(ha_restart_priority = "") ?(tags = []) ?(blocked_operations = []) ?(protection_policy = Ref.null) - ?(appliance = Ref.null) ?(start_delay = 0L) ?(snapshot_schedule = Ref.null) - ?(is_vmss_snapshot = false) ?(shutdown_delay = 0L) ?(order = 0L) - ?(suspend_SR = Ref.null) ?(suspend_VDI = Ref.null) ?(version = 0L) - ?(generation_id = "0:0") ?(hardware_platform_version = 0L) - ?has_vendor_device:_ ?(has_vendor_device = false) ?(reference_label = "") - ?(domain_type = `hvm) ?(nVRAM = []) ?(last_booted_record = "") - ?(last_boot_CPU_flags = []) ?(power_state = `Halted) () = + ?(is_snapshot_from_vmpp = false) ?(appliance = Ref.null) ?(start_delay = 0L) + ?(snapshot_schedule = Ref.null) ?(is_vmss_snapshot = false) + ?(shutdown_delay = 0L) ?(order = 0L) ?(suspend_SR = Ref.null) + ?(suspend_VDI = Ref.null) ?(version = 0L) ?(generation_id = "0:0") + ?(hardware_platform_version = 0L) ?has_vendor_device:_ + ?(has_vendor_device = false) ?(reference_label = "") ?(domain_type = `hvm) + ?(nVRAM = []) ?(last_booted_record = "") ?(last_boot_CPU_flags = []) + ?(power_state = `Halted) () = Xapi_vm.create ~__context ~name_label ~name_description ~user_version ~is_a_template ~affinity ~memory_target ~memory_static_max ~memory_dynamic_max ~memory_dynamic_min ~memory_static_min ~vCPUs_params @@ -154,11 +155,11 @@ let make_vm ~__context ?(name_label = "name_label") ~pV_legacy_args ~hVM_boot_policy ~hVM_boot_params ~hVM_shadow_multiplier ~platform ~nVRAM ~pCI_bus ~other_config ~xenstore_data ~recommendations ~ha_always_run ~ha_restart_priority ~tags ~blocked_operations - ~protection_policy ~appliance ~start_delay ~shutdown_delay ~order - ~suspend_SR ~suspend_VDI ~snapshot_schedule ~is_vmss_snapshot ~version - ~generation_id ~hardware_platform_version ~has_vendor_device - ~reference_label ~domain_type ~last_booted_record ~last_boot_CPU_flags - ~power_state + ~protection_policy ~is_snapshot_from_vmpp ~appliance ~start_delay + ~shutdown_delay ~order ~suspend_SR ~suspend_VDI ~snapshot_schedule + ~is_vmss_snapshot ~version ~generation_id ~hardware_platform_version + ~has_vendor_device ~reference_label ~domain_type ~last_booted_record + ~last_boot_CPU_flags ~power_state let make_host ~__context ?(uuid = make_uuid ()) ?(name_label = "host") ?(name_description = "description") ?(hostname = "localhost") @@ -252,8 +253,8 @@ let make_vif ~__context ?(ref = Ref.make ()) ?(uuid = make_uuid ()) ?(qos_algorithm_type = "") ?(qos_algorithm_params = []) ?(qos_supported_algorithms = []) ?(currently_attached = false) ?(status_code = 0L) ?(status_detail = "") ?(runtime_properties = []) - ?(other_config = []) ?(locking_mode = `unlocked) ?(ipv4_allowed = []) - ?(ipv6_allowed = []) ?(ipv4_configuration_mode = `None) + ?(other_config = []) ?(metrics = Ref.null) ?(locking_mode = `unlocked) + ?(ipv4_allowed = []) ?(ipv6_allowed = []) ?(ipv4_configuration_mode = `None) ?(ipv4_addresses = []) ?(ipv4_gateway = "") ?(ipv6_configuration_mode = `None) ?(ipv6_addresses = []) ?(ipv6_gateway = "") () = @@ -261,7 +262,7 @@ let make_vif ~__context ?(ref = Ref.make ()) ?(uuid = make_uuid ()) ~reserved ~device ~network ~vM ~mAC ~mAC_autogenerated ~mTU ~qos_algorithm_type ~qos_algorithm_params ~qos_supported_algorithms ~currently_attached ~status_code ~status_detail ~runtime_properties - ~other_config ~locking_mode ~ipv4_allowed ~ipv6_allowed + ~other_config ~metrics ~locking_mode ~ipv4_allowed ~ipv6_allowed ~ipv4_configuration_mode ~ipv4_addresses ~ipv4_gateway ~ipv6_configuration_mode ~ipv6_addresses ~ipv6_gateway ~reserved_pci:Ref.null ; @@ -361,12 +362,13 @@ let make_vbd ~__context ?(ref = Ref.make ()) ?(uuid = make_uuid ()) ?(storage_lock = false) ?(empty = false) ?(reserved = false) ?(other_config = []) ?(currently_attached = false) ?(status_code = 0L) ?(status_detail = "") ?(runtime_properties = []) ?(qos_algorithm_type = "") - ?(qos_algorithm_params = []) ?(qos_supported_algorithms = []) () = + ?(qos_algorithm_params = []) ?(qos_supported_algorithms = []) + ?(metrics = Ref.make ()) () = Db.VBD.create ~__context ~ref ~uuid ~allowed_operations ~current_operations ~vM ~vDI ~device ~userdevice ~bootable ~mode ~_type ~unpluggable ~storage_lock ~empty ~reserved ~other_config ~currently_attached ~status_code ~status_detail ~runtime_properties ~qos_algorithm_type - ~qos_algorithm_params ~qos_supported_algorithms ; + ~qos_algorithm_params ~qos_supported_algorithms ~metrics ; ref let make_vdi ~__context ?(ref = Ref.make ()) ?(uuid = make_uuid ()) diff --git a/ocaml/xapi-cli-server/cli_operations.ml b/ocaml/xapi-cli-server/cli_operations.ml index 7f2bf6b4b28..9baf4aa9f1b 100644 --- a/ocaml/xapi-cli-server/cli_operations.ml +++ b/ocaml/xapi-cli-server/cli_operations.ml @@ -2754,15 +2754,14 @@ let vm_create printer rpc session_id params = ~hVM_boot_policy:"" ~hVM_boot_params:[] ~hVM_shadow_multiplier:1. ~platform:[] ~pCI_bus:"" ~other_config:[] ~xenstore_data:[] ~recommendations:"" ~ha_always_run:false ~ha_restart_priority:"" ~tags:[] - ~protection_policy:Ref.null ~snapshot_schedule:Ref.null - ~is_vmss_snapshot:false ~appliance:Ref.null ~start_delay:0L - ~shutdown_delay:0L ~order:0L ~suspend_SR:Ref.null ~suspend_VDI:Ref.null - ~version:0L ~generation_id:"" ~hardware_platform_version:0L - ~has_vendor_device:false ~reference_label:"" ~domain_type:`unspecified - ~nVRAM:[] ~last_booted_record:"" ~last_boot_CPU_flags:[] - ~power_state:`Halted + ~protection_policy:Ref.null ~is_snapshot_from_vmpp:false + ~snapshot_schedule:Ref.null ~is_vmss_snapshot:false ~appliance:Ref.null + ~start_delay:0L ~shutdown_delay:0L ~order:0L ~suspend_SR:Ref.null + ~suspend_VDI:Ref.null ~version:0L ~generation_id:"" + ~hardware_platform_version:0L ~has_vendor_device:false ~reference_label:"" + ~domain_type:`unspecified ~nVRAM:[] ~last_booted_record:"" + ~last_boot_CPU_flags:[] ~power_state:`Halted in - let uuid = Client.VM.get_uuid ~rpc ~session_id ~self:vm in printer (Cli_printer.PList [uuid]) diff --git a/ocaml/xapi-cli-server/records.ml b/ocaml/xapi-cli-server/records.ml index f050764f070..40073e1d344 100644 --- a/ocaml/xapi-cli-server/records.ml +++ b/ocaml/xapi-cli-server/records.ml @@ -2340,6 +2340,34 @@ let vm_record rpc session_id vm = (xgm ()) ) () + ; make_field ~name:"memory" + ~get:(fun () -> + Option.fold ~none:nid + ~some:(fun m -> + Record_util.s2sm_to_string "; " m.API.vM_guest_metrics_memory + ) + (xgm ()) + ) + ~get_map:(fun () -> + Option.fold ~none:[] + ~some:(fun m -> m.API.vM_guest_metrics_memory) + (xgm ()) + ) + () + ; make_field ~name:"disks" + ~get:(fun () -> + Option.fold ~none:nid + ~some:(fun m -> + Record_util.s2sm_to_string "; " m.API.vM_guest_metrics_disks + ) + (xgm ()) + ) + ~get_map:(fun () -> + Option.fold ~none:[] + ~some:(fun m -> m.API.vM_guest_metrics_disks) + (xgm ()) + ) + () ; make_field ~name:"VBDs" ~get:(fun () -> get_uuids_from_refs (x ()).API.vM_VBDs) ~get_set:(fun () -> List.map get_uuid_from_ref (x ()).API.vM_VBDs) @@ -3007,6 +3035,13 @@ let host_record rpc session_id host = (xm ()) ) () + ; make_field ~name:"memory-free" + ~get:(fun () -> + Option.fold ~none:nid + ~some:(fun m -> Int64.to_string m.API.host_metrics_memory_free) + (xm ()) + ) + () ; make_field ~name:"memory-free-computed" ~expensive:true ~get:(fun () -> Int64.to_string diff --git a/ocaml/xapi/create_misc.ml b/ocaml/xapi/create_misc.ml index 4325240424f..4fdce4569f5 100644 --- a/ocaml/xapi/create_misc.ml +++ b/ocaml/xapi/create_misc.ml @@ -184,6 +184,9 @@ let read_localhost_info ~__context = ; hypervisor= Option.map (fun s -> s.hypervisor) stat } +(** [mkints n] creates a list [1; 2; .. ; n] *) +let rec mkints = function 0 -> [] | n -> mkints (n - 1) @ [n] + (** Ensures that the database has all the necessary records for domain zero, and that the records are up-to-date. Includes the following: @@ -312,11 +315,11 @@ and create_domain_zero_record ~__context ~domain_zero_ref (host_info : host_info ~user_version:1L ~ha_restart_priority:"" ~ha_always_run:false ~recommendations:"" ~last_boot_CPU_flags:[] ~last_booted_record:"" ~guest_metrics:Ref.null ~metrics ~bios_strings:[] - ~protection_policy:Ref.null ~snapshot_schedule:Ref.null - ~is_vmss_snapshot:false ~appliance:Ref.null ~start_delay:0L - ~shutdown_delay:0L ~order:0L ~suspend_SR:Ref.null ~version:0L - ~generation_id:"" ~hardware_platform_version:0L ~has_vendor_device:false - ~requires_reboot:false ~reference_label:"" + ~protection_policy:Ref.null ~is_snapshot_from_vmpp:false + ~snapshot_schedule:Ref.null ~is_vmss_snapshot:false ~appliance:Ref.null + ~start_delay:0L ~shutdown_delay:0L ~order:0L ~suspend_SR:Ref.null + ~version:0L ~generation_id:"" ~hardware_platform_version:0L + ~has_vendor_device:false ~requires_reboot:false ~reference_label:"" ~domain_type:Xapi_globs.domain_zero_domain_type ~nVRAM:[] ~pending_guidances:[] ; ensure_domain_zero_metrics_record ~__context ~domain_zero_ref host_info ; @@ -367,11 +370,12 @@ and create_domain_zero_metrics_record ~__context ~domain_zero_metrics_ref ~memory_constraints ~vcpus : unit = Db.VM_metrics.create ~__context ~ref:domain_zero_metrics_ref ~uuid:(Uuidx.to_string (Uuidx.make ())) - ~memory_actual:memory_constraints.target ~vCPUs_number:(Int64.of_int vcpus) - ~vCPUs_CPU:[] ~vCPUs_params:[] ~vCPUs_flags:[] ~state:[] - ~start_time:Date.never ~install_time:Date.never ~last_updated:Date.never - ~other_config:[] ~hvm:false ~nomigrate:false ~nested_virt:false - ~current_domain_type:Xapi_globs.domain_zero_domain_type + ~memory_actual:memory_constraints.target + ~vCPUs_utilisation:(List.map (fun x -> (Int64.of_int x, 0.)) (mkints vcpus)) + ~vCPUs_number:(Int64.of_int vcpus) ~vCPUs_CPU:[] ~vCPUs_params:[] + ~vCPUs_flags:[] ~state:[] ~start_time:Date.never ~install_time:Date.never + ~last_updated:Date.never ~other_config:[] ~hvm:false ~nomigrate:false + ~nested_virt:false ~current_domain_type:Xapi_globs.domain_zero_domain_type and update_domain_zero_record ~__context ~domain_zero_ref (host_info : host_info) : unit = @@ -406,7 +410,9 @@ and update_domain_zero_metrics_record ~__context ~domain_zero_ref = let cpus' = Int64.of_int cpus in Db.VM_metrics.set_current_domain_type ~__context ~self:metrics ~value:Xapi_globs.domain_zero_domain_type ; - Db.VM_metrics.set_VCPUs_number ~__context ~self:metrics ~value:cpus' + Db.VM_metrics.set_VCPUs_number ~__context ~self:metrics ~value:cpus' ; + Db.VM_metrics.set_VCPUs_utilisation ~__context ~self:metrics + ~value:(List.map (fun x -> (Int64.of_int x, 0.)) (mkints cpus)) and create_domain_zero_memory_constraints (_ : host_info) : Vm_memory_constraints.t = diff --git a/ocaml/xapi/db_gc_util.ml b/ocaml/xapi/db_gc_util.ml index 0dca88b3654..3548aef164e 100644 --- a/ocaml/xapi/db_gc_util.ml +++ b/ocaml/xapi/db_gc_util.ml @@ -117,6 +117,8 @@ let gc_VBDs ~__context = Db.VBD.set_empty ~__context ~self ~value:true ; debug "VBD corresponds to CD. Record preserved but set to empty" ) else + let metrics = Db.VBD.get_metrics ~__context ~self in + (try Db.VBD_metrics.destroy ~__context ~self:metrics with _ -> ()) ; Db.VBD.destroy ~__context ~self ) @@ -130,7 +132,11 @@ let gc_VIFs ~__context = gc_connector ~__context Db.VIF.get_all Db.VIF.get_record (fun x -> valid_ref __context x.vIF_VM) (fun x -> valid_ref __context x.vIF_network) - (fun ~__context ~self -> Db.VIF.destroy ~__context ~self) + (fun ~__context ~self -> + let metrics = Db.VIF.get_metrics ~__context ~self in + (try Db.VIF_metrics.destroy ~__context ~self:metrics with _ -> ()) ; + Db.VIF.destroy ~__context ~self + ) let gc_PBDs ~__context = gc_connector ~__context Db.PBD.get_all Db.PBD.get_record diff --git a/ocaml/xapi/dbsync.ml b/ocaml/xapi/dbsync.ml index 36c291c3d43..0d3809fed70 100644 --- a/ocaml/xapi/dbsync.ml +++ b/ocaml/xapi/dbsync.ml @@ -41,8 +41,8 @@ let create_host_metrics ~__context = let r = Ref.make () in Db.Host_metrics.create ~__context ~ref:r ~uuid:(Uuidx.to_string (Uuidx.make ())) - ~live:false ~memory_total:0L ~last_updated:Xapi_stdext_date.Date.never - ~other_config:[] ; + ~live:false ~memory_total:0L ~memory_free:0L + ~last_updated:Xapi_stdext_date.Date.never ~other_config:[] ; Db.Host.set_metrics ~__context ~self ~value:r ) ) diff --git a/ocaml/xapi/dbsync_master.ml b/ocaml/xapi/dbsync_master.ml index 389993afbdf..14c63c8cc10 100644 --- a/ocaml/xapi/dbsync_master.ml +++ b/ocaml/xapi/dbsync_master.ml @@ -272,8 +272,8 @@ let ensure_vm_metrics_records_exist __context = let m = Ref.make () in let uuid = Uuidx.to_string (Uuidx.make ()) in Db.VM_metrics.create ~__context ~ref:m ~uuid ~vCPUs_number:0L - ~memory_actual:0L ~vCPUs_CPU:[] ~vCPUs_params:[] ~vCPUs_flags:[] - ~start_time:Xapi_stdext_date.Date.never + ~vCPUs_utilisation:[] ~memory_actual:0L ~vCPUs_CPU:[] ~vCPUs_params:[] + ~vCPUs_flags:[] ~start_time:Xapi_stdext_date.Date.never ~install_time:Xapi_stdext_date.Date.never ~state:[] ~last_updated:(Xapi_stdext_date.Date.of_float 0.) ~other_config:[] ~hvm:false ~nested_virt:false ~nomigrate:false diff --git a/ocaml/xapi/eventgen.ml b/ocaml/xapi/eventgen.ml index ecbb7ab9b05..92fe0598fbf 100644 --- a/ocaml/xapi/eventgen.ml +++ b/ocaml/xapi/eventgen.ml @@ -58,7 +58,7 @@ let compute_object_references_to_follow (obj_name : string) = | _ -> [] ) - (Datamodel_utils.active_fields_of_obj obj) + (Datamodel_utils.fields_of_obj obj) ) let obj_references_table : (string, (string * string) list) Hashtbl.t = diff --git a/ocaml/xapi/export.ml b/ocaml/xapi/export.ml index 140fdb6ddda..87562bb6704 100644 --- a/ocaml/xapi/export.ml +++ b/ocaml/xapi/export.ml @@ -293,6 +293,7 @@ let make_vif table ~preserve_power_state __context self = (if preserve_power_state then vif.API.vIF_currently_attached else false) ; API.vIF_network= lookup table (Ref.string_of vif.API.vIF_network) ; API.vIF_VM= lookup table (Ref.string_of vif.API.vIF_VM) + ; API.vIF_metrics= Ref.null ; API.vIF_current_operations= [] ; API.vIF_allowed_operations= [] } @@ -332,6 +333,7 @@ let make_vbd table ~preserve_power_state __context self = (if preserve_power_state then vbd.API.vBD_currently_attached else false) ; API.vBD_VDI= lookup table (Ref.string_of vbd.API.vBD_VDI) ; API.vBD_VM= lookup table (Ref.string_of vbd.API.vBD_VM) + ; API.vBD_metrics= Ref.null ; API.vBD_current_operations= [] ; API.vBD_allowed_operations= [] } diff --git a/ocaml/xapi/import.ml b/ocaml/xapi/import.ml index 202d7b420c3..07962315286 100644 --- a/ocaml/xapi/import.ml +++ b/ocaml/xapi/import.ml @@ -805,6 +805,8 @@ module GuestMetrics : HandlerTools = struct ~pV_drivers_version:gm_record.API.vM_guest_metrics_PV_drivers_version ~pV_drivers_up_to_date: gm_record.API.vM_guest_metrics_PV_drivers_up_to_date + ~memory:gm_record.API.vM_guest_metrics_memory + ~disks:gm_record.API.vM_guest_metrics_disks ~networks:gm_record.API.vM_guest_metrics_networks ~pV_drivers_detected:gm_record.API.vM_guest_metrics_PV_drivers_detected ~other:gm_record.API.vM_guest_metrics_other diff --git a/ocaml/xapi/monitor_dbcalls_cache.ml b/ocaml/xapi/monitor_dbcalls_cache.ml index b4ec3ed80b5..507500b20dc 100644 --- a/ocaml/xapi/monitor_dbcalls_cache.ml +++ b/ocaml/xapi/monitor_dbcalls_cache.ml @@ -40,6 +40,8 @@ let vm_memory_tmp : (string, Int64.t) Hashtbl.t = Hashtbl.create 100 (* A cache for host's free/total memory. *) let host_memory_m : Mutex.t = Mutex.create () +let host_memory_free_cached : Int64.t ref = ref Int64.zero + let host_memory_total_cached : Int64.t ref = ref Int64.zero (* A cache mapping VM uuids to PVS_proxy status. *) @@ -85,7 +87,10 @@ let clear_cache () = safe_clear ~cache:bonds_links_up_cached ~tmp:bonds_links_up_tmp ~lock:bonds_links_up_cached_m ; safe_clear ~cache:vm_memory_cached ~tmp:vm_memory_tmp ~lock:vm_memory_cached_m ; - with_lock host_memory_m (fun _ -> host_memory_total_cached := Int64.zero) + with_lock host_memory_m (fun _ -> + host_memory_free_cached := Int64.zero ; + host_memory_total_cached := Int64.zero + ) (* Helper map functions. *) let transfer_map ?(except = []) ~source ~target () = diff --git a/ocaml/xapi/monitor_master.ml b/ocaml/xapi/monitor_master.ml index 5ea2db77897..e65b4b5beca 100644 --- a/ocaml/xapi/monitor_master.ml +++ b/ocaml/xapi/monitor_master.ml @@ -192,6 +192,7 @@ let update_pifs ~__context host pifs = ~uuid:(Uuidx.to_string (Uuidx.make ())) ~carrier:false ~device_name:"" ~vendor_name:"" ~device_id:"" ~vendor_id:"" ~speed:0L ~duplex:false ~pci_bus_path:"" + ~io_read_kbs:0. ~io_write_kbs:0. ~last_updated:(Date.of_float 0.) ~other_config:[] ; Db.PIF.set_metrics ~__context ~self:pifdev ~value:ref ; ref diff --git a/ocaml/xapi/monitor_mem_host.ml b/ocaml/xapi/monitor_mem_host.ml index 00be83c8e81..afddc5d0f78 100644 --- a/ocaml/xapi/monitor_mem_host.ml +++ b/ocaml/xapi/monitor_mem_host.ml @@ -61,16 +61,20 @@ let get_changes rrd_files = rrd_files ) in + let free_bytes = List.assoc_opt "memory_free_kib" named_dss in let total_bytes = List.assoc_opt "memory_total_kib" named_dss in (* Check if anything has changed since our last reading. *) - match total_bytes with - | Some total when !Mcache.host_memory_total_cached <> total -> - Some total + match (free_bytes, total_bytes) with + | Some free, Some total + when !Mcache.host_memory_free_cached <> free + || !Mcache.host_memory_total_cached <> total -> + Some (free, total) | _ -> None -let set_changes total_bytes = +let set_changes (free_bytes, total_bytes) = Mtxext.execute Mcache.host_memory_m (fun _ -> + Mcache.host_memory_free_cached := free_bytes ; Mcache.host_memory_total_cached := total_bytes ) @@ -85,12 +89,13 @@ let update rrd_files = match changes with | None -> () - | Some total -> ( + | Some ((free, total) as c) -> ( try let host = Helpers.get_localhost ~__context in let metrics = Db.Host.get_metrics ~__context ~self:host in Db.Host_metrics.set_memory_total ~__context ~self:metrics ~value:total ; - set_changes total + Db.Host_metrics.set_memory_free ~__context ~self:metrics ~value:free ; + set_changes c with e -> error "Unable to update host memory metrics: %s" (Printexc.to_string e) ) diff --git a/ocaml/xapi/vm_evacuation.ml b/ocaml/xapi/vm_evacuation.ml index 5374cacb93d..4febd5943c0 100644 --- a/ocaml/xapi/vm_evacuation.ml +++ b/ocaml/xapi/vm_evacuation.ml @@ -3,12 +3,14 @@ module D = Debug.Make (struct let name = "vm_evacuation" end) open D let estimate_evacuate_timeout ~__context ~host = + let mref = Db.Host.get_metrics ~__context ~self:host in + let metrics = Db.Host_metrics.get_record ~__context ~self:mref in let memory_used = - Db.Host.get_resident_VMs ~__context ~self:host - |> List.map (fun self -> Db.VM.get_memory_static_max ~__context ~self) - |> List.fold_left Int64.add 0L + Int64.sub metrics.API.host_metrics_memory_total + metrics.API.host_metrics_memory_free in - (* Conservative estimation based on 1000Mbps link, ignores unused memory on each VM *) + (* Conservative estimation based on 1000Mbps link, and the memory usage of + Dom0 (which is not going to be transferred) is an intentional surplus *) let t = Int64.to_float memory_used *. 8. /. (1000. *. 1024. *. 1024.) in max 240. t diff --git a/ocaml/xapi/xapi_guest_agent.ml b/ocaml/xapi/xapi_guest_agent.ml index 7ccd6383743..4dbebfcbc7b 100644 --- a/ocaml/xapi/xapi_guest_agent.ml +++ b/ocaml/xapi/xapi_guest_agent.ml @@ -319,8 +319,8 @@ let create_and_set_guest_metrics (lookup : string -> string option) Db.VM_guest_metrics.create ~__context ~ref:new_gm_ref ~uuid:new_gm_uuid ~os_version:initial_gm.os_version ~pV_drivers_version:initial_gm.pv_drivers_version - ~pV_drivers_up_to_date:pV_drivers_detected ~networks:initial_gm.networks - ~pV_drivers_detected ~other:initial_gm.other + ~pV_drivers_up_to_date:pV_drivers_detected ~memory:[] ~disks:[] + ~networks:initial_gm.networks ~pV_drivers_detected ~other:initial_gm.other ~last_updated:(Date.of_float initial_gm.last_updated) ~other_config:[] ~live:true ~can_use_hotplug_vbd:initial_gm.can_use_hotplug_vbd diff --git a/ocaml/xapi/xapi_host.ml b/ocaml/xapi/xapi_host.ml index c83886f9d78..82ee9db89fa 100644 --- a/ocaml/xapi/xapi_host.ml +++ b/ocaml/xapi/xapi_host.ml @@ -994,7 +994,8 @@ let create ~__context ~uuid ~name_label ~name_description:_ ~hostname ~address let make_new_metrics_object ref = Db.Host_metrics.create ~__context ~ref ~uuid:(Uuidx.to_string (Uuidx.make ())) - ~live:false ~memory_total:0L ~last_updated:Date.never ~other_config:[] + ~live:false ~memory_total:0L ~memory_free:0L ~last_updated:Date.never + ~other_config:[] in let name_description = "Default install" and host = Ref.make () in let metrics = Ref.make () in diff --git a/ocaml/xapi/xapi_pif.ml b/ocaml/xapi/xapi_pif.ml index 89e523c5356..e2dcb1e984c 100644 --- a/ocaml/xapi/xapi_pif.ml +++ b/ocaml/xapi/xapi_pif.ml @@ -412,8 +412,8 @@ let make_pif_metrics ~__context = let () = Db.PIF_metrics.create ~__context ~ref:metrics ~uuid:metrics_uuid ~carrier:false ~device_name:"" ~vendor_name:"" ~device_id:"" ~vendor_id:"" - ~speed:0L ~duplex:false ~pci_bus_path:"" ~last_updated:(Date.of_float 0.) - ~other_config:[] + ~speed:0L ~duplex:false ~pci_bus_path:"" ~io_read_kbs:0. ~io_write_kbs:0. + ~last_updated:(Date.of_float 0.) ~other_config:[] in metrics diff --git a/ocaml/xapi/xapi_vbd.ml b/ocaml/xapi/xapi_vbd.ml index 4fc03348ba7..af05176d10c 100644 --- a/ocaml/xapi/xapi_vbd.ml +++ b/ocaml/xapi/xapi_vbd.ml @@ -295,6 +295,11 @@ let create ~__context ~vM ~vDI ~device ~userdevice ~bootable ~mode ~_type (* Make people aware that non-shared disks make VMs not agile *) if not empty then assert_doesnt_make_vm_non_agile ~__context ~vm:vM ~vdi:vDI ; + let metrics = Ref.make () + and metrics_uuid = Uuidx.to_string (Uuidx.make ()) in + Db.VBD_metrics.create ~__context ~ref:metrics ~uuid:metrics_uuid + ~io_read_kbs:0. ~io_write_kbs:0. ~last_updated:(Date.of_float 0.) + ~other_config:[] ; (* Enable the SM driver to specify a VBD backend kind for the VDI *) let other_config = try @@ -310,7 +315,7 @@ let create ~__context ~vM ~vDI ~device ~userdevice ~bootable ~mode ~_type ~unpluggable ~empty ~reserved:false ~qos_algorithm_type ~qos_algorithm_params ~qos_supported_algorithms:[] ~currently_attached:_currently_attached ~status_code:Int64.zero - ~status_detail:"" ~runtime_properties:[] ~other_config ; + ~status_detail:"" ~runtime_properties:[] ~other_config ~metrics ; update_allowed_operations ~__context ~self:ref ; ref ) diff --git a/ocaml/xapi/xapi_vbd_helpers.ml b/ocaml/xapi/xapi_vbd_helpers.ml index 69caf611766..d3c30bffdb9 100644 --- a/ocaml/xapi/xapi_vbd_helpers.ml +++ b/ocaml/xapi/xapi_vbd_helpers.ml @@ -424,6 +424,11 @@ let destroy ~__context ~self = ] ) ) ; + let metrics = Db.VBD.get_metrics ~__context ~self in + (* Don't let a failure to destroy the metrics stop us *) + Helpers.log_exn_continue "VBD_metrics.destroy" + (fun self -> Db.VBD_metrics.destroy ~__context ~self) + metrics ; Db.VBD.destroy ~__context ~self (** Type of a function which does the actual hotplug/ hotunplug *) @@ -434,7 +439,12 @@ let copy ~__context ?vdi ~vm vbd = let all = Db.VBD.get_record ~__context ~self:vbd in let new_vbd = Ref.make () in let vbd_uuid = Uuidx.to_string (Uuidx.make ()) in + let metrics = Ref.make () in + let metrics_uuid = Uuidx.to_string (Uuidx.make ()) in let vdi = Option.value ~default:all.API.vBD_VDI vdi in + Db.VBD_metrics.create ~__context ~ref:metrics ~uuid:metrics_uuid + ~io_read_kbs:0. ~io_write_kbs:0. ~last_updated:(Date.of_float 0.) + ~other_config:[] ; Db.VBD.create ~__context ~ref:new_vbd ~uuid:vbd_uuid ~allowed_operations:[] ~current_operations:[] ~storage_lock:false ~vM:vm ~vDI:vdi ~empty:(all.API.vBD_empty || vdi = Ref.null) @@ -445,5 +455,5 @@ let copy ~__context ?vdi ~vm vbd = ~status_detail:"" ~other_config:all.API.vBD_other_config ~qos_algorithm_type:all.API.vBD_qos_algorithm_type ~qos_algorithm_params:all.API.vBD_qos_algorithm_params - ~qos_supported_algorithms:[] ~runtime_properties:[] ; + ~qos_supported_algorithms:[] ~runtime_properties:[] ~metrics ; new_vbd diff --git a/ocaml/xapi/xapi_vif_helpers.ml b/ocaml/xapi/xapi_vif_helpers.ml index e2d103fb009..ea0346044b6 100644 --- a/ocaml/xapi/xapi_vif_helpers.ml +++ b/ocaml/xapi/xapi_vif_helpers.ml @@ -294,15 +294,22 @@ let create ~__context ~device ~network ~vM ~mAC ~mTU ~other_config if List.mem device all_devices then raise (Api_errors.Server_error (Api_errors.device_already_exists, [device])) ; + let metrics = Ref.make () + and metrics_uuid = Uuidx.to_string (Uuidx.make ()) in + Db.VIF_metrics.create ~__context ~ref:metrics ~uuid:metrics_uuid + ~io_read_kbs:0. ~io_write_kbs:0. + ~last_updated:(Xapi_stdext_date.Date.of_float 0.) + ~other_config:[] ; let (_ : unit) = Db.VIF.create ~__context ~ref ~uuid:(Uuidx.to_string uuid) ~current_operations:[] ~allowed_operations:[] ~reserved:false ~device ~network ~vM ~mAC ~mAC_autogenerated ~mTU ~qos_algorithm_type ~qos_algorithm_params ~qos_supported_algorithms:[] ~currently_attached ~status_code:0L ~status_detail:"" ~runtime_properties:[] ~other_config - ~locking_mode ~ipv4_allowed ~ipv6_allowed ~ipv4_configuration_mode - ~ipv4_addresses ~ipv4_gateway ~ipv6_configuration_mode ~ipv6_addresses - ~ipv6_gateway ~reserved_pci:Ref.null + ~metrics ~locking_mode ~ipv4_allowed ~ipv6_allowed + ~ipv4_configuration_mode ~ipv4_addresses ~ipv4_gateway + ~ipv6_configuration_mode ~ipv6_addresses ~ipv6_gateway + ~reserved_pci:Ref.null in () ) ; @@ -324,6 +331,14 @@ let destroy ~__context ~self = , ["VIF currently attached to a running VM"] ) ) ; + let metrics = Db.VIF.get_metrics ~__context ~self in + (* Don't let a failure to destroy the metrics stop us *) + Helpers.log_exn_continue "VIF_metrics.destroy" + (fun self -> + if Db.is_valid_ref __context self then + Db.VIF_metrics.destroy ~__context ~self + ) + metrics ; Db.VIF.destroy ~__context ~self (* copy a vif *) diff --git a/ocaml/xapi/xapi_vm.ml b/ocaml/xapi/xapi_vm.ml index 22e36190879..3071fbaf6d7 100644 --- a/ocaml/xapi/xapi_vm.ml +++ b/ocaml/xapi/xapi_vm.ml @@ -591,10 +591,11 @@ let create ~__context ~name_label ~name_description ~power_state ~user_version ~pV_legacy_args ~hVM_boot_policy ~hVM_boot_params ~hVM_shadow_multiplier ~platform ~pCI_bus ~other_config ~last_boot_CPU_flags ~last_booted_record ~recommendations ~xenstore_data ~ha_always_run ~ha_restart_priority ~tags - ~blocked_operations:_ ~protection_policy:_ ~snapshot_schedule:_ - ~is_vmss_snapshot:_ ~appliance ~start_delay ~shutdown_delay ~order - ~suspend_SR ~version ~generation_id ~hardware_platform_version - ~has_vendor_device ~reference_label ~domain_type ~nVRAM : API.ref_VM = + ~blocked_operations:_ ~protection_policy:_ ~is_snapshot_from_vmpp:_ + ~snapshot_schedule:_ ~is_vmss_snapshot:_ ~appliance ~start_delay + ~shutdown_delay ~order ~suspend_SR ~version ~generation_id + ~hardware_platform_version ~has_vendor_device ~reference_label ~domain_type + ~nVRAM : API.ref_VM = if has_vendor_device then Pool_features.assert_enabled ~__context ~f:Features.PCI_device_for_auto_update ; @@ -618,13 +619,14 @@ let create ~__context ~name_label ~name_description ~power_state ~user_version *) let metrics = Ref.make () and metrics_uuid = Uuidx.to_string (Uuidx.make ()) in + let vCPUs_utilisation = [(0L, 0.)] in let suspended = power_state = `Suspended in let current_domain_type = if suspended then domain_type else `unspecified in Db.VM_metrics.create ~__context ~ref:metrics ~uuid:metrics_uuid - ~memory_actual:0L ~vCPUs_number:0L ~vCPUs_CPU:[] ~vCPUs_params:[] - ~vCPUs_flags:[] ~state:[] ~start_time:Date.never ~install_time:Date.never - ~last_updated:Date.never ~other_config:[] ~hvm:false ~nested_virt:false - ~nomigrate:false ~current_domain_type ; + ~memory_actual:0L ~vCPUs_number:0L ~vCPUs_utilisation ~vCPUs_CPU:[] + ~vCPUs_params:[] ~vCPUs_flags:[] ~state:[] ~start_time:Date.never + ~install_time:Date.never ~last_updated:Date.never ~other_config:[] + ~hvm:false ~nested_virt:false ~nomigrate:false ~current_domain_type ; let domain_type = if domain_type = `unspecified then derive_domain_type ~hVM_boot_policy @@ -671,10 +673,11 @@ let create ~__context ~name_label ~name_description ~power_state ~user_version ~guest_metrics:Ref.null ~last_booted_record:_last_booted_record ~xenstore_data ~recommendations ~blobs:[] ~ha_restart_priority ~ha_always_run ~tags ~bios_strings:[] ~protection_policy:Ref.null - ~snapshot_schedule:Ref.null ~is_vmss_snapshot:false ~appliance ~start_delay - ~shutdown_delay ~order ~suspend_SR ~version ~generation_id - ~hardware_platform_version ~has_vendor_device ~requires_reboot:false - ~reference_label ~domain_type ~pending_guidances:[] ; + ~is_snapshot_from_vmpp:false ~snapshot_schedule:Ref.null + ~is_vmss_snapshot:false ~appliance ~start_delay ~shutdown_delay ~order + ~suspend_SR ~version ~generation_id ~hardware_platform_version + ~has_vendor_device ~requires_reboot:false ~reference_label ~domain_type + ~pending_guidances:[] ; Xapi_vm_lifecycle.update_allowed_operations ~__context ~self:vm_ref ; update_memory_overhead ~__context ~vm:vm_ref ; update_vm_virtual_hardware_platform_version ~__context ~vm:vm_ref ; diff --git a/ocaml/xapi/xapi_vm.mli b/ocaml/xapi/xapi_vm.mli index e7364c2e214..1c79730539e 100644 --- a/ocaml/xapi/xapi_vm.mli +++ b/ocaml/xapi/xapi_vm.mli @@ -189,6 +189,7 @@ val create : -> tags:string list -> blocked_operations:'a -> protection_policy:[`VMPP] Ref.t + -> is_snapshot_from_vmpp:bool -> snapshot_schedule:[`VMSS] Ref.t -> is_vmss_snapshot:bool -> appliance:API.ref_VM_appliance diff --git a/ocaml/xapi/xapi_vm_clone.ml b/ocaml/xapi/xapi_vm_clone.ml index 0a420482cc4..58df5c354a1 100644 --- a/ocaml/xapi/xapi_vm_clone.ml +++ b/ocaml/xapi/xapi_vm_clone.ml @@ -389,9 +389,9 @@ let copy_vm_record ?snapshot_info_record ~__context ~vm ~disk_op ~new_name ~ha_restart_priority:all.Db_actions.vM_ha_restart_priority ~ha_always_run:false ~tags:all.Db_actions.vM_tags ~bios_strings:all.Db_actions.vM_bios_strings ~protection_policy:Ref.null - ~is_vmss_snapshot ~snapshot_schedule:Ref.null ~appliance:Ref.null - ~start_delay:0L ~shutdown_delay:0L ~order:0L ~suspend_SR:Ref.null - ~version:0L ~generation_id + ~is_snapshot_from_vmpp:false ~is_vmss_snapshot ~snapshot_schedule:Ref.null + ~appliance:Ref.null ~start_delay:0L ~shutdown_delay:0L ~order:0L + ~suspend_SR:Ref.null ~version:0L ~generation_id ~hardware_platform_version:all.Db_actions.vM_hardware_platform_version ~has_vendor_device:all.Db_actions.vM_has_vendor_device ~requires_reboot:false ~reference_label:all.Db_actions.vM_reference_label diff --git a/ocaml/xapi/xapi_vm_helpers.ml b/ocaml/xapi/xapi_vm_helpers.ml index 38250bd7445..e77fedfdfdf 100644 --- a/ocaml/xapi/xapi_vm_helpers.ml +++ b/ocaml/xapi/xapi_vm_helpers.ml @@ -233,11 +233,25 @@ let destroy ~__context ~self = ) ; let vbds = Db.VM.get_VBDs ~__context ~self in List.iter - (fun vbd -> try Db.VBD.destroy ~__context ~self:vbd with _ -> ()) + (fun vbd -> + ( try + let metrics = Db.VBD.get_metrics ~__context ~self:vbd in + Db.VBD_metrics.destroy ~__context ~self:metrics + with _ -> () + ) ; + try Db.VBD.destroy ~__context ~self:vbd with _ -> () + ) vbds ; let vifs = Db.VM.get_VIFs ~__context ~self in List.iter - (fun vif -> try Db.VIF.destroy ~__context ~self:vif with _ -> ()) + (fun vif -> + ( try + let metrics = Db.VIF.get_metrics ~__context ~self:vif in + Db.VIF_metrics.destroy ~__context ~self:metrics + with _ -> () + ) ; + try Db.VIF.destroy ~__context ~self:vif with _ -> () + ) vifs ; let vgpus = Db.VM.get_VGPUs ~__context ~self in List.iter @@ -1302,6 +1316,11 @@ let copy_metrics ~__context ~vm = ~some:(fun x -> x.Db_actions.vM_metrics_VCPUs_number) m ) + ~vCPUs_utilisation: + (Option.fold ~none:[(0L, 0.)] + ~some:(fun x -> x.Db_actions.vM_metrics_VCPUs_utilisation) + m + ) ~vCPUs_CPU: (Option.fold ~none:[] ~some:(fun x -> x.Db_actions.vM_metrics_VCPUs_CPU) m) ~vCPUs_params: @@ -1365,6 +1384,8 @@ let copy_guest_metrics ~__context ~vm = ~os_version:all.API.vM_guest_metrics_os_version ~pV_drivers_version:all.API.vM_guest_metrics_PV_drivers_version ~pV_drivers_up_to_date:all.API.vM_guest_metrics_PV_drivers_up_to_date + ~memory:all.API.vM_guest_metrics_memory + ~disks:all.API.vM_guest_metrics_disks ~networks:all.API.vM_guest_metrics_networks ~pV_drivers_detected:all.API.vM_guest_metrics_PV_drivers_detected ~other:all.API.vM_guest_metrics_other From dc725558d35ea63a7870b684e64fff678ee0f4e1 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Tue, 7 Feb 2023 15:17:16 +0000 Subject: [PATCH 3/3] CA-374989: Bump datamodel version Signed-off-by: Pau Ruiz Safont --- ocaml/idl/datamodel_common.ml | 2 +- ocaml/idl/schematest.ml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ocaml/idl/datamodel_common.ml b/ocaml/idl/datamodel_common.ml index 6bee4c977f9..23274fffa22 100644 --- a/ocaml/idl/datamodel_common.ml +++ b/ocaml/idl/datamodel_common.ml @@ -10,7 +10,7 @@ open Datamodel_roles to leave a gap for potential hotfixes needing to increment the schema version.*) let schema_major_vsn = 5 -let schema_minor_vsn = 759 +let schema_minor_vsn = 760 (* Historical schema versions just in case this is useful later *) let rio_schema_major_vsn = 5 diff --git a/ocaml/idl/schematest.ml b/ocaml/idl/schematest.ml index 21efb0afca1..997f42a9926 100644 --- a/ocaml/idl/schematest.ml +++ b/ocaml/idl/schematest.ml @@ -1,7 +1,7 @@ let hash x = Digest.string x |> Digest.to_hex (* BEWARE: if this changes, check that schema has been bumped accordingly *) -let last_known_schema_hash = "24ec58d6f7fb24a2a6c48aa4973521ed" +let last_known_schema_hash = "021c461d774cbfc67ba65e5616d21f41" let current_schema_hash : string = let open Datamodel_types in