Fixup perfmon

.github/workflows/main.yml

@@ -194,3 +194,71 @@ jobs:
           github_token: ${{ secrets.github_token }}
           reporter: github-pr-review
           level: error
+
+      # uses: tsuyoshicho/action-mypy@v3.13.0
+      - uses: bernhardkaindl/action-mypy@36cb3a857d01c1bdaa2811893106c71580132d71
+        env:
+          MYPYPATH: scripts/examples/python:scripts/examples:scripts/plugins:scripts:.
+        with:
+          filter_mode: nofilter
+          setup_method: install
+          setup_command: pip install mypy mock
+          mypy_flags: |
+            --scripts-are-modules
+            --check-untyped-defs
+            --allow-redefinition
+            --hide-error-context
+            --ignore-missing-imports
+            --no-pretty
+            --warn-unreachable
+            --disable-error-code attr-defined
+            --disable-error-code call-overload
+            --disable-error-code import-not-found
+            --disable-error-code import-untyped
+            --disable-error-code type-arg
+            --disable-error-code var-annotated
+            --exclude ocaml/events/event_listen.py
+            --exclude ocaml/idl
+            --exclude ocaml/message-switch
+            --exclude ocaml/tests/tests/looper.py
+            --exclude ocaml/xcp-rrdd/scripts/rrdd/rrdd.py
+            --exclude scripts/examples
+            --exclude scripts/examples/python/monitor-unwanted-domains.py
+            --exclude scripts/scalability-tests/ping-master.py
+            --exclude scripts/backup-sr-metadata.py
+            --exclude scripts/restore-sr-metadata.py
+            --exclude scripts/nbd_client_manager.py
+          target: |
+            scripts/perfmon
+            scripts/hfx_filename
+            scripts/mail-alarm
+            scripts/host-display
+          github_token: ${{ secrets.github_token }}
+          reporter: github-pr-review
+
+      - uses: dciborow/action-pylint@0.1.1
+        with:
+          filter_mode: nofilter
+          glob_pattern: scripts/examples/python/XenAPIPlugin.py
+          pylint_args: |
+            --verbose
+            --disable bad-indentation
+            --disable consider-using-dict-comprehension
+            --disable consider-using-f-string
+            --disable consider-using-with
+            --disable import-error
+            --disable import-outside-toplevel
+            --disable invalid-name
+            --disable missing-final-newline
+            --disable missing-class-docstring
+            --disable missing-function-docstring
+            --disable multiple-imports
+            --disable redefined-outer-name
+            --disable trailing-whitespace
+            --max-line-length 112
+            scripts/perfmon
+            scripts/hfx_filename
+            scripts/test_perfmon.py
+            scripts/test_hfx_filename.py
+          github_token: ${{ secrets.github_token }}
+          reporter: github-pr-review

ocaml/xenopsd/scripts/swtpm-wrapper

@@ -87,7 +87,7 @@ def check_state_needs_init(fname):
         return False
 
     # Check if block device has non-zero header
-    with open(fname, "r") as file:
+    with open(fname, "rb") as file:
         hdr = file.read(8)
     int_hdr = struct.unpack("<Q", hdr)[0]
 

pyproject.toml

@@ -38,9 +38,21 @@ ensure_newline_before_comments = false
 [tool.mypy]
 # Note mypy has no config setting for PYTHONPATH, so you need to call it with:
 # PYTHONPATH="scripts/examples/python:.:scripts:scripts/plugins:scripts/examples"
+exclude = [
+    "ocaml/xapi-storage/python/examples/volume/org.xen.xapi.storage.simple-file/plugin.py",
+    "scripts/examples",
+    "scripts/examples/python/monitor-unwanted-domains.py",
+    "scripts/scalability-tests/ping-master.py",
+    "scripts/backup-sr-metadata.py",
+    "scripts/restore-sr-metadata.py",
+    "scripts/nbd_client_manager.py",
+]
 files = [
+    "scripts/perfmon",
+    "scripts/hfx_filename",
+    "scripts/mail-alarm",
+    "scripts/host-display",
     "scripts/usb_reset.py",
-    "scripts/unit_tests",
 ]
 pretty = true
 error_summary = true
@@ -70,8 +82,6 @@ discard_messages_matching = [
     "No Node.TEXT_NODE in module xml.dom.minidom, referenced from 'xml.dom.expatbuilder'"
 ]
 expected_to_fail = [
-    "scripts/hfx_filename",
-    "scripts/perfmon",
     # Need 2to3 -w <file> and maybe a few other minor updates:
     "scripts/hatests",
     "scripts/backup-sr-metadata.py",
@@ -112,13 +122,12 @@ inputs = [
     "scripts/xe-scsi-dev-map",
     "scripts/examples/python",
     "scripts/yum-plugins",
-    "scripts/*.py",
-
     # To be added later,
     # when converted to Python3-compatible syntax:
     # "ocaml/message-switch/python",
     # "ocaml/idl/ocaml_backend/python",
     # "ocaml/xapi-storage/python",
+    "ocaml/xenopsd/python",
 ]
 disable = [
     "import-error",  # xenfsimage, xcp.bootloader. xcp.cmd

scripts/hfx_filename

@@ -28,18 +28,14 @@ def rpc(session_id, request):
         headers = [
           "POST %s?session_id=%s HTTP/1.0" % (db_url, session_id),
           "Connection:close",
-          "content-length:%d" % (len(request)),
+          "content-length:%d" % (len(request.encode('utf-8'))),
           ""
         ]
-        #print "Sending HTTP request:"
         for h in headers:
-          s.send("%s\r\n" % h)
-          #print "%s\r\n" % h,
-        s.send(request)
+          s.send((h + "\r\n").encode('utf-8'))
+        s.send(request.encode('utf-8'))
 
-        result = s.recv(1024)
-        #print "Received HTTP response:"
-        #print result
+        result = s.recv(1024).decode('utf-8')
         if "200 OK" not in result:
           print("Expected an HTTP 200, got %s" % result, file=sys.stderr)
           return
@@ -55,13 +51,14 @@ def rpc(session_id, request):
         s.close()
 
 def parse_string(txt):
+    assert isinstance(txt, str)
     prefix = "<value><array><data><value>success</value><value>"
     if not txt.startswith(prefix):
-        raise "Unable to parse string response"
+        raise Exception("Unable to parse string response")
     txt = txt[len(prefix):]
     suffix = "</value></data></array></value>"
     if not txt.endswith(suffix):
-        raise "Unable to parse string response"
+        raise Exception("Unable to parse string response")
     txt = txt[:len(txt)-len(suffix)]
     return txt
 

scripts/import_file.py

@@ -0,0 +1,24 @@
+# Used for importing a non-".py" file as a module
+
+import sys
+import os
+
+def import_from_file(module_name, file_path):
+    """Import a file as a module"""
+    # Only for python3, but CI has python2 pytest check, so add this line
+    if sys.version_info.major == 2:
+        return None
+    from importlib import machinery, util
+    loader = machinery.SourceFileLoader(module_name, file_path)
+    spec = util.spec_from_loader(module_name, loader)
+    assert spec
+    assert spec.loader
+    module = util.module_from_spec(spec)
+    # Probably a good idea to add manually imported module stored in sys.modules
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+    return module
+
+def get_module(module_name, file_path):
+    testdir = os.path.dirname(__file__)
+    return import_from_file(module_name, testdir + file_path)

scripts/perfmon

@@ -37,7 +37,7 @@
 import XenAPI
 import urllib.request
 from xml import sax            # used to parse rrd_updates because this may be large and sax is more efficient
 from xml.dom import minidom    # used to parse other-config:perfmon.  Efficiency is less important than reliability here
 from xml.parsers.expat import ExpatError
 import time
 import re
@@ -278,7 +278,7 @@
             paramname = col_details.paramname
 
             # Update object_report
             obj_report.insert_value(paramname, index=0, value=v) # use index=0 as this is the earliest sample so far
 
             # Update position in row
             self.col += 1
@@ -313,7 +313,7 @@
         print_debug("Calling http://localhost/rrd_updates?%s" % paramstr)
 
         sock = urllib.request.urlopen("http://localhost/rrd_updates?%s" % paramstr)
-        xmlsource = sock.read()
+        xmlsource = sock.read().decode('utf-8')
         sock.close()
 
         # Use sax rather than minidom and save Vvvast amounts of time and memory.
@@ -345,7 +345,7 @@
 
 
 # Consolidation functions:
 supported_consolidation_functions = [ 'sum', 'average', 'max', 'get_percent_fs_usage', 'get_percent_log_fs_usage', 'get_percent_mem_usage', 'get_percent_sr_usage' ]
 
 def average(mylist):
     return sum(mylist)/float(len(mylist))
@@ -357,7 +357,7 @@
     fs_output = ' '.join(fs_output.splitlines()[1:])
     log_fs_output = ' '.join(log_fs_output.splitlines()[1:])
     # Get the percent usage only when there is a separate logs partition
     if (fs_output.split()[0] != log_fs_output.split()[0]):
         percentage = log_fs_output.split()[4]
         # remove % character and convert to float
         return float(percentage[0:-1])/100.0
@@ -380,7 +380,7 @@
         memlist = memfd.readlines()
         memfd.close()
         memdict = [ m.split(':', 1) for m in memlist ]
-        memdict = dict([(k.strip(), float(re.search('\d+', v.strip()).group(0))) for (k,v) in memdict])
+        memdict = dict([(k.strip(), float(re.search(r'\d+', v.strip()).group(0))) for (k,v) in memdict])
         # We consider the sum of res memory and swap in use as the hard demand
         # of mem usage, it is bad if this number is beyond the physical mem, as
         # in such case swapping is obligatory rather than voluntary, hence
@@ -481,8 +481,8 @@
     """
     def __init__(self):
         self.value = None
         self.timeof_last_alarm = time.time() - self.alarm_auto_inhibit_period
         self.trigger_down_counter = self.alarm_trigger_period
 
 class Variable(VariableConfig, VariableState):
     """ Variable() is used by ObjectMonitor to create one Variable object for each
@@ -510,7 +510,7 @@
         """
         t = time.time()
         delta = t - self.timeof_last_alarm
         print_debug("Time since last alarm for var %s is %d - %d = %d. Refractory period = %d." % (self.name, t, self.timeof_last_alarm, delta, self.alarm_auto_inhibit_period))
         if delta < self.alarm_auto_inhibit_period:
             return # we are in the auto inhibit period - do nothing
         self.timeof_last_alarm = t
@@ -614,7 +614,7 @@
                     append_var = False
 
             if append_var:
                 print_debug("Appending %s to list of variables for %s UUID=%s" % (var.name, self.monitortype, self.uuid))
                 self.variables.append(var)
 
         # Now delete any old variables that do not appear in the new variable_nodes
@@ -668,8 +668,8 @@
        * alarm_trigger_period: num seconds of 'bad' values before an alarm is sent (default '60')
        * alarm_auto_inhibit_period: num seconds this alarm disabled after an alarm is sent (default '3600')
        * consolidation_fn: how to combine variables from rrd_updates into one value
          (default is 'average' for 'cpu_usage', 'get_percent_fs_usage' for 'fs_usage', 'get_percent_log_fs_usage' for 'log_fs_usage', 'get_percent_mem_usage' for 'mem_usage', & 'sum' for everything else)
        * rrd_regex matches the names of variables from (xe vm-data-sources-list uuid=$vmuuid) used to compute value
          (only has defaults for "cpu_usage", "network_usage", and "disk_usage")
     """
     def __init__(self, *args):
@@ -693,13 +693,13 @@
             elif variable_name == "log_fs_usage": return "_$_DUMMY__" # match nothing
             elif variable_name == "mem_usage": return "_$_DUMMY__"    # match nothing
             elif variable_name == "memory_internal_free": return variable_name
             else:raise XmlConfigException("variable %s: no default rrd_regex - please specify one" % variable_name)
         elif config_tag == 'alarm_trigger_period':      return '60'   # 1 minute
         elif config_tag == 'alarm_auto_inhibit_period': return '3600' # 1 hour
         elif config_tag == 'alarm_trigger_level': 
             if variable_name == "fs_usage": return '0.9'              # trigger when 90% full
             elif variable_name == "log_fs_usage": return '0.9'        # trigger when 90% full
             elif variable_name == "mem_usage": return '0.95'          # tigger when mem demanded is close to phy_mem
             else:raise XmlConfigException("variable %s: no default alarm_trigger_level - please specify one" % variable_name)
         elif config_tag == 'alarm_trigger_sense':
             if variable_name == "memory_internal_free": return "low"
@@ -1067,7 +1067,8 @@
             vm_uuid_list = rrd_updates.get_uuid_list_by_objtype('vm')
 
             # Remove any monitors for VMs no longer listed in rrd_updates page
-            for uuid in vm_mon_lookup:
+            # We use .pop() inside the loop, use list(dict_var.keys()):
+            for uuid in list(vm_mon_lookup.keys()):
                 if uuid not in vm_uuid_list:
                     vm_mon_lookup.pop(uuid)
 
@@ -1103,7 +1104,8 @@
             print_debug("sr_uuid_list = %s" % sr_uuid_list)
 
             # Remove monitors for SRs no longer listed in the rrd_updates page
-            for uuid in sr_mon_lookup:
+            # We use .pop() inside the loop, use list(dict_var.keys()):
+            for uuid in list(sr_mon_lookup.keys()):
                 if uuid not in sr_uuid_list:
                     sr_mon_lookup.pop(uuid)
             # Create monitors for SRs that have just appeared in rrd_updates page
@@ -1246,7 +1248,7 @@
             log_err(errmsg)
         except Exception as ignored:
             try:
                 errmsg = "\n".join([ str(x) for x in e.details ])
                 # print the exception args nicely
                 log_err(errmsg)
             except Exception as ignored:

scripts/test_hfx_filename.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# unittest for hfx_filename
+
+import unittest
+from mock import MagicMock, patch, call
+import sys
+from import_file import get_module
+
+
+# mock modules to avoid dependencies
+sys.modules["XenAPI"] = MagicMock()
+
+hfx_filename = get_module("hfx_filename", "/hfx_filename")
+
+@unittest.skipIf(sys.version_info < (3, 0), reason="requires python3")
+@patch("socket.socket")
+class TestRpc(unittest.TestCase):
+
+    def test_rpc(self, mock_socket):
+        mock_connected_socket = MagicMock()
+        mock_socket.return_value = mock_connected_socket
+
+        recv_data = b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHelloWorld"
+        mock_connected_socket.recv.return_value = recv_data
+
+        session_id = 0
+        request = "socket request"
+        body = hfx_filename.rpc(session_id, request)
+
+        # Assert that the socket methods were called as expected
+        expected_data = [
+            b"POST /remote_db_access?session_id=0 HTTP/1.0\r\n",
+            b"Connection:close\r\n",
+            b"content-length:14\r\n",
+            b"\r\n",
+            b"socket request"
+        ]
+        mock_connected_socket.send.assert_has_calls([call(data) for data in expected_data])
+
+        expected_return = "HelloWorld"
+        self.assertEqual(expected_return, body)
+
+    def test_rpc_international_character(self, mock_socket):
+        mock_connected_socket = MagicMock()
+        mock_socket.return_value = mock_connected_socket
+
+        recv_data = b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHelloWorld"
+        mock_connected_socket.recv.return_value = recv_data
+
+        session_id = 0
+        # Use international character"socket 请求" as request
+        # Not using literal string is for passing python2 check
+        request = "socket 请求"
+        body = hfx_filename.rpc(session_id, request)
+
+        # Assert that the socket methods were called as expected
+        expected_data = [
+            b"POST /remote_db_access?session_id=0 HTTP/1.0\r\n",
+            b"Connection:close\r\n",
+            b"content-length:13\r\n",
+            b"\r\n",
+            request.encode('utf-8')
+        ]
+        mock_connected_socket.send.assert_has_calls([call(data) for data in expected_data])
+
+        expected_return = "HelloWorld"
+        self.assertEqual(expected_return, body)
+
+    def test_db_get_uuid(self, mock_socket):
+        mock_connected_socket = MagicMock()
+        mock_socket.return_value = mock_connected_socket
+
+        header = "HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n"
+        body = "<value><array><data><value>success</value><value>HelloWorld</value></data></array></value>"
+        recv_data = (header + body).encode('utf-8')
+        mock_connected_socket.recv.return_value = recv_data
+
+        expected_response = "HelloWorld"
+        response = hfx_filename.db_get_by_uuid(0, "pool_patch", "22345")
+        self.assertEqual(expected_response, response)
+
+    def test_read_field(self, mock_socket):
+        mock_connected_socket = MagicMock()
+        mock_socket.return_value = mock_connected_socket
+
+        header = "HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n"
+        body = "<value><array><data><value>success</value><value>file_name</value></data></array></value>"
+        recv_data = (header + body).encode('utf-8')
+        mock_connected_socket.recv.return_value = recv_data
+
+        expected_filename = "file_name"
+        filename = hfx_filename.read_field(0, "pool_patch", "filename", "rf")
+        self.assertEqual(expected_filename, filename)
+
+
+@unittest.skipIf(sys.version_info < (3, 0), reason="requires python3")
+class TestParse(unittest.TestCase):
+
+    def test_parse_string(self): 
+        txt = "<value><array><data><value>success</value><value>abcde</value></data></array></value>" 
+        expected_txt = "abcde"
+        return_txt = hfx_filename.parse_string(txt)
+        self.assertEqual(expected_txt, return_txt)
+
+

scripts/test_perfmon.py

@@ -5,35 +5,13 @@
 import unittest
 from mock import MagicMock, patch
 import sys
-import os
-import subprocess
 import math
+from import_file import get_module
 
 # mock modules to avoid dependencies
 sys.modules["XenAPI"] = MagicMock()
 
-def import_from_file(module_name, file_path):
-    """Import a file as a module"""
-    if sys.version_info.major == 2:
-        return None
-    else:
-        from importlib import machinery, util
-        loader = machinery.SourceFileLoader(module_name, file_path)
-        spec = util.spec_from_loader(module_name, loader)
-        assert spec
-        assert spec.loader
-        module = util.module_from_spec(spec)
-        # Probably a good idea to add manually imported module stored in sys.modules
-        sys.modules[module_name] = module
-        spec.loader.exec_module(module)
-        return module
-
-def get_module():
-    """Import the perfmon script as a module for executing unit tests on functions"""
-    testdir = os.path.dirname(__file__)
-    return import_from_file("perfmon", testdir + "/perfmon")
-
-perfmon = get_module()
+perfmon = get_module("perfmon", "/perfmon")
 @unittest.skipIf(sys.version_info < (3, 0), reason="requires python3")
 @patch("subprocess.getoutput")
 class TestGetPercentage(unittest.TestCase):