重构测试用例数据缓存 (#49)

* 添加部分文字描述到README文档 * 重构测试数据缓存并更新全局单例模式 * 更新README * 保证测试数据一致性 * 优化重复用例id详情并强制检测 * 更新运行方法命名 * 美化logo字符
wu-clan · Sep 16, 2023 · 8e4adf9 · 8e4adf9
1 parent de0d230
commit 8e4adf9
Show file tree

Hide file tree

Showing 20 changed files with 311 additions and 318 deletions.
diff --git a/README.md b/README.md
@@ -2,6 +2,10 @@
 
 基于 HTTP 请求的快速数据驱动 pytest 接口自动化测试框架
 
+我在掘金发表了关于 `HttpFpt` 的前身和由来，包括部分功能点的说明， 感兴趣
+
+的小伙伴可以一睹为快，[点击跳转](https://juejin.cn/post/7224314619867136037)
+
 ## 功能点
 
 - 多项目分级，自由切换，互不干扰
@@ -19,6 +23,41 @@
 - 自动测试结果通知，飞书，钉钉，~~企业微信~~，邮箱
 - ......
 
+## ⬇️ 下载
+
+克隆:
+
+```shell
+git clone https://github.com/wu-clan/httpfpt.git
+```
+
+## 🧑‍💻 DEV
+
+1. 安装依赖:
+
+    ```shell
+    pip install -r requirements.txt
+    ```
+
+2. 安装 redis 并启动服务
+
+   [Redis Windows](https://github.com/redis-windows/redis-windows)
+
+   [Linux / macOS](https://redis.io/download/)
+
+   [Docker](https://hub.docker.com/_/redis)
+
+3. 安装 mysql 并创建一个任意名称数据库，同步修改 conf.toml 中的数据库配置
+
+   [Windows / Linux / macOS](https://dev.mysql.com/downloads/installer/)
+
+   [Docker](https://hub.docker.com/_/mysql)
+
+> [!WARNING]
+> allure 测试报告默认使用 allure-pytest
+> 生成，但是不能直接访问，你必须安装 [allure](https://www.yuque.com/poloyy/python/aiqlmi)
+> 本地程序和 [Java JDK](https://adoptopenjdk.net/archive.html?variant=openjdk8&jvmVariant=hotspot) 才能进行可视化浏览
+
 ## 帮助
 
 有关更多详细信息，请参阅 [文档](https://wu-clan.github.io/httpfpt_docs)
diff --git a/httpfpt/common/send_request.py b/httpfpt/common/send_request.py
@@ -14,16 +14,16 @@
 from httpfpt.common.errors import SendRequestError, AssertError
 from httpfpt.common.log import log
 from httpfpt.core import get_conf
-from httpfpt.db.mysql_db import MysqlDB
+from httpfpt.db.mysql_db import mysql_client
 from httpfpt.enums.request.body import BodyType
 from httpfpt.enums.request.engin import EnginType
 from httpfpt.utils.allure_control import allure_attach_file, allure_step
-from httpfpt.utils.assert_control import Asserter
+from httpfpt.utils.assert_control import asserter
 from httpfpt.utils.enum_control import get_enum_values
 from httpfpt.utils.relate_testcase_executor import exec_setup_testcase
-from httpfpt.utils.request.hooks_executor import HookExecutor
+from httpfpt.utils.request.hooks_executor import hook_executor
 from httpfpt.utils.request.request_data_parse import RequestDataParse
-from httpfpt.utils.request.vars_extractor import VarsExtractor
+from httpfpt.utils.request.vars_extractor import var_extractor
 from httpfpt.utils.time_control import get_current_time
 
 
@@ -147,14 +147,14 @@ def send_request(
                 if setup_testcase is not None:
                     new_parsed = exec_setup_testcase(request_data_parse, setup_testcase)
                     if isinstance(new_parsed, RequestDataParse):
-                        # 对呀引用了关联测试用例变量的测试来讲, 这里可能造成微小的性能损耗
+                        # 获取最新数据，对于引用了关联测试用例变量的测试来讲, 可能造成性能损耗
                         parsed_data = request_data_parse.get_request_data_parsed
                 setup_sql = parsed_data['setup_sql']
                 if setup_sql is not None:
-                    MysqlDB().exec_case_sql(setup_sql, parsed_data['env'])
+                    mysql_client.exec_case_sql(setup_sql, parsed_data['env'])
                 setup_hooks = parsed_data['setup_hooks']
                 if setup_hooks is not None:
-                    HookExecutor().exec_hook_func(setup_hooks)
+                    hook_executor.exec_hook_func(setup_hooks)
                 wait_time = parsed_data['setup_wait_time']
                 if wait_time is not None:
                     log.info(f'执行请求前等待：{wait_time} s')
@@ -231,16 +231,16 @@ def send_request(
             try:
                 teardown_sql = parsed_data['teardown_sql']
                 if teardown_sql is not None:
-                    MysqlDB().exec_case_sql(teardown_sql, parsed_data['env'])
+                    mysql_client.exec_case_sql(teardown_sql, parsed_data['env'])
                 teardown_hooks = parsed_data['teardown_hooks']
                 if teardown_hooks is not None:
-                    HookExecutor().exec_hook_func(teardown_hooks)
+                    hook_executor.exec_hook_func(teardown_hooks)
                 teardown_extract = parsed_data['teardown_extract']
                 if teardown_extract is not None:
-                    VarsExtractor().teardown_var_extract(response_data, teardown_extract, parsed_data['env'])
+                    var_extractor.teardown_var_extract(response_data, teardown_extract, parsed_data['env'])
                 teardown_assert = parsed_data['teardown_assert']
                 if teardown_assert is not None:
-                    Asserter().exec_asserter(response_data, assert_text=teardown_assert)
+                    asserter.exec_asserter(response_data, assert_text=teardown_assert)
                 wait_time = parsed_data['teardown_wait_time']
                 if wait_time is not None:
                     log.info(f'执行请求后等待：{wait_time} s')

diff --git a/httpfpt/common/variable_cache.py b/httpfpt/common/variable_cache.py
@@ -85,3 +85,6 @@ def clear(self) -> bool:
         if result:
             log.info('清空临时变量')
         return result
+
+
+variable_cache = VariableCache()
diff --git a/httpfpt/conftest.py b/httpfpt/conftest.py
@@ -8,7 +8,7 @@
 from py._xmlgen import html
 
 from httpfpt.common.log import log
-from httpfpt.common.variable_cache import VariableCache
+from httpfpt.common.variable_cache import variable_cache
 from httpfpt.common.yaml_handler import write_yaml_report
 from httpfpt.core.get_conf import TESTER_NAME, PROJECT_NAME, TEST_REPORT_TITLE
 
@@ -30,7 +30,7 @@ def package_fixture():
     # 预留空行
     log.info('')
     # 清理临时变量
-    VariableCache().clear()
+    variable_cache.clear()
 
 
 @pytest.fixture(scope='module', autouse=True)

diff --git a/httpfpt/core/conf.toml b/httpfpt/core/conf.toml
@@ -14,7 +14,7 @@ port = 3306
 user = 'root'
 password = '123456'
 database = 'test'
-charset = 'utf-8'
+charset = 'utf8mb4'
 
 # redis 数据库
 [redis]

diff --git a/httpfpt/db/mysql_db.py b/httpfpt/db/mysql_db.py
@@ -11,7 +11,7 @@
 from httpfpt.common.env_handler import write_env_vars
 from httpfpt.common.errors import SQLSyntaxError, JsonPathFindError, VariableError
 from httpfpt.common.log import log
-from httpfpt.common.variable_cache import VariableCache
+from httpfpt.common.variable_cache import variable_cache
 from httpfpt.common.yaml_handler import write_yaml_vars
 from httpfpt.core import get_conf
 from httpfpt.core.path_conf import RUN_ENV_PATH
@@ -142,7 +142,7 @@ def exec_case_sql(self, sql: str | list, env: Optional[str] = None) -> dict | in
                     else:
                         raise JsonPathFindError(f'jsonpath 取值失败, 表达式: {json_path}')
                     if set_type == VarType.CACHE:
-                        VariableCache().set(key, value)
+                        variable_cache.set(key, value)
                     elif set_type == VarType.ENV:
                         if env is None:
                             raise ValueError('写入环境变量准备失败, 缺少参数 env, 请检查传参')
@@ -153,3 +153,6 @@ def exec_case_sql(self, sql: str | list, env: Optional[str] = None) -> dict | in
                         raise VariableError(
                             f'前置 sql 设置变量失败, 用例参数 "type: {set_type}" 值错误, 请使用 cache / env / global'  # noqa: E501
                         )
+
+
+mysql_client = MysqlDB()
diff --git a/httpfpt/db/redis_db.py b/httpfpt/db/redis_db.py
@@ -8,9 +8,9 @@
 from httpfpt.core import get_conf
 
 
-class RedisDB:
+class RedisDB(Redis):
     def __init__(self) -> None:
-        self.redis = Redis(
+        super().__init__(
             host=get_conf.REDIS_HOST,
             port=get_conf.REDIS_PORT,
             password=get_conf.REDIS_PASSWORD,
@@ -22,7 +22,7 @@ def __init__(self) -> None:
 
     def init(self) -> None:
         try:
-            self.redis.ping()
+            self.ping()
         except TimeoutError:
             log.error('数据库 redis 连接超时')
         except AuthenticationError:
@@ -39,23 +39,24 @@ def get(self, key: Any) -> Any:
         :param key:
         :return:
         """
-        data = self.redis.get(key)
-        if data:
-            log.info(f'获取 redis 数据 {key} 成功')
-        else:
+        data = super().get(key)
+        if not data:
             log.warning(f'获取 redis 数据 {key} 失败, 此数据不存在')
         return data
 
-    def set(self, key: Any, value: Any, **kwargs) -> None:
+    def get_prefix(self, prefix: str) -> list:
         """
-        设置 redis 数据
+        获取 redis 符合前缀的数据
 
-        :param key:
-        :param value:
+        :param prefix: key 前缀
         :return:
         """
-        self.redis.set(key, value, **kwargs)
-        log.info(f'设置 redis 数据 {key} 成功')
+        data = []
+        for key in self.scan_iter(match=f'{prefix}*'):
+            value = super().get(key)
+            if value:
+                data.append(value)
+        return data
 
     def rset(self, key: Any, value: Any, **kwargs) -> None:
         """
@@ -66,20 +67,18 @@ def rset(self, key: Any, value: Any, **kwargs) -> None:
         :param kwargs:
         :return:
         """
-        self.redis.delete(key)
-        self.redis.set(key, value, **kwargs)
-        log.info(f'重置 redis 数据 {key} 成功')
+        self.delete(key)
+        self.set(key, value, **kwargs)
 
-    def delete(self, *key: Any) -> None:
+    def delete_prefix(self, prefix: str) -> None:
         """
-        删除 redis 数据
+        删除 redis 符合前缀的数据
 
-        :param key:
+        :param prefix: key 前缀
         :return:
         """
-        count = self.redis.delete(*key)
-        if count > 0:
-            log.info(f'删除 redis 数据 {key} 成功')
+        for key in self.scan_iter(match=f'{prefix}*'):
+            self.delete(key)
 
     def exists(self, *key: Any) -> int:
         """
@@ -88,35 +87,10 @@ def exists(self, *key: Any) -> int:
         :param key:
         :return:
         """
-        num = self.redis.exists(*key)
-        if num:
-            log.info(f'判断 redis 数据 {key} 存在')
-        else:
-            log.warning(f'判断 redis 数据 {key} 不存在')
+        num = super().exists(*key)
+        if not num:
+            log.error(f'不存在 redis 数据 {key}')
         return num
 
-    def lpush(self, key: Any, *value: Any) -> None:
-        """
-        从左侧插入列表数据
-
-        :param key:
-        :param value:
-        :return:
-        """
-        self.redis.lpush(key, *value)
-        log.info(f'从左侧插入 redis 数据 {key} 成功')
-
-    def relpush(self, key: Any, *value: Any) -> None:
-        """
-        删除原数据并重新从左侧插入列表数据
-
-        :param key:
-        :param value:
-        :return:
-        """
-        self.redis.delete(key)
-        self.redis.rpush(key, *value)
-        log.info(f'删除原数据并重新从左侧插入 redis 数据 {key} 成功')
-
 
 redis_client = RedisDB()
diff --git a/httpfpt/run.py b/httpfpt/run.py
@@ -19,13 +19,13 @@
     YAML_REPORT_PATH,
 )
 from httpfpt.db.redis_db import redis_client
-from httpfpt.utils.relate_testcase_executor import get_all_testcase_id, get_all_testcase_data
+from httpfpt.utils.request import case_data_parse as case_data
 from httpfpt.utils.send_report.ding_talk import DingTalk
 from httpfpt.utils.send_report.lark_talk import LarkTalk
 from httpfpt.utils.send_report.send_email import SendMail
 
 
-def run(
+def startup(
     *args,
     # log level
     log_level: Literal['-q', '-s', '-v', '-vs'] = '-v',
@@ -49,7 +49,7 @@ def run(
     **kwargs,
 ) -> None:
     """
-    运行入口
+    运行启动程序
 
     :param log_level: 控制台打印输出级别, 默认"-v"
     :param case_path: 指定测试用例函数, 默认为空，如果指定，则执行指定用例，否则执行全部
@@ -177,33 +177,33 @@ def run(
     ) if allure and allure_serve else ...
 
 
-def main(*args, **kwargs) -> None:
+def run(*args, pydantic_verify: bool = True, **kwargs) -> None:
+    """
+    运行入口
+
+    :param pydantic_verify: 用例数据完整架构 pydantic 快速检测, 默认开启
+    :param args: pytest 运行参数
+    :param kwargs: pytest 运行参数
+    :return:
+    """
     try:
         logo = """\n
          /$$   /$$ /$$$$$$$$ /$$$$$$$$ /$$$$$$$  /$$$$$$$$ /$$$$$$$  /$$$$$$$$
         | $$  | $$|__  $$__/|__  $$__/| $$__  $$| $$_____/| $$__  $$|__  $$__/
         | $$  | $$   | $$      | $$   | $$  | $$| $$      | $$  | $$   | $$   
-        | $$$$$$$$   | $$      | $$   | $$$$$$$/| $$$$$   | $$$$$$$/   | $$   
-        | $$__  $$   | $$      | $$   | $$____/ | $$__/   | $$____/    | $$   
+        | $$$$$$$$   | $$      | $$   | $$$$$$$/| $$$$$$  | $$$$$$$/   | $$   
+        | $$__  $$   | $$      | $$   | $$____/ | $$___/  | $$____/    | $$   
         | $$  | $$   | $$      | $$   | $$      | $$      | $$         | $$   
         | $$  | $$   | $$      | $$   | $$      | $$      | $$         | $$   
         |__/  |__/   |__/      |__/   |__/      |__/      |__/         |__/   
     
             """
         print(logo)
         log.info(logo)
-
-        # 初始化 redis 数据库 (必选)
         redis_client.init()
-
-        # 用例数据唯一 case_id 检测（可选）
-        get_all_testcase_id(get_all_testcase_data())
-
-        # 用例数据完整架构 pydantic 快速检测（可选）
-        get_all_testcase_data(pydantic_verify=True)
-
-        # 执行程序 (必选)
-        run(*args, **kwargs)
+        case_data.case_data_init(pydantic_verify)
+        case_data.case_id_unique_verify()
+        startup(*args, **kwargs)
     except Exception as e:
         log.error(f'运行异常：{e}')
         import traceback
@@ -212,4 +212,4 @@ def main(*args, **kwargs) -> None:
 
 
 if __name__ == '__main__':
-    main()
+    run()
diff --git a/httpfpt/testcases/test_project/test_api_testcase_template.py b/httpfpt/testcases/test_project/test_api_testcase_template.py
@@ -1,19 +1,13 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-import os
-
 import allure
 import pytest
 
 from httpfpt.common.send_request import send_request
-from httpfpt.common.yaml_handler import read_yaml
-from httpfpt.core.get_conf import PROJECT_NAME
-from httpfpt.utils.request.case_data_file_parse import get_request_data
+from httpfpt.utils.request.case_data_parse import get_request_data
 from httpfpt.utils.request.ids_extract import get_ids
 
-request_data = get_request_data(
-    file_data=read_yaml(filename=os.sep.join([PROJECT_NAME, 'api_testcase_template.yaml'])), use_pydantic_verify=False
-)
+request_data = get_request_data(filename='api_testcase_template.yaml')
 allure_text = request_data[0]['config']['allure']
 request_ids = get_ids(request_data)