diff --git a/scripts/test/func_template/template__init__.py b/scripts/test/func_template/template__init__.py new file mode 100644 index 0000000000..f30c492687 --- /dev/null +++ b/scripts/test/func_template/template__init__.py @@ -0,0 +1,8 @@ +# coding=utf-8 +# ------------------------------------------------------------------------- +# +# Part of the CodeChecker project, under the Apache License v2.0 with +# LLVM Exceptions. See LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ------------------------------------------------------------------------- diff --git a/scripts/test/func_template/template_test.py b/scripts/test/func_template/template_test.py new file mode 100644 index 0000000000..0019432d0f --- /dev/null +++ b/scripts/test/func_template/template_test.py @@ -0,0 +1,169 @@ +# +# ------------------------------------------------------------------------- +# +# Part of the CodeChecker project, under the Apache License v2.0 with +# LLVM Exceptions. See LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ------------------------------------------------------------------------- + +""" $TEST_NAME$ function test. + +WARNING!!! +This is a generated test skeleton remove the parts not required by the tests. +WARNING!!! + +""" + + +import os +import shutil +import unittest +import uuid + +from libtest import codechecker +from libtest import env +from libtest import project + + +class TestSkeleton(unittest.TestCase): + + _ccClient = None + + def setup_class(self): + """Setup the environment for testing $TEST_NAME$.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('$TEST_NAME$') + + # Set the TEST_WORKSPACE used by the tests. + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + # Get the clang version which is used for testing. + # Important because different clang releases might + # find different errors. + clang_version = env.clang_to_test() + + test_config = {} + + test_project = 'cpp' + + project_info = project.get_info(test_project) + + # Copy the test project to the workspace. The tests should + # work only on this test project. + test_proj_path = os.path.join(TEST_WORKSPACE, "test_proj") + shutil.copytree(project.path(test_project), test_proj_path) + + project_info['project_path'] = test_proj_path + + # Generate a unique name for this test run. + test_project_name = project_info['name'] + '_' + uuid.uuid4().hex + + test_config['test_project'] = project_info + + # Suppress file should be set here if needed by the tests. + suppress_file = None + + # Skip list file should be set here if needed by the tests. + skip_list_file = None + + # Get an environment which should be used by the tests. + test_env = env.test_env(TEST_WORKSPACE) + + # Create a basic CodeChecker config for the tests, this should + # be imported by the tests and they should only depend on these + # configuration options. + codechecker_cfg = { + 'suppress_file': suppress_file, + 'skip_list_file': skip_list_file, + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'analyzers': ['clangsa', 'clang-tidy'] + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = '$TEST_NAME$' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Clean the test project, if needed by the tests. + ret = project.clean(test_project) + if ret: + sys.exit(ret) + + # Check the test project, if needed by the tests. + ret = codechecker.check_and_store(codechecker_cfg, + test_project_name, + project.path(test_project)) + if ret: + sys.exit(1) + print("Analyzing the test project was successful.") + + # Save the run names in the configuration. + codechecker_cfg['run_names'] = [test_project_name] + + test_config['codechecker_cfg'] = codechecker_cfg + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, test_config) + + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): + """ + WARNING!!! + This is an example how to get the configurations needed by the tests. + WARNING!!! + """ + + # TEST_WORKSPACE is automatically set by test package __init__.py . + test_workspace = os.environ['TEST_WORKSPACE'] + + test_class = self.__class__.__name__ + print('Running ' + test_class + ' tests in ' + test_workspace) + + # Get the test configuration from the prepared int the test workspace. + test_cfg = env.import_test_cfg(test_workspace) + + # Get the test project configuration from the prepared test workspace. + self._testproject_data = env.setup_test_proj_cfg(test_workspace) + self.assertIsNotNone(self._testproject_data) + + # Setup a viewer client to test viewer API calls. + self._cc_client = env.setup_viewer_client(test_workspace) + self.assertIsNotNone(self._cc_client) + + # Get the CodeChecker cmd if needed for the tests. + self._codechecker_cmd = env.codechecker_cmd() + + # Get the run names which belong to this test. + run_names = env.get_run_names(test_workspace) + + runs = self._cc_client.getRunData(None, None, 0, None) + test_runs = [run for run in runs if run.name in run_names] + + def test_skel(self): + """ + Test some feature. + """ + pass diff --git a/web/.noserc b/web/.noserc deleted file mode 100644 index 512f4e1a08..0000000000 --- a/web/.noserc +++ /dev/null @@ -1,13 +0,0 @@ -[nosetests] - -# increase verbosity level -verbosity=3 - -# more detailed error messages on failed asserts -detailed-errors=1 - -# stop running tests on first error -stop=1 - -# do not capture stdout -#nocapture=1 diff --git a/web/client/tests/Makefile b/web/client/tests/Makefile index 2fea93ec1f..9fceab69b8 100644 --- a/web/client/tests/Makefile +++ b/web/client/tests/Makefile @@ -1,5 +1,5 @@ -CLIENT_UNIT_TEST_CMD = $(REPO_ROOT) BUILD_DIR=$(BUILD_DIR) nosetests $(NOSECFG) -w client tests/unit -CLIENT_UNIT_TEST_COV_CMD = $(REPO_ROOT) BUILD_DIR=$(BUILD_DIR) coverage run -m nose $(NOSECFG) -w client tests/unit && coverage report && coverage html +CLIENT_UNIT_TEST_CMD = $(REPO_ROOT) BUILD_DIR=$(BUILD_DIR) pytest $(PYTESTCFG) client client/tests/unit +CLIENT_UNIT_TEST_COV_CMD = $(REPO_ROOT) BUILD_DIR=$(BUILD_DIR) coverage run -m pytest $(PYTESTCFG) client client/tests/unit && coverage report && coverage html test_unit_client: diff --git a/web/client/tests/unit/__init__.py b/web/client/tests/unit/__init__.py index 5ded218171..4d5f1c0d08 100644 --- a/web/client/tests/unit/__init__.py +++ b/web/client/tests/unit/__init__.py @@ -9,15 +9,15 @@ Setup python modules for the unit tests. """ + import os import sys -# Add the generated thrift files for the unit tests. -BUILD_DIR = os.path.abspath(os.environ['BUILD_DIR']) - REPO_ROOT = os.path.abspath(os.environ['REPO_ROOT']) PKG_ROOT = os.path.join(REPO_ROOT, 'build', 'CodeChecker') +os.environ["CC_DATA_FILES_DIR"] = PKG_ROOT + sys.path.append(REPO_ROOT) -sys.path.append(os.path.join(REPO_ROOT, 'web')) sys.path.append(os.path.join(REPO_ROOT, 'tools', 'report-converter')) +sys.path.append(os.path.join(PKG_ROOT, 'lib', 'python3')) diff --git a/web/pytest.ini b/web/pytest.ini new file mode 100644 index 0000000000..c69c091b12 --- /dev/null +++ b/web/pytest.ini @@ -0,0 +1,13 @@ +[pytest] + +addopts = +# increase verbosity level + --verbose + +# stop running tests on first error +# FIXME: Pretty please comment this horrible setting out, I hate it with a +# passion +# --maxfail=1 + +# do not capture stdout + --capture=fd diff --git a/web/requirements_py/dev/requirements.txt b/web/requirements_py/dev/requirements.txt index 5290e08e7d..2764687553 100644 --- a/web/requirements_py/dev/requirements.txt +++ b/web/requirements_py/dev/requirements.txt @@ -7,7 +7,7 @@ pg8000==1.15.2 psutil==5.8.0 portalocker==2.2.1 pylint==2.8.2 -nose==1.3.7 +pytest==7.3.1 mkdocs==1.2.3 mypy_extensions==0.4.3 coverage==5.5.0 diff --git a/web/server/tests/Makefile b/web/server/tests/Makefile index c57ccff424..e048aa400f 100644 --- a/web/server/tests/Makefile +++ b/web/server/tests/Makefile @@ -1,5 +1,5 @@ -SERVER_UNIT_TEST_CMD = $(REPO_ROOT) BUILD_DIR=$(BUILD_DIR) nosetests $(NOSECFG) -w server tests/unit -SERVER_UNIT_TEST_COV_CMD = $(REPO_ROOT) BUILD_DIR=$(BUILD_DIR) coverage run -m nose $(NOSECFG) -w server tests/unit && coverage report && coverage html +SERVER_UNIT_TEST_CMD = $(REPO_ROOT) BUILD_DIR=$(BUILD_DIR) pytest $(PYTESTCFG) server server/tests/unit +SERVER_UNIT_TEST_COV_CMD = $(REPO_ROOT) BUILD_DIR=$(BUILD_DIR) coverage run -m pytest $(PYTESTCFG) server server/tests/unit && coverage report && coverage html test_unit_server: $(SERVER_UNIT_TEST_CMD) diff --git a/web/tests/Makefile b/web/tests/Makefile index 54e9e27ba9..968b73909c 100644 --- a/web/tests/Makefile +++ b/web/tests/Makefile @@ -20,8 +20,8 @@ WOKSPACE_GLOBAL_SIMPLE_SERVER = $(CC_TEST_WORKSPACE_ROOT)/global_simple_server CLEAR_WORKSPACE_CMD = rm -rf $(CC_TEST_WORKSPACE_ROOT) -# Nose test runner configuration options. -NOSECFG = --config .noserc +# pytest test runner configuration options. +PYTESTCFG = -c pytest.ini test: pycodestyle pylint test_unit test_functional @@ -75,7 +75,7 @@ SHUTDOWN_GLOBAL_SERVERS_CMD = \ EXIT_ERROR = { ${SHUTDOWN_GLOBAL_SERVERS_CMD}; exit 1; } FUNCTIONAL_TEST_CMD = $(REPO_ROOT) BUILD_DIR=$(BUILD_DIR) $(CLANG_VERSION) $(TEST_PROJECT) \ - nosetests $(NOSECFG) tests/functional \ + pytest $(PYTESTCFG) tests/functional \ && { ${SHUTDOWN_GLOBAL_SERVERS_CMD}; } || ${EXIT_ERROR} MAKE_DB_CMD = bash -c 'psql -h localhost \ @@ -88,7 +88,7 @@ DROP_DB_CMD = bash -c 'psql -h localhost \ RUN_TEST_CMD = $(CLEAR_WORKSPACE_CMD) && \ $(REPO_ROOT) BUILD_DIR=$(BUILD_DIR) $(CLANG_VERSION) $(TEST_PROJECT) \ - nosetests $(NOSECFG) $(ROOT)/web/${TEST} \ + pytest $(PYTESTCFG) $(ROOT)/web/${TEST} \ && { ${SHUTDOWN_GLOBAL_SERVERS_CMD}; } || ${EXIT_ERROR} run_test: diff --git a/web/tests/functional/authentication/__init__.py b/web/tests/functional/authentication/__init__.py index 269a4da0a3..77fc7da15a 100644 --- a/web/tests/functional/authentication/__init__.py +++ b/web/tests/functional/authentication/__init__.py @@ -24,7 +24,7 @@ TEST_WORKSPACE = None -def setup_package(): +def setup_class_common(): """Setup the environment for the tests then start the server.""" global TEST_WORKSPACE @@ -65,7 +65,7 @@ def setup_package(): codechecker.add_test_package_product(host_port_cfg, TEST_WORKSPACE) -def teardown_package(): +def teardown_class_common(): """Stop the CodeChecker server and clean up after the tests.""" # TODO If environment variable is set keep the workspace # and print out the path. @@ -77,6 +77,7 @@ def teardown_package(): codechecker_cfg['check_env']) __STOP_SERVER.set() + __STOP_SERVER.clear() print("Removing: " + TEST_WORKSPACE) shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/authentication/test_authentication.py b/web/tests/functional/authentication/test_authentication.py index 57d0f89956..d0244a5f90 100644 --- a/web/tests/functional/authentication/test_authentication.py +++ b/web/tests/functional/authentication/test_authentication.py @@ -22,14 +22,21 @@ from libtest import codechecker from libtest import env +from . import setup_class_common, teardown_class_common + class DictAuth(unittest.TestCase): """ Dictionary based authentication tests. """ - def setUp(self): + def setup_class(self): + setup_class_common() + + def teardown_class(self): + teardown_class_common() + def setup_method(self, method): # Get the test workspace used to authentication tests. self._test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/authentication/test_permission_management.py b/web/tests/functional/authentication/test_permission_management.py index e4cb239a51..8f93d1a5c4 100644 --- a/web/tests/functional/authentication/test_permission_management.py +++ b/web/tests/functional/authentication/test_permission_management.py @@ -18,10 +18,18 @@ from libtest import codechecker, env +from . import setup_class_common, teardown_class_common + class PermissionManagement(unittest.TestCase): - def setUp(self): + def setup_class(self): + setup_class_common() + + def teardown_class(self): + teardown_class_common() + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . self._test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/authentication/test_permission_view.py b/web/tests/functional/authentication/test_permission_view.py index 945ef153e2..b2b64f72f3 100644 --- a/web/tests/functional/authentication/test_permission_view.py +++ b/web/tests/functional/authentication/test_permission_view.py @@ -19,10 +19,18 @@ from libtest import codechecker, env +from . import setup_class_common, teardown_class_common + class PermissionManagement(unittest.TestCase): - def setUp(self): + def setup_class(self): + setup_class_common() + + def teardown_class(self): + teardown_class_common() + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . self._test_workspace = os.environ['TEST_WORKSPACE'] @@ -108,9 +116,9 @@ def test_get_access_control_with_permission_viewer(self): 'admin': ['PRODUCT_ADMIN']}, auth_product_permissions.user) - self.assertDictContainsSubset( - {'ADMIN_group': ['PRODUCT_ADMIN']}, - auth_product_permissions.group) + # Previously, the test files in this directory interfered at one + # another, and the group permission dict wasn't empty. Check git blame. + self.assertEqual({}, auth_product_permissions.group) # Remove previously added extra permissions. ret = self.root_client.removePermission( @@ -164,14 +172,15 @@ def test_get_access_control_from_cli(self): cmd, env=cmd_env, encoding="utf-8", errors="ignore") access_control = json.loads(out) + + # Previously, the test files in this directory interfered at one + # another, and the group permission dict wasn't empty. Check git blame. self.assertDictEqual({ "version": 1, "global_permissions": { "user_permissions": { "root": ["SUPERUSER"]}, - "group_permissions": { - 'admins_custom_group': ['SUPERUSER'] - } + "group_permissions": {} }, "product_permissions": { "authentication": { @@ -179,6 +188,5 @@ def test_get_access_control_from_cli(self): "cc": ["PRODUCT_STORE"], "john": ["PRODUCT_STORE"], "admin": ["PRODUCT_ADMIN"]}, - "group_permissions": { - 'ADMIN_group': ['PRODUCT_ADMIN']}}}}, + "group_permissions": {}}}}, access_control) diff --git a/web/tests/functional/blame/__init__.py b/web/tests/functional/blame/__init__.py index c47818b253..f30c492687 100644 --- a/web/tests/functional/blame/__init__.py +++ b/web/tests/functional/blame/__init__.py @@ -6,94 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- -"""Setup for the test package blame info.""" - - -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import project - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing blame information.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('blame') - - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_project = 'cpp' - test_config = {} - project_info = project.get_info(test_project) - - test_project_path = os.path.join(TEST_WORKSPACE, "test_proj") - shutil.copytree(project.path(test_project), test_project_path) - - project_info['project_path'] = test_project_path - test_project_name = project_info['name'] + '_' + uuid.uuid4().hex - test_config['test_project'] = project_info - - suppress_file = None - skip_list_file = None - test_env = env.test_env(TEST_WORKSPACE) - - codechecker_cfg = { - 'suppress_file': suppress_file, - 'skip_list_file': skip_list_file, - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'clean': True - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server(auth_required=True) - server_access['viewer_product'] = 'blame' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Clean the test project, if needed by the tests. - ret = project.clean(test_project) - if ret: - sys.exit(ret) - - ret = codechecker.check_and_store( - codechecker_cfg, test_project_name, project.path(test_project)) - if ret: - sys.exit(1) - print("Analyzing test project was succcessful.") - - # Save the run names in the configuration. - codechecker_cfg['run_names'] = [test_project_name] - - test_config['codechecker_cfg'] = codechecker_cfg - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/blame/test_blame_info.py b/web/tests/functional/blame/test_blame_info.py index 9e797f26e9..a8482d185a 100644 --- a/web/tests/functional/blame/test_blame_info.py +++ b/web/tests/functional/blame/test_blame_info.py @@ -12,8 +12,11 @@ import json import os import subprocess +import shutil +import sys import tempfile import unittest +import uuid from codechecker_api.codeCheckerDBAccess_v6.ttypes import Order, \ ReportFilter, RunFilter, RunSortMode, RunSortType @@ -21,11 +24,87 @@ from libtest import codechecker from libtest import env +from libtest import project class TestBlameInfo(unittest.TestCase): - def setUp(self): + def setup_class(self): + """Setup the environment for testing blame information.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('blame') + + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_project = 'cpp' + test_config = {} + project_info = project.get_info(test_project) + + test_project_path = os.path.join(TEST_WORKSPACE, "test_proj") + shutil.copytree(project.path(test_project), test_project_path) + + project_info['project_path'] = test_project_path + test_project_name = project_info['name'] + '_' + uuid.uuid4().hex + test_config['test_project'] = project_info + + suppress_file = None + skip_list_file = None + test_env = env.test_env(TEST_WORKSPACE) + + codechecker_cfg = { + 'suppress_file': suppress_file, + 'skip_list_file': skip_list_file, + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'clean': True + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server(auth_required=True) + server_access['viewer_product'] = 'blame' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Clean the test project, if needed by the tests. + ret = project.clean(test_project) + if ret: + sys.exit(ret) + + ret = codechecker.check_and_store( + codechecker_cfg, test_project_name, project.path(test_project)) + if ret: + sys.exit(1) + print("Analyzing test project was succcessful.") + + # Save the run names in the configuration. + codechecker_cfg['run_names'] = [test_project_name] + + test_config['codechecker_cfg'] = codechecker_cfg + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . self.test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/cleanup_plan/__init__.py b/web/tests/functional/cleanup_plan/__init__.py index b5b84366bc..f30c492687 100644 --- a/web/tests/functional/cleanup_plan/__init__.py +++ b/web/tests/functional/cleanup_plan/__init__.py @@ -6,82 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- -"""Setup for the test package cleanup plan.""" - - -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import project - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for the tests. """ - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('cleanup_plan') - - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_project = 'cpp' - - test_project_path = project.path(test_project) - - test_config = {} - - project_info = project.get_info(test_project) - - test_config['test_project'] = project_info - test_env = env.test_env(TEST_WORKSPACE) - - codechecker_cfg = { - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), - 'checkers': [] - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'cleanup_plan' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Check the test project for the first time. - test_project_name = project_info['name'] + '_' + uuid.uuid4().hex - ret = codechecker.check_and_store(codechecker_cfg, - test_project_name, - test_project_path) - if ret: - sys.exit(1) - print("Analyzing test project was successful.") - - codechecker_cfg['run_names'] = [test_project_name] - test_config['codechecker_cfg'] = codechecker_cfg - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/cleanup_plan/test_cleanup_plan.py b/web/tests/functional/cleanup_plan/test_cleanup_plan.py index 9780f01af8..e36023f002 100644 --- a/web/tests/functional/cleanup_plan/test_cleanup_plan.py +++ b/web/tests/functional/cleanup_plan/test_cleanup_plan.py @@ -13,19 +13,87 @@ import datetime import os +import shutil +import sys +import uuid import time import unittest from codechecker_api.codeCheckerDBAccess_v6.ttypes import CleanupPlanFilter, \ ReportFilter +from libtest import codechecker from libtest import env +from libtest import project from libtest.thrift_client_to_db import get_all_run_results class TestComment(unittest.TestCase): - def setUp(self): + def setup_class(self): + """Setup the environment for the tests. """ + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('cleanup_plan') + + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_project = 'cpp' + + test_project_path = project.path(test_project) + + test_config = {} + + project_info = project.get_info(test_project) + + test_config['test_project'] = project_info + test_env = env.test_env(TEST_WORKSPACE) + + codechecker_cfg = { + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), + 'checkers': [] + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'cleanup_plan' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Check the test project for the first time. + test_project_name = project_info['name'] + '_' + uuid.uuid4().hex + ret = codechecker.check_and_store(codechecker_cfg, + test_project_name, + test_project_path) + if ret: + sys.exit(1) + print("Analyzing test project was successful.") + + codechecker_cfg['run_names'] = [test_project_name] + test_config['codechecker_cfg'] = codechecker_cfg + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): self._test_workspace = os.environ.get('TEST_WORKSPACE') test_class = self.__class__.__name__ diff --git a/web/tests/functional/cli_config/__init__.py b/web/tests/functional/cli_config/__init__.py index cfa0f7e73d..786eeaa671 100644 --- a/web/tests/functional/cli_config/__init__.py +++ b/web/tests/functional/cli_config/__init__.py @@ -22,11 +22,11 @@ TEST_WORKSPACE = None -def setup_package(): +def setup_class_common(workspace_name): """Setup the environment for the tests.""" global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('config') + TEST_WORKSPACE = env.get_workspace(workspace_name) # Set the TEST_WORKSPACE used by the tests. os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE @@ -44,7 +44,7 @@ def setup_package(): # details. print("This test uses a CodeChecker server... connecting...") server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'config' + server_access['viewer_product'] = workspace_name codechecker.add_test_package_product(server_access, TEST_WORKSPACE) # Extend the checker configuration with the server access. @@ -56,7 +56,7 @@ def setup_package(): env.export_test_cfg(TEST_WORKSPACE, test_config) -def teardown_package(): +def teardown_class_common(): """ Delete the workspace associated with this test. """ # TODO: If environment variable is set keep the workspace diff --git a/web/tests/functional/cli_config/test_server_config.py b/web/tests/functional/cli_config/test_server_config.py index 2366d1c500..151dd9d849 100644 --- a/web/tests/functional/cli_config/test_server_config.py +++ b/web/tests/functional/cli_config/test_server_config.py @@ -20,11 +20,19 @@ from libtest import codechecker from libtest import env +from . import setup_class_common, teardown_class_common + class TestServerConfig(unittest.TestCase): _ccClient = None - def setUp(self): + def setup_class(self): + setup_class_common("server_config") + + def teardown_class(self): + teardown_class_common() + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . self.test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/cli_config/test_store_config.py b/web/tests/functional/cli_config/test_store_config.py index 1835018f52..3aab095197 100644 --- a/web/tests/functional/cli_config/test_store_config.py +++ b/web/tests/functional/cli_config/test_store_config.py @@ -19,11 +19,19 @@ from libtest import env +from . import setup_class_common, teardown_class_common + class TestStoreConfig(unittest.TestCase): _ccClient = None - def setUp(self): + def setup_class(self): + setup_class_common("store_config") + + def teardown_class(self): + teardown_class_common() + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . self.test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/cmdline/__init__.py b/web/tests/functional/cmdline/__init__.py index 8e1c9e0086..f30c492687 100644 --- a/web/tests/functional/cmdline/__init__.py +++ b/web/tests/functional/cmdline/__init__.py @@ -6,111 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the package tests.""" - - -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import project - - -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for the tests.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('cmdline') - - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_project = 'cpp' - - test_config = {} - - project_info = project.get_info(test_project) - - # Copy the test project to the workspace. The tests should - # work only on this test project. - test_proj_path = os.path.join(TEST_WORKSPACE, "test_proj") - shutil.copytree(project.path(test_project), test_proj_path) - - project_info['project_path'] = test_proj_path - - test_config['test_project'] = project_info - - suppress_file = None - - skip_list_file = None - - test_env = env.test_env(TEST_WORKSPACE) - - codechecker_cfg = { - 'suppress_file': suppress_file, - 'skip_list_file': skip_list_file, - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'description': "Runs for command line test." - } - - ret = project.clean(test_project) - if ret: - sys.exit(ret) - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'cmdline' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Generate a unique name for this test run. - test_project_name_1 = project_info['name'] + '1_' + uuid.uuid4().hex - - ret = codechecker.check_and_store(codechecker_cfg, - test_project_name_1, - project.path(test_project)) - if ret: - sys.exit(1) - print("Analyzing the test project was successful.") - - test_project_name_2 = project_info['name'] + '2_' + uuid.uuid4().hex - - ret = codechecker.check_and_store(codechecker_cfg, - test_project_name_2, - project.path(test_project)) - if ret: - sys.exit(1) - print("Analyzing the test project was successful.") - - codechecker_cfg['run_names'] = [test_project_name_1, test_project_name_2] - - test_config['codechecker_cfg'] = codechecker_cfg - - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: if environment variable is set keep the workspace - # and print out the path - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/cmdline/test_cmdline.py b/web/tests/functional/cmdline/test_cmdline.py index 3ad90281dc..eec3ff5efe 100644 --- a/web/tests/functional/cmdline/test_cmdline.py +++ b/web/tests/functional/cmdline/test_cmdline.py @@ -13,11 +13,16 @@ import json import os +import shutil import subprocess +import sys +import uuid import tempfile import unittest +from libtest import codechecker from libtest import env +from libtest import project def run_cmd(cmd, env=None): @@ -40,7 +45,99 @@ class TestCmdline(unittest.TestCase): Simple tests to check CodeChecker command line. """ - def setUp(self): + def setup_class(self): + """Setup the environment for the tests.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('cmdline') + + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_project = 'cpp' + + test_config = {} + + project_info = project.get_info(test_project) + + # Copy the test project to the workspace. The tests should + # work only on this test project. + test_proj_path = os.path.join(TEST_WORKSPACE, "test_proj") + shutil.copytree(project.path(test_project), test_proj_path) + + project_info['project_path'] = test_proj_path + + test_config['test_project'] = project_info + + suppress_file = None + + skip_list_file = None + + test_env = env.test_env(TEST_WORKSPACE) + + codechecker_cfg = { + 'suppress_file': suppress_file, + 'skip_list_file': skip_list_file, + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'description': "Runs for command line test." + } + + ret = project.clean(test_project) + if ret: + sys.exit(ret) + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'cmdline' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Generate a unique name for this test run. + test_project_name_1 = project_info['name'] + '1_' + uuid.uuid4().hex + + ret = codechecker.check_and_store(codechecker_cfg, + test_project_name_1, + project.path(test_project)) + if ret: + sys.exit(1) + print("Analyzing the test project was successful.") + + test_project_name_2 = project_info['name'] + '2_' + uuid.uuid4().hex + + ret = codechecker.check_and_store(codechecker_cfg, + test_project_name_2, + project.path(test_project)) + if ret: + sys.exit(1) + print("Analyzing the test project was successful.") + + codechecker_cfg['run_names'] = [test_project_name_1, + test_project_name_2] + + test_config['codechecker_cfg'] = codechecker_cfg + + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: if environment variable is set keep the workspace + # and print out the path + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): test_workspace = os.environ.get('TEST_WORKSPACE') diff --git a/web/tests/functional/comment/__init__.py b/web/tests/functional/comment/__init__.py index 3a466e0df7..f30c492687 100644 --- a/web/tests/functional/comment/__init__.py +++ b/web/tests/functional/comment/__init__.py @@ -6,105 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- -"""Setup for the test package comment.""" - - -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import project - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for the tests. """ - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('comment') - - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_project = 'cpp' - - test_project_path = project.path(test_project) - - test_config = {} - - project_info = project.get_info(test_project) - - test_config['test_project'] = project_info - - suppress_file = None - - skip_list_file = None - - test_env = env.test_env(TEST_WORKSPACE) - - codechecker_cfg = { - 'suppress_file': suppress_file, - 'skip_list_file': skip_list_file, - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), - 'checkers': ['-d', 'core.CallAndMessage', - '-e', 'core.StackAddressEscape'], - 'analyzers': ['clangsa', 'clang-tidy'] - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server(auth_required=True) - server_access['viewer_product'] = 'comment' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Check the test project for the first time. - test_project_names = [] - test_project_name = project_info['name'] + '_' + uuid.uuid4().hex - test_project_names.append(test_project_name) - - ret = codechecker.check_and_store(codechecker_cfg, - test_project_name, - test_project_path) - if ret: - sys.exit(1) - print("Analyzing test project was successful.") - - # Check the test project again. - test_project_name = project_info['name'] + '_' + uuid.uuid4().hex - test_project_names.append(test_project_name) - - ret = codechecker.check_and_store(codechecker_cfg, - test_project_name, - test_project_path) - if ret: - sys.exit(1) - print("Analyzing test project was succcessful.") - - codechecker_cfg['run_names'] = test_project_names - test_config['codechecker_cfg'] = codechecker_cfg - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/comment/test_comment.py b/web/tests/functional/comment/test_comment.py index 0b4055a0d5..e20f01ca68 100644 --- a/web/tests/functional/comment/test_comment.py +++ b/web/tests/functional/comment/test_comment.py @@ -13,13 +13,19 @@ import html import logging import os +import shutil +import sys +import uuid + import unittest from codechecker_api_shared.ttypes import RequestFailed from codechecker_api.codeCheckerDBAccess_v6.ttypes import CommentData, \ CommentKind +from libtest import codechecker from libtest import env +from libtest import project from libtest.thrift_client_to_db import get_all_run_results @@ -40,7 +46,93 @@ class TestComment(unittest.TestCase): _ccClient = None - def setUp(self): + def setup_class(self): + """Setup the environment for the tests. """ + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('comment') + + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_project = 'cpp' + + test_project_path = project.path(test_project) + + test_config = {} + + project_info = project.get_info(test_project) + + test_config['test_project'] = project_info + + suppress_file = None + + skip_list_file = None + + test_env = env.test_env(TEST_WORKSPACE) + + codechecker_cfg = { + 'suppress_file': suppress_file, + 'skip_list_file': skip_list_file, + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), + 'checkers': ['-d', 'core.CallAndMessage', + '-e', 'core.StackAddressEscape'], + 'analyzers': ['clangsa', 'clang-tidy'] + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server(auth_required=True) + server_access['viewer_product'] = 'comment' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Check the test project for the first time. + test_project_names = [] + test_project_name = project_info['name'] + '_' + uuid.uuid4().hex + test_project_names.append(test_project_name) + + ret = codechecker.check_and_store(codechecker_cfg, + test_project_name, + test_project_path) + if ret: + sys.exit(1) + print("Analyzing test project was successful.") + + # Check the test project again. + test_project_name = project_info['name'] + '_' + uuid.uuid4().hex + test_project_names.append(test_project_name) + + ret = codechecker.check_and_store(codechecker_cfg, + test_project_name, + test_project_path) + if ret: + sys.exit(1) + print("Analyzing test project was succcessful.") + + codechecker_cfg['run_names'] = test_project_names + test_config['codechecker_cfg'] = codechecker_cfg + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): self._test_workspace = os.environ.get('TEST_WORKSPACE') test_class = self.__class__.__name__ diff --git a/web/tests/functional/component/__init__.py b/web/tests/functional/component/__init__.py index 17c305087b..f30c492687 100644 --- a/web/tests/functional/component/__init__.py +++ b/web/tests/functional/component/__init__.py @@ -6,101 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- -"""Setup for the test package component.""" - - -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import project - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing components.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('component') - - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_project = 'cpp' - - test_config = {} - - project_info = project.get_info(test_project) - - test_project_path = os.path.join(TEST_WORKSPACE, "test_proj") - shutil.copytree(project.path(test_project), test_project_path) - - project_info['project_path'] = test_project_path - - test_project_name = project_info['name'] + '_' + uuid.uuid4().hex - - test_config['test_project'] = project_info - - suppress_file = None - - skip_list_file = None - - test_env = env.test_env(TEST_WORKSPACE) - - codechecker_cfg = { - 'suppress_file': suppress_file, - 'skip_list_file': skip_list_file, - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'analyzers': ['clangsa', 'clang-tidy'] - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server(auth_required=True) - server_access['viewer_product'] = 'component' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Clean the test project, if needed by the tests. - ret = project.clean(test_project_path) - if ret: - sys.exit(ret) - - ret = codechecker.check_and_store(codechecker_cfg, - test_project_name, - test_project_path) - if ret: - sys.exit(1) - print("Analyzing test project was succcessful.") - - # Save the run names in the configuration. - codechecker_cfg['run_names'] = [test_project_name] - - test_config['codechecker_cfg'] = codechecker_cfg - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/component/test_component.py b/web/tests/functional/component/test_component.py index 6fe0dbaf58..c6f92810ec 100644 --- a/web/tests/functional/component/test_component.py +++ b/web/tests/functional/component/test_component.py @@ -12,20 +12,107 @@ import os +import shutil +import sys import unittest +import uuid from codechecker_api_shared.ttypes import Permission from codechecker_api.codeCheckerDBAccess_v6.ttypes import ReportFilter +from libtest import codechecker from libtest import env +from libtest import project GEN_OTHER_COMPONENT_NAME = "Other (auto-generated)" class TestComponent(unittest.TestCase): - def setUp(self): + def setup_class(self): + """Setup the environment for testing components.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('component') + + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_project = 'cpp' + + test_config = {} + + project_info = project.get_info(test_project) + + test_project_path = os.path.join(TEST_WORKSPACE, "test_proj") + shutil.copytree(project.path(test_project), test_project_path) + + project_info['project_path'] = test_project_path + + test_project_name = project_info['name'] + '_' + uuid.uuid4().hex + + test_config['test_project'] = project_info + + suppress_file = None + + skip_list_file = None + + test_env = env.test_env(TEST_WORKSPACE) + + codechecker_cfg = { + 'suppress_file': suppress_file, + 'skip_list_file': skip_list_file, + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'analyzers': ['clangsa', 'clang-tidy'] + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server(auth_required=True) + server_access['viewer_product'] = 'component' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Clean the test project, if needed by the tests. + ret = project.clean(test_project_path) + if ret: + sys.exit(ret) + + ret = codechecker.check_and_store(codechecker_cfg, + test_project_name, + test_project_path) + if ret: + sys.exit(1) + print("Analyzing test project was succcessful.") + + # Save the run names in the configuration. + codechecker_cfg['run_names'] = [test_project_name] + + test_config['codechecker_cfg'] = codechecker_cfg + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): self._test_workspace = os.environ.get('TEST_WORKSPACE') test_class = self.__class__.__name__ diff --git a/web/tests/functional/cppcheck/__init__.py b/web/tests/functional/cppcheck/__init__.py index ef660ef4e0..f30c492687 100644 --- a/web/tests/functional/cppcheck/__init__.py +++ b/web/tests/functional/cppcheck/__init__.py @@ -6,67 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the package tests.""" - - -import multiprocessing -import os -import shutil - -from libtest import codechecker -from libtest import env - -# Stopping event for CodeChecker server. -__STOP_SERVER = multiprocessing.Event() - -# Test workspace initialized at setup for cppcheck tests. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for the tests then start the server.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('cppcheck') - - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - # Configuration options. - codechecker_cfg = { - 'suppress_file': None, - 'skip_list_file': None, - 'check_env': env.test_env(TEST_WORKSPACE), - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), - 'test_project': 'cppcheck' - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'cppcheck' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, {'codechecker_cfg': codechecker_cfg}) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/cppcheck/test_cppcheck.py b/web/tests/functional/cppcheck/test_cppcheck.py index 5e1dbbafaa..20c4040c6a 100644 --- a/web/tests/functional/cppcheck/test_cppcheck.py +++ b/web/tests/functional/cppcheck/test_cppcheck.py @@ -17,6 +17,7 @@ import subprocess import unittest +from libtest import codechecker from libtest import env from libtest import plist_test @@ -26,7 +27,54 @@ class CppCheck(unittest.TestCase): Test storage of cppcheck reports """ - def setUp(self): + def setup_class(self): + """Setup the environment for the tests then start the server.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('cppcheck') + + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + # Configuration options. + codechecker_cfg = { + 'suppress_file': None, + 'skip_list_file': None, + 'check_env': env.test_env(TEST_WORKSPACE), + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), + 'test_project': 'cppcheck' + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'cppcheck' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, + {'codechecker_cfg': codechecker_cfg}) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # Get the test workspace used to cppcheck tests. self._test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/db_cleanup/__init__.py b/web/tests/functional/db_cleanup/__init__.py index f6cc3f9a48..f30c492687 100644 --- a/web/tests/functional/db_cleanup/__init__.py +++ b/web/tests/functional/db_cleanup/__init__.py @@ -6,48 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - - -import os -import shutil - -from libtest import env - - -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing db_cleanup.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('db_cleanup') - - # Set the TEST_WORKSPACE used by the tests. - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - # Create a basic CodeChecker config for the tests, this should - # be imported by the tests and they should only depend on these - # configuration options. - codechecker_cfg = { - 'suppress_file': None, - 'skip_list_file': None, - 'check_env': env.test_env(TEST_WORKSPACE), - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'viewer_host': 'localhost', - 'viewer_product': 'db_cleanup', - 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), - 'analyzers': ['clangsa', 'clang-tidy'] - } - - env.export_test_cfg(TEST_WORKSPACE, {'codechecker_cfg': codechecker_cfg}) - - -def teardown_package(): - """Clean up after the test.""" - - global TEST_WORKSPACE - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/db_cleanup/test_db_cleanup.py b/web/tests/functional/db_cleanup/test_db_cleanup.py index 3443838f72..119333480f 100644 --- a/web/tests/functional/db_cleanup/test_db_cleanup.py +++ b/web/tests/functional/db_cleanup/test_db_cleanup.py @@ -14,6 +14,7 @@ import json import multiprocessing import os +import shutil import unittest from shutil import copytree, rmtree @@ -27,7 +28,42 @@ class TestDbCleanup(unittest.TestCase): - def setUp(self): + def setup_class(self): + """Setup the environment for testing db_cleanup.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('db_cleanup') + + # Set the TEST_WORKSPACE used by the tests. + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + # Create a basic CodeChecker config for the tests, this should + # be imported by the tests and they should only depend on these + # configuration options. + codechecker_cfg = { + 'suppress_file': None, + 'skip_list_file': None, + 'check_env': env.test_env(TEST_WORKSPACE), + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'viewer_host': 'localhost', + 'viewer_product': 'db_cleanup', + 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), + 'analyzers': ['clangsa', 'clang-tidy'] + } + + env.export_test_cfg(TEST_WORKSPACE, + {'codechecker_cfg': codechecker_cfg}) + + def teardown_class(self): + """Clean up after the test.""" + + global TEST_WORKSPACE + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): self.test_workspace = os.environ['TEST_WORKSPACE'] test_class = self.__class__.__name__ diff --git a/web/tests/functional/delete_runs/__init__.py b/web/tests/functional/delete_runs/__init__.py index fea3e6373a..f30c492687 100644 --- a/web/tests/functional/delete_runs/__init__.py +++ b/web/tests/functional/delete_runs/__init__.py @@ -6,122 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the test package delete_runs.""" - - -import os -import shutil -import sys -import time - -from libtest import codechecker -from libtest import env -from libtest import project - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing delete_runs.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('delete_runs') - - # Set the TEST_WORKSPACE used by the tests. - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_config = {} - - test_project = 'simple' - - project_info = project.get_info(test_project) - - # Copy the test project to the workspace. The tests should - # work only on this test project. - test_proj_path = os.path.join(TEST_WORKSPACE, "test_proj") - shutil.copytree(project.path(test_project), test_proj_path) - - project_info['project_path'] = test_proj_path - - # Generate a unique name for this test run. - test_project_name = project_info['name'] - - test_config['test_project'] = project_info - - # Suppress file should be set here if needed by the tests. - suppress_file = None - - # Skip list file should be set here if needed by the tests. - skip_list_file = None - - # Get an environment which should be used by the tests. - test_env = env.test_env(TEST_WORKSPACE) - - # Create a basic CodeChecker config for the tests, this should - # be imported by the tests and they should only depend on these - # configuration options. - codechecker_cfg = { - 'suppress_file': suppress_file, - 'skip_list_file': skip_list_file, - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'analyzers': ['clangsa', 'clang-tidy'] - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'delete_runs' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - for i in range(0, 5): - # Clean the test project, if needed by the tests. - ret = project.clean(test_project) - if ret: - sys.exit(ret) - - # Check the test project, if needed by the tests. - ret = codechecker.check_and_store(codechecker_cfg, - test_project_name + '_' + str(i), - test_proj_path) - if ret: - sys.exit(1) - - print("Analyzing the test project was successful {}.".format(str(i))) - - # If the check process is very fast, datetime of multiple runs can be - # almost the same different in microseconds. Test cases of delete runs - # can be failed for this reason because we didn't process microseconds - # in command line arguments. - time.sleep(1) - - # Save the run names in the configuration. - codechecker_cfg['run_names'] \ - = [test_project_name + '_' + str(i) for i in range(0, 5)] - - test_config['codechecker_cfg'] = codechecker_cfg - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/delete_runs/test_delete_runs.py b/web/tests/functional/delete_runs/test_delete_runs.py index 92fb002bc7..4e21ab03b8 100644 --- a/web/tests/functional/delete_runs/test_delete_runs.py +++ b/web/tests/functional/delete_runs/test_delete_runs.py @@ -14,11 +14,17 @@ import json import os -import unittest +import shutil +import sys +import time import subprocess +import unittest + from datetime import datetime +from libtest import codechecker from libtest import env +from libtest import project def run_cmd(cmd, env): @@ -34,7 +40,110 @@ def run_cmd(cmd, env): class TestCmdLineDeletion(unittest.TestCase): - def setUp(self): + def setup_class(self): + """Setup the environment for testing delete_runs.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('delete_runs') + + # Set the TEST_WORKSPACE used by the tests. + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_config = {} + + test_project = 'simple' + + project_info = project.get_info(test_project) + + # Copy the test project to the workspace. The tests should + # work only on this test project. + test_proj_path = os.path.join(TEST_WORKSPACE, "test_proj") + shutil.copytree(project.path(test_project), test_proj_path) + + project_info['project_path'] = test_proj_path + + # Generate a unique name for this test run. + test_project_name = project_info['name'] + + test_config['test_project'] = project_info + + # Suppress file should be set here if needed by the tests. + suppress_file = None + + # Skip list file should be set here if needed by the tests. + skip_list_file = None + + # Get an environment which should be used by the tests. + test_env = env.test_env(TEST_WORKSPACE) + + # Create a basic CodeChecker config for the tests, this should + # be imported by the tests and they should only depend on these + # configuration options. + codechecker_cfg = { + 'suppress_file': suppress_file, + 'skip_list_file': skip_list_file, + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'analyzers': ['clangsa', 'clang-tidy'] + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'delete_runs' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + for i in range(0, 5): + # Clean the test project, if needed by the tests. + ret = project.clean(test_project) + if ret: + sys.exit(ret) + + # Check the test project, if needed by the tests. + ret = codechecker.check_and_store(codechecker_cfg, + test_project_name + '_' + str(i), + test_proj_path) + if ret: + sys.exit(1) + + print("Analyzing the test project was successful {}." + .format(str(i))) + + # If the check process is very fast, datetime of multiple runs can + # be almost the same different in microseconds. Test cases of + # delete runs can be failed for this reason because we didn't + # process microseconds in command line arguments. + time.sleep(1) + + # Save the run names in the configuration. + codechecker_cfg['run_names'] \ + = [test_project_name + '_' + str(i) for i in range(0, 5)] + + test_config['codechecker_cfg'] = codechecker_cfg + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/detection_status/__init__.py b/web/tests/functional/detection_status/__init__.py index 6faff00d64..f30c492687 100644 --- a/web/tests/functional/detection_status/__init__.py +++ b/web/tests/functional/detection_status/__init__.py @@ -6,66 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the test package detection_status.""" - - -import os -import shutil -import time - -from libtest import codechecker -from libtest import env - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing detection_status.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('detection_status') - - # Set the TEST_WORKSPACE used by the tests. - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - # Configuration options. - codechecker_cfg = { - 'suppress_file': None, - 'skip_list_file': None, - 'check_env': env.test_env(TEST_WORKSPACE), - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), - 'test_project': 'hello', - 'analyzers': ['clangsa', 'clang-tidy'] - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'detection_status' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, {'codechecker_cfg': codechecker_cfg}) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/detection_status/test_detection_status.py b/web/tests/functional/detection_status/test_detection_status.py index a50d489024..23c4d1e0b5 100644 --- a/web/tests/functional/detection_status/test_detection_status.py +++ b/web/tests/functional/detection_status/test_detection_status.py @@ -23,7 +23,56 @@ class TestDetectionStatus(unittest.TestCase): - def setUp(self): + def setup_class(self): + """Setup the environment for testing detection_status.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('detection_status') + + # Set the TEST_WORKSPACE used by the tests. + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + # Configuration options. + codechecker_cfg = { + 'suppress_file': None, + 'skip_list_file': None, + 'check_env': env.test_env(TEST_WORKSPACE), + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), + 'test_project': 'hello', + 'analyzers': ['clangsa', 'clang-tidy'] + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'detection_status' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, + {'codechecker_cfg': codechecker_cfg}) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . self.test_workspace = os.environ['TEST_WORKSPACE'] @@ -124,7 +173,7 @@ def setUp(self): sizeof(42); }"""] - def tearDown(self): + def teardown_method(self, method): """Restore environment after tests have ran.""" os.chdir(self.__old_pwd) diff --git a/web/tests/functional/diff_local/__init__.py b/web/tests/functional/diff_local/__init__.py index c2465b6904..f30c492687 100644 --- a/web/tests/functional/diff_local/__init__.py +++ b/web/tests/functional/diff_local/__init__.py @@ -6,110 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup part for the test package diff_local. """ - - -import os -import shutil -import sys - -from libtest import codechecker -from libtest import env -from libtest import project - - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing diff_local.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('diff_local') - - # Set the TEST_WORKSPACE used by the tests. - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_config = {} - - test_project = 'cpp' - - project_info = project.get_info(test_project) - - # Copy the test project to the workspace. The tests should - # work only on this test project. - test_proj_path_base = os.path.join(TEST_WORKSPACE, "test_proj_base") - shutil.copytree(project.path(test_project), test_proj_path_base) - - # Copy the test project to the workspace. The tests should - # work only on this test project. - test_proj_path_new = os.path.join(TEST_WORKSPACE, "test_proj_new") - shutil.copytree(project.path(test_project), test_proj_path_new) - - project_info['project_path_base'] = test_proj_path_base - project_info['project_path_new'] = test_proj_path_new - - test_config['test_project'] = project_info - - # Suppress file should be set here if needed by the tests. - suppress_file = None - - # Skip list file should be set here if needed by the tests. - skip_list_file = None - - # Get an environment which should be used by the tests. - test_env = env.test_env(TEST_WORKSPACE) - - # Create a basic CodeChecker config for the tests, this should - # be imported by the tests and they should only depend on these - # configuration options. - codechecker_cfg = { - 'suppress_file': suppress_file, - 'skip_list_file': skip_list_file, - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'analyzers': ['clangsa', 'clang-tidy'] - } - - # Base analysis - codechecker_cfg['reportdir'] = os.path.join(test_proj_path_base, - 'reports') - codechecker_cfg['checkers'] = ['-e', 'core.CallAndMessage', - '-d', 'core.NullDereference'] - - ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_base) - if ret: - sys.exit(1) - - # New analysis - codechecker_cfg['reportdir'] = os.path.join(test_proj_path_new, - 'reports') - codechecker_cfg['checkers'] = ['-d', 'core.CallAndMessage', - '-e', 'core.NullDereference'] - - ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_new) - if ret: - sys.exit(1) - - codechecker_cfg['reportdir_base'] = os.path.join(test_proj_path_base, - 'reports') - codechecker_cfg['reportdir_new'] = os.path.join(test_proj_path_new, - 'reports') - test_config['codechecker_cfg'] = codechecker_cfg - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/diff_local/test_diff_local.py b/web/tests/functional/diff_local/test_diff_local.py index 085858302c..c51ecc30f5 100644 --- a/web/tests/functional/diff_local/test_diff_local.py +++ b/web/tests/functional/diff_local/test_diff_local.py @@ -17,16 +17,110 @@ import json import os import re +import shutil import subprocess +import sys import unittest -from libtest import env, codechecker +from libtest import codechecker +from libtest import env +from libtest import project from libtest.codechecker import create_baseline_file, get_diff_results class DiffLocal(unittest.TestCase): - def setUp(self): + def setup_class(self): + """Setup the environment for testing diff_local.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('diff_local') + + # Set the TEST_WORKSPACE used by the tests. + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_config = {} + + test_project = 'cpp' + + project_info = project.get_info(test_project) + + # Copy the test project to the workspace. The tests should + # work only on this test project. + test_proj_path_base = os.path.join(TEST_WORKSPACE, "test_proj_base") + shutil.copytree(project.path(test_project), test_proj_path_base) + + # Copy the test project to the workspace. The tests should + # work only on this test project. + test_proj_path_new = os.path.join(TEST_WORKSPACE, "test_proj_new") + shutil.copytree(project.path(test_project), test_proj_path_new) + + project_info['project_path_base'] = test_proj_path_base + project_info['project_path_new'] = test_proj_path_new + + test_config['test_project'] = project_info + + # Suppress file should be set here if needed by the tests. + suppress_file = None + + # Skip list file should be set here if needed by the tests. + skip_list_file = None + + # Get an environment which should be used by the tests. + test_env = env.test_env(TEST_WORKSPACE) + + # Create a basic CodeChecker config for the tests, this should + # be imported by the tests and they should only depend on these + # configuration options. + codechecker_cfg = { + 'suppress_file': suppress_file, + 'skip_list_file': skip_list_file, + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'analyzers': ['clangsa', 'clang-tidy'] + } + + # Base analysis + codechecker_cfg['reportdir'] = os.path.join(test_proj_path_base, + 'reports') + codechecker_cfg['checkers'] = ['-e', 'core.CallAndMessage', + '-d', 'core.NullDereference'] + + ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_base) + if ret: + sys.exit(1) + + # New analysis + codechecker_cfg['reportdir'] = os.path.join(test_proj_path_new, + 'reports') + codechecker_cfg['checkers'] = ['-d', 'core.CallAndMessage', + '-e', 'core.NullDereference'] + + ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_new) + if ret: + sys.exit(1) + + codechecker_cfg['reportdir_base'] = os.path.join(test_proj_path_base, + 'reports') + codechecker_cfg['reportdir_new'] = os.path.join(test_proj_path_new, + 'reports') + test_config['codechecker_cfg'] = codechecker_cfg + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . test_workspace = os.environ['TEST_WORKSPACE'] @@ -62,7 +156,6 @@ def test_resolved_json(self): """ resolved_results, _, _ = get_diff_results( [self.base_reports], [self.new_reports], '--resolved', 'json') - print(resolved_results) for resolved in resolved_results: self.assertEqual(resolved['checker_name'], "core.CallAndMessage") diff --git a/web/tests/functional/diff_local_remote/__init__.py b/web/tests/functional/diff_local_remote/__init__.py index 0b7813ee63..f30c492687 100644 --- a/web/tests/functional/diff_local_remote/__init__.py +++ b/web/tests/functional/diff_local_remote/__init__.py @@ -6,145 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the test package diff_local_remote.""" - - -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import project - - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing diff_local_remote.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('diff_local_remote') - - # Set the TEST_WORKSPACE used by the tests. - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_config = {} - - test_project = 'cpp' - - project_info = project.get_info(test_project) - - # Copy the test project to the workspace. The tests should - # work only on this test project. - test_proj_path_local = os.path.join(TEST_WORKSPACE, "test_proj_local") - shutil.copytree(project.path(test_project), test_proj_path_local) - - # Copy the test project to the workspace. The tests should - # work only on this test project. - test_proj_path_remote = os.path.join(TEST_WORKSPACE, "test_proj_remote") - shutil.copytree(project.path(test_project), test_proj_path_remote) - - project_info['project_path_local'] = test_proj_path_local - project_info['project_path_remote'] = test_proj_path_remote - - test_config['test_project'] = project_info - - # Suppress file should be set here if needed by the tests. - suppress_file = None - - # Skip list file should be set here if needed by the tests. - skip_list_file = None - - # Get an environment which should be used by the tests. - test_env = env.test_env(TEST_WORKSPACE) - - # Create a basic CodeChecker config for the tests, this should - # be imported by the tests and they should only depend on these - # configuration options. - codechecker_cfg = { - 'suppress_file': suppress_file, - 'skip_list_file': skip_list_file, - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'analyzers': ['clangsa'] - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'diff_local_remote' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Analyze local, these reports will not be stored to the server. - altered_file = os.path.join(test_proj_path_local, "call_and_message.cpp") - project.insert_suppression(altered_file) - - codechecker_cfg['reportdir'] = os.path.join(test_proj_path_local, - 'reports') - codechecker_cfg['checkers'] = ['-e', 'core.CallAndMessage', - '-d', 'core.NullDereference'] - - ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_local) - if ret: - sys.exit(1) - print('Analyzing local was successful.') - - # Store results to the remote server. - test_project_name_remote = project_info['name'] + '_' + uuid.uuid4().hex - ret = codechecker.store(codechecker_cfg, test_project_name_remote) - if ret: - sys.exit(1) - - # Remote analysis, results will be stored to the remote server. - altered_file = os.path.join(test_proj_path_local, "call_and_message.cpp") - project.insert_suppression(altered_file) - - codechecker_cfg['reportdir'] = os.path.join(test_proj_path_remote, - 'reports') - codechecker_cfg['checkers'] = ['-d', 'core.CallAndMessage', - '-e', 'core.NullDereference'] - - ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_remote) - if ret: - sys.exit(1) - print('Analyzing new was successful.') - - # Store results again to the remote server. We need this second store to - # set the detection status to the required states. - ret = codechecker.store(codechecker_cfg, test_project_name_remote) - if ret: - sys.exit(1) - print('Analyzing remote was successful.') - - # Save the run names in the configuration. - codechecker_cfg['run_names'] = [test_project_name_remote] - - test_config['codechecker_cfg'] = codechecker_cfg - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/diff_local_remote/test_diff_local_remote.py b/web/tests/functional/diff_local_remote/test_diff_local_remote.py index 53a5ab800b..33f7358f64 100644 --- a/web/tests/functional/diff_local_remote/test_diff_local_remote.py +++ b/web/tests/functional/diff_local_remote/test_diff_local_remote.py @@ -19,15 +19,149 @@ import re import shutil import subprocess +import sys import unittest +import uuid +from libtest import codechecker from libtest import env +from libtest import project from libtest.codechecker import create_baseline_file, get_diff_results class LocalRemote(unittest.TestCase): - def setUp(self): + def setup_class(self): + """Setup the environment for testing diff_local_remote.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('diff_local_remote') + + # Set the TEST_WORKSPACE used by the tests. + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_config = {} + + test_project = 'cpp' + + project_info = project.get_info(test_project) + + # Copy the test project to the workspace. The tests should + # work only on this test project. + test_proj_path_local = os.path.join(TEST_WORKSPACE, "test_proj_local") + shutil.copytree(project.path(test_project), test_proj_path_local) + + # Copy the test project to the workspace. The tests should + # work only on this test project. + test_proj_path_remote = os.path.join(TEST_WORKSPACE, + "test_proj_remote") + shutil.copytree(project.path(test_project), test_proj_path_remote) + + project_info['project_path_local'] = test_proj_path_local + project_info['project_path_remote'] = test_proj_path_remote + + test_config['test_project'] = project_info + + # Suppress file should be set here if needed by the tests. + suppress_file = None + + # Skip list file should be set here if needed by the tests. + skip_list_file = None + + # Get an environment which should be used by the tests. + test_env = env.test_env(TEST_WORKSPACE) + + # Create a basic CodeChecker config for the tests, this should + # be imported by the tests and they should only depend on these + # configuration options. + codechecker_cfg = { + 'suppress_file': suppress_file, + 'skip_list_file': skip_list_file, + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'analyzers': ['clangsa'] + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'diff_local_remote' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Analyze local, these reports will not be stored to the server. + altered_file = os.path.join(test_proj_path_local, + "call_and_message.cpp") + project.insert_suppression(altered_file) + + codechecker_cfg['reportdir'] = os.path.join(test_proj_path_local, + 'reports') + codechecker_cfg['checkers'] = ['-e', 'core.CallAndMessage', + '-d', 'core.NullDereference'] + + ret = codechecker.log_and_analyze(codechecker_cfg, + test_proj_path_local) + if ret: + sys.exit(1) + print('Analyzing local was successful.') + + # Store results to the remote server. + test_project_name_remote = \ + project_info['name'] + '_' + uuid.uuid4().hex + ret = codechecker.store(codechecker_cfg, test_project_name_remote) + if ret: + sys.exit(1) + + # Remote analysis, results will be stored to the remote server. + altered_file = os.path.join(test_proj_path_local, + "call_and_message.cpp") + project.insert_suppression(altered_file) + + codechecker_cfg['reportdir'] = os.path.join(test_proj_path_remote, + 'reports') + codechecker_cfg['checkers'] = ['-d', 'core.CallAndMessage', + '-e', 'core.NullDereference'] + + ret = codechecker.log_and_analyze(codechecker_cfg, + test_proj_path_remote) + if ret: + sys.exit(1) + print('Analyzing new was successful.') + + # Store results again to the remote server. We need this second store + # to set the detection status to the required states. + ret = codechecker.store(codechecker_cfg, test_project_name_remote) + if ret: + sys.exit(1) + print('Analyzing remote was successful.') + + # Save the run names in the configuration. + codechecker_cfg['run_names'] = [test_project_name_remote] + + test_config['codechecker_cfg'] = codechecker_cfg + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/diff_local_remote_suppress/__init__.py b/web/tests/functional/diff_local_remote_suppress/__init__.py index 446b34dea4..1e7204f557 100644 --- a/web/tests/functional/diff_local_remote_suppress/__init__.py +++ b/web/tests/functional/diff_local_remote_suppress/__init__.py @@ -7,7 +7,7 @@ # # ------------------------------------------------------------------------- -"""Setup for the test package diff_local_remote_suppress.""" +"""Setup for the test package workspace_name.""" import os @@ -20,9 +20,9 @@ from libtest import project -def init_projects(): +def setup_class_common(workspace_name): """ - Setup the environment for testing diff_local_remote_suppress. + Setup the environment for testing workspace_name. Original project ------------------------------------------------------ @@ -62,6 +62,15 @@ def init_projects(): ------------------------------------------------------ """ + os.environ['TEST_WORKSPACE'] = \ + env.get_workspace(workspace_name) + + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = workspace_name + codechecker.add_test_package_product( + server_access, os.environ['TEST_WORKSPACE']) + TEST_WORKSPACE = os.environ['TEST_WORKSPACE'] test_project = 'cpp' @@ -87,9 +96,6 @@ def init_projects(): # Start or connect to the running CodeChecker server and get connection # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'diff_local_remote_suppress' codechecker_cfg.update(server_access) env.export_test_cfg(TEST_WORKSPACE, test_config) @@ -180,24 +186,7 @@ def init_projects(): env.export_test_cfg(TEST_WORKSPACE, test_config) -def setup_package(): - """ - The test files in this diff_local_remote_suppress test share the analyzed - projects. These tests are checking report suppression where the order of - suppressions matters. Since we can't rely on the ordering of tests, the - projects are set up by these tests individually. The setup happens in - function init_projects() and that is imported and executed in each test. - """ - os.environ['TEST_WORKSPACE'] = \ - env.get_workspace('diff_local_remote_suppress') - - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'diff_local_remote_suppress' - codechecker.add_test_package_product( - server_access, os.environ['TEST_WORKSPACE']) - - -def teardown_package(): +def teardown_class_common(): TEST_WORKSPACE = os.environ['TEST_WORKSPACE'] check_env = env.import_test_cfg(TEST_WORKSPACE)[ diff --git a/web/tests/functional/diff_local_remote_suppress/test_diff_local_remote_suppress.py b/web/tests/functional/diff_local_remote_suppress/test_diff_local_remote_suppress.py index e74887daa6..e091006653 100644 --- a/web/tests/functional/diff_local_remote_suppress/test_diff_local_remote_suppress.py +++ b/web/tests/functional/diff_local_remote_suppress/test_diff_local_remote_suppress.py @@ -21,16 +21,18 @@ from libtest import env from libtest.codechecker import get_diff_results -from .__init__ import init_projects +from . import setup_class_common, teardown_class_common class DiffLocalRemoteSuppress(unittest.TestCase): - @classmethod - def setUpClass(cls): - init_projects() + def setup_class(self): + setup_class_common("diff_local_remote_suppress") - def setUp(self): + def teardown_class(self): + teardown_class_common() + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/diff_local_remote_suppress/test_diff_local_remote_suppress_rule.py b/web/tests/functional/diff_local_remote_suppress/test_diff_local_remote_suppress_rule.py index 4ef223a4ce..278674e81f 100644 --- a/web/tests/functional/diff_local_remote_suppress/test_diff_local_remote_suppress_rule.py +++ b/web/tests/functional/diff_local_remote_suppress/test_diff_local_remote_suppress_rule.py @@ -24,16 +24,18 @@ from libtest import env from libtest.codechecker import get_diff_results -from .__init__ import init_projects +from . import setup_class_common, teardown_class_common class DiffLocalRemoteSuppressRule(unittest.TestCase): - @classmethod - def setUpClass(cls): - init_projects() + def setup_class(self): + setup_class_common("diff_local_remote_suppress_rule") - def setUp(self): + def teardown_class(self): + teardown_class_common() + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/diff_remote/__init__.py b/web/tests/functional/diff_remote/__init__.py index 7aa676faf6..f30c492687 100644 --- a/web/tests/functional/diff_remote/__init__.py +++ b/web/tests/functional/diff_remote/__init__.py @@ -6,216 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the test package diff_remote.""" - - -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import project - - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing diff_remote.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('diff_remote') - - # Set the TEST_WORKSPACE used by the tests. - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_config = {} - - test_project = 'cpp' - - project_info = project.get_info(test_project) - - # Copy the test project to the workspace. The tests should - # work only on this test project. - test_proj_path_base = os.path.join(TEST_WORKSPACE, "test_proj_base") - shutil.copytree(project.path(test_project), test_proj_path_base) - - # Copy the test project to the workspace. The tests should - # work only on this test project. - test_proj_path_new = os.path.join(TEST_WORKSPACE, "test_proj_new") - shutil.copytree(project.path(test_project), test_proj_path_new) - - # Copy the test project to the workspace. The tests should - # work only on this test project. - test_proj_path_update = os.path.join(TEST_WORKSPACE, "test_proj_update") - shutil.copytree(project.path(test_project), test_proj_path_update) - - project_info['project_path_base'] = test_proj_path_base - project_info['project_path_new'] = test_proj_path_new - project_info['project_path_update'] = test_proj_path_update - - # Suppress file should be set here if needed by the tests. - suppress_file = None - - # Skip list file should be set here if needed by the tests. - skip_list_file = None - - # Get an environment which should be used by the tests. - test_env = env.test_env(TEST_WORKSPACE) - - # Create a basic CodeChecker config for the tests, this should - # be imported by the tests and they should only depend on these - # configuration options. - codechecker_cfg = { - 'suppress_file': suppress_file, - 'skip_list_file': skip_list_file, - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'trim_path_prefix': TEST_WORKSPACE, - 'analyzers': ['clangsa'] - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'diff_remote' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Base analysis - - altered_file = os.path.join(test_proj_path_base, "call_and_message.cpp") - project.insert_suppression(altered_file) - codechecker_cfg['reportdir'] = os.path.join(test_proj_path_base, - 'reports') - codechecker_cfg['checkers'] = ['-e', 'core.CallAndMessage', - '-d', 'core.NullDereference'] - - ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_base) - if ret: - sys.exit(1) - print('Analyzing base was successful.') - - # Store base results. - codechecker_cfg['reportdir'] = os.path.join(test_proj_path_base, - 'reports') - - test_project_name_base = project_info['name'] + '_' + uuid.uuid4().hex - ret = codechecker.store(codechecker_cfg, test_project_name_base) - if ret: - sys.exit(1) - - # Store with a literal ':' in the name. - ret = codechecker.store(codechecker_cfg, test_project_name_base + ":base") - if ret: - sys.exit(1) - - # New analysis - altered_file = os.path.join(test_proj_path_new, "call_and_message.cpp") - project.insert_suppression(altered_file) - codechecker_cfg['reportdir'] = os.path.join(test_proj_path_new, - 'reports') - codechecker_cfg['checkers'] = ['-d', 'core.CallAndMessage', - '-e', 'core.NullDereference'] - - ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_new) - if ret: - sys.exit(1) - print('Analyzing new was successful.') - - # Store new results. - codechecker_cfg['reportdir'] = os.path.join(test_proj_path_new, - 'reports') - - test_project_name_new = project_info['name'] + '_' + uuid.uuid4().hex - ret = codechecker.store(codechecker_cfg, test_project_name_new) - if ret: - sys.exit(1) - - # Store with a literal ':' in the name. - ret = codechecker.store(codechecker_cfg, test_project_name_new + ":new") - if ret: - sys.exit(1) - - # Analyze multiple times to store results with multiple tags. - codechecker_cfg['reportdir'] = os.path.join(test_proj_path_update, - 'reports') - - test_project_name_update = project_info['name'] + '_' + uuid.uuid4().hex - codechecker_cfg['tag'] = 't1' - codechecker_cfg['checkers'] = ['-d', 'core.CallAndMessage', - '-e', 'core.StackAddressEscape' - ] - - codechecker_cfg['reportdir'] = os.path.join(test_proj_path_update, - 'reports') - - ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_update) - if ret: - sys.exit(1) - - # Store update with t1 tag. - ret = codechecker.store(codechecker_cfg, test_project_name_update) - if ret: - sys.exit(1) - - codechecker_cfg['tag'] = 't2' - codechecker_cfg['checkers'] = ['-e', 'core.CallAndMessage', - '-d', 'core.StackAddressEscape' - ] - ret = codechecker.analyze(codechecker_cfg, test_proj_path_update) - if ret: - sys.exit(1) - - # Store update with t2 tag. - ret = codechecker.store(codechecker_cfg, test_project_name_update) - if ret: - sys.exit(1) - - codechecker_cfg['tag'] = 't3' - ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_update) - if ret: - sys.exit(1) - - # Store update with t3 tag. - ret = codechecker.store(codechecker_cfg, test_project_name_update) - if ret: - sys.exit(1) - - # Order of the test run names matter at comparison! - codechecker_cfg['run_names'] = [test_project_name_base, - test_project_name_new, - test_project_name_update] - - test_config['test_project'] = project_info - test_config['codechecker_cfg'] = codechecker_cfg - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, test_config) - - # Remove report directories which are not used anymore. - shutil.rmtree(test_proj_path_base, ignore_errors=True) - shutil.rmtree(test_proj_path_new, ignore_errors=True) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/diff_remote/test_diff_remote.py b/web/tests/functional/diff_remote/test_diff_remote.py index 405b3dcc39..1fe1ae8902 100644 --- a/web/tests/functional/diff_remote/test_diff_remote.py +++ b/web/tests/functional/diff_remote/test_diff_remote.py @@ -15,7 +15,9 @@ import os import re import shutil +import sys import unittest +import uuid from datetime import datetime, timedelta from codechecker_api.codeCheckerDBAccess_v6.ttypes import CompareData, \ @@ -24,7 +26,9 @@ from codechecker_report_converter.report import InvalidFileContentMsg +from libtest import codechecker from libtest import env +from libtest import project from libtest.codechecker import get_diff_results from libtest.debug_printer import print_run_results from libtest.thrift_client_to_db import get_all_run_results @@ -48,7 +52,209 @@ def str_to_date(date_str): class DiffRemote(unittest.TestCase): - def setUp(self): + def setup_class(self): + """Setup the environment for testing diff_remote.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('diff_remote') + + # Set the TEST_WORKSPACE used by the tests. + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_config = {} + + test_project = 'cpp' + + project_info = project.get_info(test_project) + + # Copy the test project to the workspace. The tests should + # work only on this test project. + test_proj_path_base = os.path.join(TEST_WORKSPACE, "test_proj_base") + shutil.copytree(project.path(test_project), test_proj_path_base) + + # Copy the test project to the workspace. The tests should + # work only on this test project. + test_proj_path_new = os.path.join(TEST_WORKSPACE, "test_proj_new") + shutil.copytree(project.path(test_project), test_proj_path_new) + + # Copy the test project to the workspace. The tests should + # work only on this test project. + test_proj_path_update = \ + os.path.join(TEST_WORKSPACE, "test_proj_update") + shutil.copytree(project.path(test_project), test_proj_path_update) + + project_info['project_path_base'] = test_proj_path_base + project_info['project_path_new'] = test_proj_path_new + project_info['project_path_update'] = test_proj_path_update + + # Suppress file should be set here if needed by the tests. + suppress_file = None + + # Skip list file should be set here if needed by the tests. + skip_list_file = None + + # Get an environment which should be used by the tests. + test_env = env.test_env(TEST_WORKSPACE) + + # Create a basic CodeChecker config for the tests, this should + # be imported by the tests and they should only depend on these + # configuration options. + codechecker_cfg = { + 'suppress_file': suppress_file, + 'skip_list_file': skip_list_file, + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'trim_path_prefix': TEST_WORKSPACE, + 'analyzers': ['clangsa'] + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'diff_remote' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Base analysis + + altered_file = os.path.join(test_proj_path_base, + "call_and_message.cpp") + project.insert_suppression(altered_file) + codechecker_cfg['reportdir'] = os.path.join(test_proj_path_base, + 'reports') + codechecker_cfg['checkers'] = ['-e', 'core.CallAndMessage', + '-d', 'core.NullDereference'] + + ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_base) + if ret: + sys.exit(1) + print('Analyzing base was successful.') + + # Store base results. + codechecker_cfg['reportdir'] = os.path.join(test_proj_path_base, + 'reports') + + test_project_name_base = project_info['name'] + '_' + uuid.uuid4().hex + ret = codechecker.store(codechecker_cfg, test_project_name_base) + if ret: + sys.exit(1) + + # Store with a literal ':' in the name. + ret = codechecker.store(codechecker_cfg, + test_project_name_base + ":base") + if ret: + sys.exit(1) + + # New analysis + altered_file = os.path.join(test_proj_path_new, "call_and_message.cpp") + project.insert_suppression(altered_file) + codechecker_cfg['reportdir'] = os.path.join(test_proj_path_new, + 'reports') + codechecker_cfg['checkers'] = ['-d', 'core.CallAndMessage', + '-e', 'core.NullDereference'] + + ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_new) + if ret: + sys.exit(1) + print('Analyzing new was successful.') + + # Store new results. + codechecker_cfg['reportdir'] = os.path.join(test_proj_path_new, + 'reports') + + test_project_name_new = project_info['name'] + '_' + uuid.uuid4().hex + ret = codechecker.store(codechecker_cfg, test_project_name_new) + if ret: + sys.exit(1) + + # Store with a literal ':' in the name. + ret = codechecker.store(codechecker_cfg, + test_project_name_new + ":new") + if ret: + sys.exit(1) + + # Analyze multiple times to store results with multiple tags. + codechecker_cfg['reportdir'] = os.path.join(test_proj_path_update, + 'reports') + + test_project_name_update = \ + project_info['name'] + '_' + uuid.uuid4().hex + codechecker_cfg['tag'] = 't1' + codechecker_cfg['checkers'] = ['-d', 'core.CallAndMessage', + '-e', 'core.StackAddressEscape' + ] + + codechecker_cfg['reportdir'] = os.path.join(test_proj_path_update, + 'reports') + + ret = codechecker.log_and_analyze(codechecker_cfg, + test_proj_path_update) + if ret: + sys.exit(1) + + # Store update with t1 tag. + ret = codechecker.store(codechecker_cfg, test_project_name_update) + if ret: + sys.exit(1) + + codechecker_cfg['tag'] = 't2' + codechecker_cfg['checkers'] = ['-e', 'core.CallAndMessage', + '-d', 'core.StackAddressEscape' + ] + ret = codechecker.analyze(codechecker_cfg, test_proj_path_update) + if ret: + sys.exit(1) + + # Store update with t2 tag. + ret = codechecker.store(codechecker_cfg, test_project_name_update) + if ret: + sys.exit(1) + + codechecker_cfg['tag'] = 't3' + ret = codechecker.log_and_analyze(codechecker_cfg, + test_proj_path_update) + if ret: + sys.exit(1) + + # Store update with t3 tag. + ret = codechecker.store(codechecker_cfg, test_project_name_update) + if ret: + sys.exit(1) + + # Order of the test run names matter at comparison! + codechecker_cfg['run_names'] = [test_project_name_base, + test_project_name_new, + test_project_name_update] + + test_config['test_project'] = project_info + test_config['codechecker_cfg'] = codechecker_cfg + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, test_config) + + # Remove report directories which are not used anymore. + shutil.rmtree(test_proj_path_base, ignore_errors=True) + shutil.rmtree(test_proj_path_new, ignore_errors=True) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . self.test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/dynamic_results/__init__.py b/web/tests/functional/dynamic_results/__init__.py index 51f116e2a0..f30c492687 100644 --- a/web/tests/functional/dynamic_results/__init__.py +++ b/web/tests/functional/dynamic_results/__init__.py @@ -6,76 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the test package dynamic_results.""" - - -import os -import shutil -import sys - -from libtest import codechecker -from libtest import env -from libtest import project - - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing dynamic_results.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('dynamic_results') - - # Set the TEST_WORKSPACE used by the tests. - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - shutil.copytree( - project.path("dynamic"), TEST_WORKSPACE, dirs_exist_ok=True) - - for plist in os.listdir(TEST_WORKSPACE): - if plist.endswith('.plist'): - with open(os.path.join(TEST_WORKSPACE, plist), 'r+', - encoding='utf-8', - errors='ignore') as plist_file: - content = plist_file.read() - new_content = content.replace("$FILE_PATH$", TEST_WORKSPACE) - plist_file.seek(0) - plist_file.truncate() - plist_file.write(new_content) - - server_access = codechecker.start_or_get_server() - server_access["viewer_product"] = "dynamic_results" - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - codechecker_cfg = { - 'workspace': TEST_WORKSPACE, - 'reportdir': TEST_WORKSPACE, - 'check_env': env.test_env(TEST_WORKSPACE) - } - - codechecker_cfg.update(server_access) - - env.export_test_cfg(TEST_WORKSPACE, { - 'codechecker_cfg': codechecker_cfg - }) - - if codechecker.store(codechecker_cfg, "dynamic_results"): - sys.exit(1) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/dynamic_results/test_dynamic_results.py b/web/tests/functional/dynamic_results/test_dynamic_results.py index 4cbda78fe0..4fa4c7a2b6 100644 --- a/web/tests/functional/dynamic_results/test_dynamic_results.py +++ b/web/tests/functional/dynamic_results/test_dynamic_results.py @@ -13,8 +13,14 @@ """ import os +import shutil +import sys import unittest +from libtest import codechecker +from libtest import env +from libtest import project + from codechecker_api.codeCheckerDBAccess_v6.ttypes import \ Order, Pair, ReportFilter, SortMode, SortType @@ -22,7 +28,65 @@ class DiffRemote(unittest.TestCase): - def setUp(self): + + def setup_class(self): + """Setup the environment for testing dynamic_results.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('dynamic_results') + + # Set the TEST_WORKSPACE used by the tests. + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + shutil.copytree( + project.path("dynamic"), TEST_WORKSPACE, dirs_exist_ok=True) + + for plist in os.listdir(TEST_WORKSPACE): + if plist.endswith('.plist'): + with open(os.path.join(TEST_WORKSPACE, plist), 'r+', + encoding='utf-8', + errors='ignore') as plist_file: + content = plist_file.read() + new_content = content.replace("$FILE_PATH$", + TEST_WORKSPACE) + plist_file.seek(0) + plist_file.truncate() + plist_file.write(new_content) + + server_access = codechecker.start_or_get_server() + server_access["viewer_product"] = "dynamic_results" + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + codechecker_cfg = { + 'workspace': TEST_WORKSPACE, + 'reportdir': TEST_WORKSPACE, + 'check_env': env.test_env(TEST_WORKSPACE) + } + + codechecker_cfg.update(server_access) + + env.export_test_cfg(TEST_WORKSPACE, { + 'codechecker_cfg': codechecker_cfg + }) + + if codechecker.store(codechecker_cfg, "dynamic_results"): + sys.exit(1) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): self.test_workspace = os.environ['TEST_WORKSPACE'] test_class = self.__class__.__name__ diff --git a/web/tests/functional/export_import/__init__.py b/web/tests/functional/export_import/__init__.py index 614f745362..f30c492687 100644 --- a/web/tests/functional/export_import/__init__.py +++ b/web/tests/functional/export_import/__init__.py @@ -6,115 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -""" -Setup for the test package export_import. -""" - - -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import project - - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing export and import.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('export_import') - - # Set the TEST_WORKSPACE used by the tests. - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_config = {} - - test_project = 'cpp' - - project_info = project.get_info(test_project) - - # Copy the test project to the workspace. The tests should - # work only on this test project. - test_proj_path = os.path.join(TEST_WORKSPACE, "test_proj") - shutil.copytree(project.path(test_project), test_proj_path) - - project_info['project_path'] = test_proj_path - - # Generate a unique name for this test run. - test_project_name = project_info['name'] + '_' + uuid.uuid4().hex - - test_config['test_project'] = project_info - - # Suppress file should be set here if needed by the tests. - suppress_file = None - - # Skip list file should be set here if needed by the tests. - skip_list_file = None - - # Get an environment which should be used by the tests. - test_env = env.test_env(TEST_WORKSPACE) - - # Create a basic CodeChecker config for the tests, this should - # be imported by the tests and they should only depend on these - # configuration options. - codechecker_cfg = { - 'suppress_file': suppress_file, - 'skip_list_file': skip_list_file, - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [] - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'export_import' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Clean the test project, if needed by the tests. - ret = project.clean(test_project) - if ret: - sys.exit(ret) - - # Check the test project, if needed by the tests. - ret = codechecker.check_and_store(codechecker_cfg, - test_project_name, - project.path(test_project)) - if ret: - sys.exit(1) - print("Analyzing the test project was successful.") - - # Save the run names in the configuration. - codechecker_cfg['run_names'] = [test_project_name] - - test_config['codechecker_cfg'] = codechecker_cfg - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/export_import/test_export_import.py b/web/tests/functional/export_import/test_export_import.py index de079ca9fd..cef256220a 100644 --- a/web/tests/functional/export_import/test_export_import.py +++ b/web/tests/functional/export_import/test_export_import.py @@ -12,15 +12,20 @@ """ import os +import shutil import unittest import logging import json +import sys +import uuid import tempfile from codechecker_api.codeCheckerDBAccess_v6.ttypes import CommentData, \ ReviewStatus, CommentKind, RunFilter +from libtest import codechecker from libtest import env +from libtest import project from libtest.thrift_client_to_db import get_all_run_results import subprocess @@ -39,7 +44,99 @@ class TestExport(unittest.TestCase): _ccClient = None - def setUp(self): + def setup_class(self): + """Setup the environment for testing export and import.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('export_import') + + # Set the TEST_WORKSPACE used by the tests. + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_config = {} + + test_project = 'cpp' + + project_info = project.get_info(test_project) + + # Copy the test project to the workspace. The tests should + # work only on this test project. + test_proj_path = os.path.join(TEST_WORKSPACE, "test_proj") + shutil.copytree(project.path(test_project), test_proj_path) + + project_info['project_path'] = test_proj_path + + # Generate a unique name for this test run. + test_project_name = project_info['name'] + '_' + uuid.uuid4().hex + + test_config['test_project'] = project_info + + # Suppress file should be set here if needed by the tests. + suppress_file = None + + # Skip list file should be set here if needed by the tests. + skip_list_file = None + + # Get an environment which should be used by the tests. + test_env = env.test_env(TEST_WORKSPACE) + + # Create a basic CodeChecker config for the tests, this should + # be imported by the tests and they should only depend on these + # configuration options. + codechecker_cfg = { + 'suppress_file': suppress_file, + 'skip_list_file': skip_list_file, + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [] + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'export_import' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Clean the test project, if needed by the tests. + ret = project.clean(test_project) + if ret: + sys.exit(ret) + + # Check the test project, if needed by the tests. + ret = codechecker.check_and_store(codechecker_cfg, + test_project_name, + project.path(test_project)) + if ret: + sys.exit(1) + print("Analyzing the test project was successful.") + + # Save the run names in the configuration. + codechecker_cfg['run_names'] = [test_project_name] + + test_config['codechecker_cfg'] = codechecker_cfg + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/extended_report_data/__init__.py b/web/tests/functional/extended_report_data/__init__.py index dcf10b2f09..f30c492687 100644 --- a/web/tests/functional/extended_report_data/__init__.py +++ b/web/tests/functional/extended_report_data/__init__.py @@ -6,93 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the test package extended_report_data.""" - - -import os -import shutil -import sys - -from libtest import codechecker -from libtest import env -from libtest import project - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing detection_status.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('extended_report_data') - - # Set the TEST_WORKSPACE used by the tests. - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_env = env.test_env(TEST_WORKSPACE) - - # Configuration options. - codechecker_cfg = { - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'run_names': [], - 'analyzers': ['clangsa', 'clang-tidy'] - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'extended_report_data' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Copy test projects and replace file path in plist files. - test_projects = ['notes', 'macros'] - for test_project in test_projects: - test_project_path = os.path.join(TEST_WORKSPACE, "test_proj", - test_project) - shutil.copytree(project.path(test_project), test_project_path) - - for test_file in os.listdir(test_project_path): - if test_file.endswith(".plist"): - test_file_path = os.path.join(test_project_path, test_file) - with open(test_file_path, 'r+', - encoding="utf-8", errors="ignore") as plist_file: - content = plist_file.read() - new_content = content.replace("$FILE_PATH$", - test_project_path) - plist_file.seek(0) - plist_file.truncate() - plist_file.write(new_content) - - codechecker_cfg['reportdir'] = test_project_path - - ret = codechecker.store(codechecker_cfg, test_project) - if ret: - sys.exit(1) - print("Analyzing test project was succcessful.") - - codechecker_cfg['run_names'].append(test_project) - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, {'codechecker_cfg': codechecker_cfg}) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - # shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/extended_report_data/test_extended_report_data.py b/web/tests/functional/extended_report_data/test_extended_report_data.py index 98e0a26a5c..0b7dc5f348 100644 --- a/web/tests/functional/extended_report_data/test_extended_report_data.py +++ b/web/tests/functional/extended_report_data/test_extended_report_data.py @@ -12,9 +12,13 @@ import os +import shutil +import sys import unittest +from libtest import codechecker from libtest import env +from libtest import project from codechecker_api.codeCheckerDBAccess_v6.ttypes import ReportFilter, \ RunFilter, ExtendedReportDataType @@ -22,7 +26,82 @@ class TestExtendedReportData(unittest.TestCase): - def setUp(self): + def setup_class(self): + """Setup the environment for testing detection_status.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('extended_report_data') + + # Set the TEST_WORKSPACE used by the tests. + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_env = env.test_env(TEST_WORKSPACE) + + # Configuration options. + codechecker_cfg = { + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'run_names': [], + 'analyzers': ['clangsa', 'clang-tidy'] + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'extended_report_data' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Copy test projects and replace file path in plist files. + test_projects = ['notes', 'macros'] + for test_project in test_projects: + test_project_path = os.path.join(TEST_WORKSPACE, "test_proj", + test_project) + shutil.copytree(project.path(test_project), test_project_path) + + for test_file in os.listdir(test_project_path): + if test_file.endswith(".plist"): + test_file_path = os.path.join(test_project_path, test_file) + with open(test_file_path, 'r+', + encoding="utf-8", errors="ignore") as plist_file: + content = plist_file.read() + new_content = content.replace("$FILE_PATH$", + test_project_path) + plist_file.seek(0) + plist_file.truncate() + plist_file.write(new_content) + + codechecker_cfg['reportdir'] = test_project_path + + ret = codechecker.store(codechecker_cfg, test_project) + if ret: + sys.exit(1) + print("Analyzing test project was succcessful.") + + codechecker_cfg['run_names'].append(test_project) + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, + {'codechecker_cfg': codechecker_cfg}) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): self._test_workspace = os.environ.get('TEST_WORKSPACE') test_class = self.__class__.__name__ diff --git a/web/tests/functional/func_template/template__init__.py b/web/tests/functional/func_template/template__init__.py deleted file mode 100644 index 2fefaecae0..0000000000 --- a/web/tests/functional/func_template/template__init__.py +++ /dev/null @@ -1,130 +0,0 @@ -# coding=utf-8 -# ------------------------------------------------------------------------- -# -# Part of the CodeChecker project, under the Apache License v2.0 with -# LLVM Exceptions. See LICENSE for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -# ------------------------------------------------------------------------- - -"""Setup for the test package $TEST_NAME$. - -WARNING!!! -This is a generated test skeleton remove the parts not required by the tests. -WARNING!!! - -""" - - -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import project - - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing $TEST_NAME$.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('$TEST_NAME$') - - # Set the TEST_WORKSPACE used by the tests. - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - # Get the clang version which is used for testing. - # Important because different clang releases might - # find different errors. - clang_version = env.clang_to_test() - - test_config = {} - - test_project = 'cpp' - - project_info = project.get_info(test_project) - - # Copy the test project to the workspace. The tests should - # work only on this test project. - test_proj_path = os.path.join(TEST_WORKSPACE, "test_proj") - shutil.copytree(project.path(test_project), test_proj_path) - - project_info['project_path'] = test_proj_path - - # Generate a unique name for this test run. - test_project_name = project_info['name'] + '_' + uuid.uuid4().hex - - test_config['test_project'] = project_info - - # Suppress file should be set here if needed by the tests. - suppress_file = None - - # Skip list file should be set here if needed by the tests. - skip_list_file = None - - # Get an environment which should be used by the tests. - test_env = env.test_env(TEST_WORKSPACE) - - # Create a basic CodeChecker config for the tests, this should - # be imported by the tests and they should only depend on these - # configuration options. - codechecker_cfg = { - 'suppress_file': suppress_file, - 'skip_list_file': skip_list_file, - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'analyzers': ['clangsa', 'clang-tidy'] - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = '$TEST_NAME$' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Clean the test project, if needed by the tests. - ret = project.clean(test_project) - if ret: - sys.exit(ret) - - # Check the test project, if needed by the tests. - ret = codechecker.check_and_store(codechecker_cfg, - test_project_name, - project.path(test_project)) - if ret: - sys.exit(1) - print("Analyzing the test project was successful.") - - # Save the run names in the configuration. - codechecker_cfg['run_names'] = [test_project_name] - - test_config['codechecker_cfg'] = codechecker_cfg - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/func_template/template_test.py b/web/tests/functional/func_template/template_test.py deleted file mode 100644 index a3616b9a1f..0000000000 --- a/web/tests/functional/func_template/template_test.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# ------------------------------------------------------------------------- -# -# Part of the CodeChecker project, under the Apache License v2.0 with -# LLVM Exceptions. See LICENSE for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -# ------------------------------------------------------------------------- - -""" $TEST_NAME$ function test. - -WARNING!!! -This is a generated test skeleton remove the parts not required by the tests. -WARNING!!! - -""" - - -import os -import unittest - -from libtest import env - - -class TestSkeleton(unittest.TestCase): - - _ccClient = None - - def setUp(self): - """ - WARNING!!! - This is an example how to get the configurations needed by the tests. - WARNING!!! - """ - - # TEST_WORKSPACE is automatically set by test package __init__.py . - test_workspace = os.environ['TEST_WORKSPACE'] - - test_class = self.__class__.__name__ - print('Running ' + test_class + ' tests in ' + test_workspace) - - # Get the test configuration from the prepared int the test workspace. - test_cfg = env.import_test_cfg(test_workspace) - - # Get the test project configuration from the prepared test workspace. - self._testproject_data = env.setup_test_proj_cfg(test_workspace) - self.assertIsNotNone(self._testproject_data) - - # Setup a viewer client to test viewer API calls. - self._cc_client = env.setup_viewer_client(test_workspace) - self.assertIsNotNone(self._cc_client) - - # Get the CodeChecker cmd if needed for the tests. - self._codechecker_cmd = env.codechecker_cmd() - - # Get the run names which belong to this test. - run_names = env.get_run_names(test_workspace) - - runs = self._cc_client.getRunData(None, None, 0, None) - test_runs = [run for run in runs if run.name in run_names] - - def test_skel(self): - """ - Test some feature. - """ - pass diff --git a/web/tests/functional/instance_manager/__init__.py b/web/tests/functional/instance_manager/__init__.py index 1e0c3dbba1..f30c492687 100644 --- a/web/tests/functional/instance_manager/__init__.py +++ b/web/tests/functional/instance_manager/__init__.py @@ -6,89 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the package tests.""" - - -import multiprocessing -import os -import shutil -import subprocess -import time - -from libtest import codechecker -from libtest import env - -# Stopping events for CodeChecker servers. -EVENT_1 = multiprocessing.Event() -EVENT_2 = multiprocessing.Event() - -# Test workspace initialized at setup for authentication tests. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for the tests then start the server.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('instances') - - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_config = {} - - test_env = env.test_env(TEST_WORKSPACE) - - # Setup environment variables for the test cases. - host_port_cfg = {'viewer_host': 'localhost', - 'viewer_port': env.get_free_port()} - - codechecker_cfg = { - 'workspace': TEST_WORKSPACE, - 'check_env': test_env, - 'run_names': [], - 'checkers': [], - 'analyzers': ['clangsa', 'clang-tidy'] - } - codechecker_cfg.update(host_port_cfg) - test_config['codechecker_1'] = codechecker_cfg - - # We need a second server - codechecker_cfg = { - 'workspace': TEST_WORKSPACE, - 'check_env': test_env, - 'run_names': [], - 'checkers': [], - 'analyzers': ['clangsa', 'clang-tidy'] - } - host_port_cfg = {'viewer_host': 'localhost', - 'viewer_port': env.get_free_port()} - - if host_port_cfg['viewer_port'] == \ - test_config['codechecker_1']['viewer_port']: - host_port_cfg['viewer_port'] = int(host_port_cfg['viewer_port']) + 1 - - codechecker_cfg.update(host_port_cfg) - test_config['codechecker_2'] = codechecker_cfg - - # Export configuration for the tests. - env.export_test_cfg(TEST_WORKSPACE, test_config) - - # Wait for previous test instances to terminate properly and - # clean the instance file in the user's home directory. - time.sleep(5) - - -def teardown_package(): - """Stop the CodeChecker server.""" - - # Let the remaining CodeChecker servers die. - EVENT_1.set() - EVENT_2.set() - - # TODO If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/instance_manager/test_instances.py b/web/tests/functional/instance_manager/test_instances.py index 0d8cdefeca..39adfc887a 100644 --- a/web/tests/functional/instance_manager/test_instances.py +++ b/web/tests/functional/instance_manager/test_instances.py @@ -11,7 +11,9 @@ """ +import multiprocessing import os +import shutil import subprocess import time import unittest @@ -20,7 +22,10 @@ from libtest import env from libtest.codechecker import start_server -from . import EVENT_1, EVENT_2 + +# Stopping events for CodeChecker servers. +EVENT_1 = multiprocessing.Event() +EVENT_2 = multiprocessing.Event() class TestInstances(unittest.TestCase): @@ -28,7 +33,73 @@ class TestInstances(unittest.TestCase): Server instance manager tests. """ - def setUp(self): + def setup_class(self): + """Setup the environment for the tests then start the server.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('instances') + + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_config = {} + + test_env = env.test_env(TEST_WORKSPACE) + + # Setup environment variables for the test cases. + host_port_cfg = {'viewer_host': 'localhost', + 'viewer_port': env.get_free_port()} + + codechecker_cfg = { + 'workspace': TEST_WORKSPACE, + 'check_env': test_env, + 'run_names': [], + 'checkers': [], + 'analyzers': ['clangsa', 'clang-tidy'] + } + codechecker_cfg.update(host_port_cfg) + test_config['codechecker_1'] = codechecker_cfg + + # We need a second server + codechecker_cfg = { + 'workspace': TEST_WORKSPACE, + 'check_env': test_env, + 'run_names': [], + 'checkers': [], + 'analyzers': ['clangsa', 'clang-tidy'] + } + host_port_cfg = {'viewer_host': 'localhost', + 'viewer_port': env.get_free_port()} + + if host_port_cfg['viewer_port'] == \ + test_config['codechecker_1']['viewer_port']: + host_port_cfg['viewer_port'] = \ + int(host_port_cfg['viewer_port']) + 1 + + codechecker_cfg.update(host_port_cfg) + test_config['codechecker_2'] = codechecker_cfg + + # Export configuration for the tests. + env.export_test_cfg(TEST_WORKSPACE, test_config) + + # Wait for previous test instances to terminate properly and + # clean the instance file in the user's home directory. + time.sleep(5) + + def teardown_class(self): + """Stop the CodeChecker server.""" + + # Let the remaining CodeChecker servers die. + EVENT_1.set() + EVENT_2.set() + + # TODO If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # Get the test workspace used to tests. self._test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/products/__init__.py b/web/tests/functional/products/__init__.py index 5b96c016fe..9f0643b240 100644 --- a/web/tests/functional/products/__init__.py +++ b/web/tests/functional/products/__init__.py @@ -26,11 +26,11 @@ TEST_WORKSPACE = None -def setup_package(): +def setup_class_common(workspace_name): """Setup the environment for testing products.""" global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('producttest') + TEST_WORKSPACE = env.get_workspace(workspace_name) # Set the TEST_WORKSPACE used by the tests. os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE @@ -70,7 +70,7 @@ def setup_package(): # details. print("This test uses a CodeChecker server... connecting...") server_access = codechecker.start_or_get_server(auth_required=True) - server_access['viewer_product'] = 'producttest' + server_access['viewer_product'] = workspace_name codechecker.add_test_package_product(server_access, TEST_WORKSPACE) # Extend the checker configuration with the server access. @@ -98,7 +98,7 @@ def setup_package(): env.export_test_cfg(TEST_WORKSPACE, test_config) -def teardown_package(): +def teardown_class_common(): """Clean up after the test.""" # TODO: If environment variable is set keep the workspace diff --git a/web/tests/functional/products/test_config_db_share.py b/web/tests/functional/products/test_config_db_share.py index 00b72e7357..c6f72ea93e 100644 --- a/web/tests/functional/products/test_config_db_share.py +++ b/web/tests/functional/products/test_config_db_share.py @@ -13,6 +13,8 @@ """ +from . import setup_class_common, teardown_class_common + from copy import deepcopy import multiprocessing import os @@ -35,7 +37,14 @@ class TestProductConfigShare(unittest.TestCase): - def setUp(self): + def setup_class(self): + (self).product_name = "config_db_share" + setup_class_common("config_db_share") + + def teardown_class(self): + teardown_class_common() + + def setup_method(self, method): """ Set up the environment and the test module's configuration from the package. @@ -132,8 +141,8 @@ def test_read_product_from_another_server(self): # Check if the main server's product is visible on the second server. self.assertEqual( - self._pr_client_2.getProducts('producttest', None)[0].endpoint, - 'producttest', + self._pr_client_2.getProducts(self.product_name, None)[0].endpoint, + self.product_name, "Main server's product was not loaded by the secondary server.") def create_test_product(product_name, product_endpoint): @@ -223,7 +232,7 @@ def create_test_product(product_name, product_endpoint): "the product missing should've resulted in " "an error.") - def tearDown(self): + def teardown_method(self, method): """ Clean the environment after running this test module """ diff --git a/web/tests/functional/products/test_products.py b/web/tests/functional/products/test_products.py index ab1cb735f5..ce5b2ab114 100644 --- a/web/tests/functional/products/test_products.py +++ b/web/tests/functional/products/test_products.py @@ -26,10 +26,18 @@ from libtest import env +from . import setup_class_common, teardown_class_common + class TestProducts(unittest.TestCase): - def setUp(self): + def setup_class(self): + setup_class_common("products") + + def teardown_class(self): + teardown_class_common() + + def setup_method(self, method): """ """ diff --git a/web/tests/functional/report_viewer_api/__init__.py b/web/tests/functional/report_viewer_api/__init__.py index 8e322bda7d..fb22cc68a0 100644 --- a/web/tests/functional/report_viewer_api/__init__.py +++ b/web/tests/functional/report_viewer_api/__init__.py @@ -23,11 +23,11 @@ TEST_WORKSPACE = None -def setup_package(): +def setup_class_common(workspace_name): """Setup the environment for the tests.""" global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('report_viewer_api') + TEST_WORKSPACE = env.get_workspace(workspace_name) os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE @@ -70,7 +70,7 @@ def setup_package(): # details. print("This test uses a CodeChecker server... connecting...") server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'report_viewer_api' + server_access['viewer_product'] = workspace_name codechecker.add_test_package_product(server_access, TEST_WORKSPACE) # Extend the checker configuration with the server access. @@ -131,7 +131,7 @@ def setup_package(): env.export_test_cfg(TEST_WORKSPACE, test_config) -def teardown_package(): +def teardown_class_common(): """Clean up after the test.""" # TODO: if environment variable is set keep the workspace diff --git a/web/tests/functional/report_viewer_api/test_get_lines_in_file.py b/web/tests/functional/report_viewer_api/test_get_lines_in_file.py index 35b73a1507..754746b0f4 100644 --- a/web/tests/functional/report_viewer_api/test_get_lines_in_file.py +++ b/web/tests/functional/report_viewer_api/test_get_lines_in_file.py @@ -22,12 +22,20 @@ from codechecker_web.shared import convert +from . import setup_class_common, teardown_class_common + class TestGetLinesInFile(unittest.TestCase): _ccClient = None - def setUp(self): + def setup_class(self): + setup_class_common("get_lines_in_file") + + def teardown_class(self): + teardown_class_common() + + def setup_method(self, method): test_workspace = os.environ['TEST_WORKSPACE'] test_class = self.__class__.__name__ diff --git a/web/tests/functional/report_viewer_api/test_get_run_results.py b/web/tests/functional/report_viewer_api/test_get_run_results.py index 2d2e766068..79b5f7feda 100644 --- a/web/tests/functional/report_viewer_api/test_get_run_results.py +++ b/web/tests/functional/report_viewer_api/test_get_run_results.py @@ -27,12 +27,20 @@ from libtest.result_compare import find_all from libtest import env +from . import setup_class_common, teardown_class_common + class RunResults(unittest.TestCase): _ccClient = None - def setUp(self): + def setup_class(self): + setup_class_common("get_run_results") + + def teardown_class(self): + teardown_class_common() + + def setup_method(self, method): test_workspace = os.environ['TEST_WORKSPACE'] test_class = self.__class__.__name__ diff --git a/web/tests/functional/report_viewer_api/test_hash_clash.py b/web/tests/functional/report_viewer_api/test_hash_clash.py index 3c5a679048..a2309b2411 100644 --- a/web/tests/functional/report_viewer_api/test_hash_clash.py +++ b/web/tests/functional/report_viewer_api/test_hash_clash.py @@ -26,6 +26,8 @@ from codechecker_api.codeCheckerDBAccess_v6.ttypes import Encoding, \ RunFilter, ReportFilter +from . import setup_class_common, teardown_class_common + def _generate_content(cols, lines): """Generates a random file content string.""" @@ -48,7 +50,13 @@ def _replace_path(file_path, path): class HashClash(unittest.TestCase): """Unit test for testing hash clash handling.""" - def setUp(self): + def setup_class(self): + setup_class_common("hash_clash") + + def teardown_class(self): + teardown_class_common() + + def setup_method(self, method): """ Not much setup is needed. Runs and results are automatically generated. @@ -83,7 +91,7 @@ def setUp(self): self._run_name = 'test_hash_clash_' + uuid4().hex codechecker.store(self._codechecker_cfg, self._run_name) - def tearDown(self): + def teardown_method(self, method): """ Remove the run which was stored by this test case. """ diff --git a/web/tests/functional/report_viewer_api/test_remove_run_results.py b/web/tests/functional/report_viewer_api/test_remove_run_results.py index c8af95d355..15b13d78fa 100644 --- a/web/tests/functional/report_viewer_api/test_remove_run_results.py +++ b/web/tests/functional/report_viewer_api/test_remove_run_results.py @@ -23,13 +23,21 @@ from libtest import env from libtest import project +from . import setup_class_common, teardown_class_common + class RemoveRunResults(unittest.TestCase): """ Tests for removing run results. """ _ccClient = None - def setUp(self): + def setup_class(self): + setup_class_common("remove_run_results") + + def teardown_class(self): + teardown_class_common() + + def setup_method(self, method): test_workspace = os.environ['TEST_WORKSPACE'] test_class = self.__class__.__name__ diff --git a/web/tests/functional/report_viewer_api/test_report_counting.py b/web/tests/functional/report_viewer_api/test_report_counting.py index ff92b1daf2..475fe5db23 100644 --- a/web/tests/functional/report_viewer_api/test_report_counting.py +++ b/web/tests/functional/report_viewer_api/test_report_counting.py @@ -21,6 +21,8 @@ from libtest import env +from . import setup_class_common, teardown_class_common + def get_severity_level(name): """ Convert severity name to value. """ @@ -36,7 +38,13 @@ class TestReportFilter(unittest.TestCase): _ccClient = None - def setUp(self): + def setup_class(self): + setup_class_common("report_counting") + + def teardown_class(self): + teardown_class_common() + + def setup_method(self, method): test_workspace = os.environ['TEST_WORKSPACE'] self.maxDiff = None diff --git a/web/tests/functional/report_viewer_api/test_report_filter.py b/web/tests/functional/report_viewer_api/test_report_filter.py index 30096e2413..2c5ac10a38 100644 --- a/web/tests/functional/report_viewer_api/test_report_filter.py +++ b/web/tests/functional/report_viewer_api/test_report_filter.py @@ -22,6 +22,8 @@ from libtest import env +from . import setup_class_common, teardown_class_common + def get_severity_level(name): """ Convert severity name to value. """ @@ -37,7 +39,13 @@ class TestReportFilter(unittest.TestCase): _ccClient = None - def setUp(self): + def setup_class(self): + setup_class_common("report_filter") + + def teardown_class(self): + teardown_class_common() + + def setup_method(self, method): test_workspace = os.environ['TEST_WORKSPACE'] test_class = self.__class__.__name__ diff --git a/web/tests/functional/report_viewer_api/test_run_data.py b/web/tests/functional/report_viewer_api/test_run_data.py index 8a8cf95936..b258e021f8 100644 --- a/web/tests/functional/report_viewer_api/test_run_data.py +++ b/web/tests/functional/report_viewer_api/test_run_data.py @@ -20,12 +20,20 @@ from codechecker_api.codeCheckerDBAccess_v6.ttypes import DetectionStatus, \ Order, ReportFilter, RunFilter, RunSortMode, RunSortType +from . import setup_class_common, teardown_class_common + class TestRunData(unittest.TestCase): _ccClient = None - def setUp(self): + def setup_class(self): + setup_class_common("run_data") + + def teardown_class(self): + teardown_class_common() + + def setup_method(self, method): test_workspace = os.environ['TEST_WORKSPACE'] test_class = self.__class__.__name__ diff --git a/web/tests/functional/review_status/__init__.py b/web/tests/functional/review_status/__init__.py index 1142828a69..f30c492687 100644 --- a/web/tests/functional/review_status/__init__.py +++ b/web/tests/functional/review_status/__init__.py @@ -6,97 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the test package review_status.""" - - -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import project - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing review_status.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('review_status') - - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - dir_path = os.path.dirname(os.path.realpath(__file__)) - shutil.copytree(os.path.join(dir_path, 'review_status_files'), - os.path.join(TEST_WORKSPACE, 'review_status_files')) - - test_project = 'single_bug' - - test_config = {} - - project_info = project.get_info(test_project) - - test_config['test_project'] = project_info - - skip_list_file = None - - test_env = env.test_env(TEST_WORKSPACE) - - codechecker_cfg = { - 'suppress_file': None, - 'skip_list_file': skip_list_file, - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'analyzers': ['clangsa', 'clang-tidy'] - } - - ret = project.clean(test_project, test_env) - if ret: - sys.exit(ret) - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'review_status' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - test_project_name = project_info['name'] + '_' + uuid.uuid4().hex - - ret = codechecker.check_and_store(codechecker_cfg, - test_project_name, - project.path(test_project)) - - if ret: - sys.exit(1) - print("Analyzing the test project was successful.") - - codechecker_cfg['run_names'] = [test_project_name] - - test_config['codechecker_cfg'] = codechecker_cfg - - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/review_status/test_review_status.py b/web/tests/functional/review_status/test_review_status.py index 30f8c7f7ca..6d51132b8f 100644 --- a/web/tests/functional/review_status/test_review_status.py +++ b/web/tests/functional/review_status/test_review_status.py @@ -13,8 +13,11 @@ import datetime import logging import os +import shutil +import sys import time import unittest +import uuid from typing import Callable, List @@ -31,7 +34,84 @@ class TestReviewStatus(unittest.TestCase): _ccClient = None - def setUp(self): + def setup_class(self): + """Setup the environment for testing review_status.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('review_status') + + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + dir_path = os.path.dirname(os.path.realpath(__file__)) + shutil.copytree(os.path.join(dir_path, 'review_status_files'), + os.path.join(TEST_WORKSPACE, 'review_status_files')) + + test_project = 'single_bug' + + test_config = {} + + project_info = project.get_info(test_project) + + test_config['test_project'] = project_info + + skip_list_file = None + + test_env = env.test_env(TEST_WORKSPACE) + + codechecker_cfg = { + 'suppress_file': None, + 'skip_list_file': skip_list_file, + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'analyzers': ['clangsa', 'clang-tidy'] + } + + ret = project.clean(test_project, test_env) + if ret: + sys.exit(ret) + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'review_status' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + test_project_name = project_info['name'] + '_' + uuid.uuid4().hex + + ret = codechecker.check_and_store(codechecker_cfg, + test_project_name, + project.path(test_project)) + + if ret: + sys.exit(1) + print("Analyzing the test project was successful.") + + codechecker_cfg['run_names'] = [test_project_name] + + test_config['codechecker_cfg'] = codechecker_cfg + + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): self.test_workspace = os.environ['TEST_WORKSPACE'] test_class = self.__class__.__name__ @@ -57,7 +137,7 @@ def setUp(self): 'with the given name configured at the test init.') self._runid = test_runs[0].runId - def tearDown(self): + def teardown_method(self, method): """ Remove all review status rules after each test cases. """ self.__remove_all_rules() diff --git a/web/tests/functional/run_tag/__init__.py b/web/tests/functional/run_tag/__init__.py index 0b699de1e8..f30c492687 100644 --- a/web/tests/functional/run_tag/__init__.py +++ b/web/tests/functional/run_tag/__init__.py @@ -6,65 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the test package detection_status.""" - - -import os -import shutil - -from libtest import codechecker -from libtest import env - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing detection_status.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('run_tag') - - # Set the TEST_WORKSPACE used by the tests. - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - # Configuration options. - codechecker_cfg = { - 'suppress_file': None, - 'skip_list_file': None, - 'check_env': env.test_env(TEST_WORKSPACE), - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), - 'test_project': 'test_run_tag', - 'analyzers': ['clangsa', 'clang-tidy'] - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'run_tag' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, {'codechecker_cfg': codechecker_cfg}) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/run_tag/test_run_tag.py b/web/tests/functional/run_tag/test_run_tag.py index 18bcbbf3d1..1bad0e8549 100644 --- a/web/tests/functional/run_tag/test_run_tag.py +++ b/web/tests/functional/run_tag/test_run_tag.py @@ -11,6 +11,7 @@ import json import os +import shutil import unittest from codechecker_api.codeCheckerDBAccess_v6.ttypes import Order, \ @@ -22,7 +23,56 @@ class TestRunTag(unittest.TestCase): - def setUp(self): + def setup_class(self): + """Setup the environment for testing detection_status.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('run_tag') + + # Set the TEST_WORKSPACE used by the tests. + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + # Configuration options. + codechecker_cfg = { + 'suppress_file': None, + 'skip_list_file': None, + 'check_env': env.test_env(TEST_WORKSPACE), + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), + 'test_project': 'test_run_tag', + 'analyzers': ['clangsa', 'clang-tidy'] + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'run_tag' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, + {'codechecker_cfg': codechecker_cfg}) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . self.test_workspace = os.environ['TEST_WORKSPACE'] @@ -81,7 +131,7 @@ def setUp(self): }"""] self.tags = ['v1.0', 'v1.1', 'v1.2'] - def tearDown(self): + def teardown_method(self, method): """Restore environment after tests have ran.""" os.chdir(self.__old_pwd) diff --git a/web/tests/functional/server_configuration/__init__.py b/web/tests/functional/server_configuration/__init__.py index f336538813..f30c492687 100644 --- a/web/tests/functional/server_configuration/__init__.py +++ b/web/tests/functional/server_configuration/__init__.py @@ -6,75 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -""" -Setup for the test package server_configuration. -""" - - -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import project - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing server_configuration.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('server_configuration') - - # Set the TEST_WORKSPACE used by the tests. - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_config = {} - - # Get an environment which should be used by the tests. - test_env = env.test_env(TEST_WORKSPACE) - - codechecker_cfg = { - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'analyzers': ['clangsa', 'clang-tidy'] - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server(auth_required=True) - server_access['viewer_product'] = 'server_configuration' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Save the run names in the configuration. - codechecker_cfg['run_names'] = [] - - test_config['codechecker_cfg'] = codechecker_cfg - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/server_configuration/test_server_configuration.py b/web/tests/functional/server_configuration/test_server_configuration.py index 25b26d9444..d3e801b50d 100644 --- a/web/tests/functional/server_configuration/test_server_configuration.py +++ b/web/tests/functional/server_configuration/test_server_configuration.py @@ -12,6 +12,7 @@ import os +import shutil import unittest from codechecker_api_shared.ttypes import Permission @@ -19,6 +20,7 @@ from codechecker_web.shared import convert +from libtest import codechecker from libtest import env @@ -26,7 +28,60 @@ class ConfigTests(unittest.TestCase): _ccClient = None - def setUp(self): + def setup_class(self): + """Setup the environment for testing server_configuration.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('server_configuration') + + # Set the TEST_WORKSPACE used by the tests. + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_config = {} + + # Get an environment which should be used by the tests. + test_env = env.test_env(TEST_WORKSPACE) + + codechecker_cfg = { + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'analyzers': ['clangsa', 'clang-tidy'] + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server(auth_required=True) + server_access['viewer_product'] = 'server_configuration' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Save the run names in the configuration. + codechecker_cfg['run_names'] = [] + + test_config['codechecker_cfg'] = codechecker_cfg + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): """ Setup Configuration for tests. """ diff --git a/web/tests/functional/skip/__init__.py b/web/tests/functional/skip/__init__.py index ad4d770b5f..f30c492687 100644 --- a/web/tests/functional/skip/__init__.py +++ b/web/tests/functional/skip/__init__.py @@ -6,142 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the package tests.""" - - -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import project - - -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for the tests.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('skip') - - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_project = 'cpp' - - test_config = {} - - project_info = project.get_info(test_project) - - test_config['test_project'] = project_info - - suppress_file = None - - # Generate skip list file for the tests. - skip_list_file = os.path.join(TEST_WORKSPACE, 'skip_file') - if os.path.isfile(skip_list_file): - os.remove(skip_list_file) - _generate_skip_list_file(skip_list_file) - - test_env = env.test_env(TEST_WORKSPACE) - - codechecker_cfg = { - 'suppress_file': suppress_file, - 'skip_file': skip_list_file, - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'analyzers': ['clangsa'] - } - - ret = project.clean(test_project, test_env) - if ret: - sys.exit(ret) - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'skip' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - test_project_name = project_info['name'] + '_' + uuid.uuid4().hex - - skip_file = codechecker_cfg.pop('skip_file') - - output_dir = codechecker_cfg['reportdir'] \ - if 'reportdir' in codechecker_cfg \ - else os.path.join(codechecker_cfg['workspace'], 'reports') - - codechecker_cfg['reportdir'] = output_dir - - # Analyze without skip. - ret = codechecker.log_and_analyze(codechecker_cfg, - project.path(test_project)) - if ret: - print("Analyzing the test project without a skip file failed.") - sys.exit(1) - - codechecker_cfg['skip_file'] = skip_file - - # Analyze with skip. - ret = codechecker.log_and_analyze(codechecker_cfg, - project.path(test_project)) - - if ret: - print("Analyzing the test project with a skip file failed.") - sys.exit(1) - - ret = codechecker.store(codechecker_cfg, - test_project_name) - if ret: - print("Storing the results failed.") - sys.exit(1) - - codechecker_cfg['run_names'] = [test_project_name] - - test_config['codechecker_cfg'] = codechecker_cfg - - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) - - -def _generate_skip_list_file(skip_list_file): - """ - Generate skip list file. - file_to_be_skipped.cpp is a valid file in the cpp project - with bugs in it. - """ - skip_list_content = ["-*randtable.c", "-*blocksort.c", "-*huffman.c", - "-*decompress.c", "-*crctable.c", - "-*file_to_be_skipped.cpp", "-*path_end.h", - "-*skip.h"] - print('Skip list file content: ' + skip_list_file) - print('\n'.join(skip_list_content)) - - s_file = open(skip_list_file, 'w', encoding="utf-8", errors="ignore") - for k in skip_list_content: - s_file.write(k + '\n') - - s_file.close() diff --git a/web/tests/functional/skip/test_skip.py b/web/tests/functional/skip/test_skip.py index 526aa74c99..f4e2436d9f 100644 --- a/web/tests/functional/skip/test_skip.py +++ b/web/tests/functional/skip/test_skip.py @@ -11,20 +11,149 @@ import logging import os +import shutil +import sys import plistlib import unittest +import uuid from libtest.debug_printer import print_run_results from libtest.thrift_client_to_db import get_all_run_results from libtest.result_compare import find_all -from libtest import codechecker, env + + +from libtest import codechecker +from libtest import env +from libtest import project + + +def _generate_skip_list_file(skip_list_file): + """ + Generate skip list file. + file_to_be_skipped.cpp is a valid file in the cpp project + with bugs in it. + """ + skip_list_content = ["-*randtable.c", "-*blocksort.c", "-*huffman.c", + "-*decompress.c", "-*crctable.c", + "-*file_to_be_skipped.cpp", "-*path_end.h", + "-*skip.h"] + print('Skip list file content: ' + skip_list_file) + print('\n'.join(skip_list_content)) + + s_file = open(skip_list_file, 'w', encoding="utf-8", errors="ignore") + for k in skip_list_content: + s_file.write(k + '\n') + + s_file.close() class TestSkip(unittest.TestCase): _ccClient = None - def setUp(self): + def setup_class(self): + """Setup the environment for the tests.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('skip') + + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_project = 'cpp' + + test_config = {} + + project_info = project.get_info(test_project) + + test_config['test_project'] = project_info + + suppress_file = None + + # Generate skip list file for the tests. + skip_list_file = os.path.join(TEST_WORKSPACE, 'skip_file') + if os.path.isfile(skip_list_file): + os.remove(skip_list_file) + _generate_skip_list_file(skip_list_file) + + test_env = env.test_env(TEST_WORKSPACE) + + codechecker_cfg = { + 'suppress_file': suppress_file, + 'skip_file': skip_list_file, + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'analyzers': ['clangsa'] + } + + ret = project.clean(test_project, test_env) + if ret: + sys.exit(ret) + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'skip' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + test_project_name = project_info['name'] + '_' + uuid.uuid4().hex + + skip_file = codechecker_cfg.pop('skip_file') + + output_dir = codechecker_cfg['reportdir'] \ + if 'reportdir' in codechecker_cfg \ + else os.path.join(codechecker_cfg['workspace'], 'reports') + + codechecker_cfg['reportdir'] = output_dir + + # Analyze without skip. + ret = codechecker.log_and_analyze(codechecker_cfg, + project.path(test_project)) + if ret: + print("Analyzing the test project without a skip file failed.") + sys.exit(1) + + codechecker_cfg['skip_file'] = skip_file + + # Analyze with skip. + ret = codechecker.log_and_analyze(codechecker_cfg, + project.path(test_project)) + + if ret: + print("Analyzing the test project with a skip file failed.") + sys.exit(1) + + ret = codechecker.store(codechecker_cfg, + test_project_name) + if ret: + print("Storing the results failed.") + sys.exit(1) + + codechecker_cfg['run_names'] = [test_project_name] + + test_config['codechecker_cfg'] = codechecker_cfg + + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py self.test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/source_change/__init__.py b/web/tests/functional/source_change/__init__.py index 019edd0653..f30c492687 100644 --- a/web/tests/functional/source_change/__init__.py +++ b/web/tests/functional/source_change/__init__.py @@ -6,99 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the package tests.""" - - -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import project - - -# Test workspace used for source change tests -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing source_change.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('source_change') - - # Set the TEST_WORKSPACE used by the tests. - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_config = {} - - test_project = 'cpp' - - project_info = project.get_info(test_project) - - # Copy the test project to the workspace. The tests should - # work only on this test project. - test_proj_path = os.path.join(TEST_WORKSPACE, "test_proj") - shutil.copytree(project.path(test_project), test_proj_path) - - project_info['project_path'] = test_proj_path - - # Generate a unique name for this test run. - test_project_name = project_info['name'] + '_' + uuid.uuid4().hex - - test_config['test_project'] = project_info - - # Get an environment which should be used by the tests. - test_env = env.test_env(TEST_WORKSPACE) - - # Create a basic CodeChecker config for the tests, this should - # be imported by the tests and they should only depend on these - # configuration options. - codechecker_cfg = { - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), - 'checkers': [], - 'analyzers': ['clangsa', 'clang-tidy'] - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'source_change' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Clean the test project, if needed by the tests. - ret = project.clean(test_project) - if ret: - sys.exit(ret) - - # Save the run names in the configuration. - codechecker_cfg['run_names'] = [test_project_name] - - test_config['codechecker_cfg'] = codechecker_cfg - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/source_change/test_source_change.py b/web/tests/functional/source_change/test_source_change.py index 530c239c65..3e6ce8a4a0 100644 --- a/web/tests/functional/source_change/test_source_change.py +++ b/web/tests/functional/source_change/test_source_change.py @@ -14,10 +14,14 @@ import os import time +import shutil +import sys import unittest +import uuid from libtest import codechecker from libtest import env +from libtest import project def touch(fname, times=None): @@ -32,7 +36,85 @@ class TestSkeleton(unittest.TestCase): _ccClient = None - def setUp(self): + def setup_class(self): + """Setup the environment for testing source_change.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('source_change') + + # Set the TEST_WORKSPACE used by the tests. + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_config = {} + + test_project = 'cpp' + + project_info = project.get_info(test_project) + + # Copy the test project to the workspace. The tests should + # work only on this test project. + test_proj_path = os.path.join(TEST_WORKSPACE, "test_proj") + shutil.copytree(project.path(test_project), test_proj_path) + + project_info['project_path'] = test_proj_path + + # Generate a unique name for this test run. + test_project_name = project_info['name'] + '_' + uuid.uuid4().hex + + test_config['test_project'] = project_info + + # Get an environment which should be used by the tests. + test_env = env.test_env(TEST_WORKSPACE) + + # Create a basic CodeChecker config for the tests, this should + # be imported by the tests and they should only depend on these + # configuration options. + codechecker_cfg = { + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), + 'checkers': [], + 'analyzers': ['clangsa', 'clang-tidy'] + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'source_change' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Clean the test project, if needed by the tests. + ret = project.clean(test_project) + if ret: + sys.exit(ret) + + # Save the run names in the configuration. + codechecker_cfg['run_names'] = [test_project_name] + + test_config['codechecker_cfg'] = codechecker_cfg + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . self.test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/ssl/__init__.py b/web/tests/functional/ssl/__init__.py index 88316d123a..f30c492687 100644 --- a/web/tests/functional/ssl/__init__.py +++ b/web/tests/functional/ssl/__init__.py @@ -6,77 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the package tests.""" - - -import multiprocessing -import os -import shutil -import subprocess -import time - -from libtest import project -from libtest import codechecker -from libtest import env - -# Stopping event for CodeChecker server. -__STOP_SERVER = multiprocessing.Event() - -# Test workspace initialized at setup for authentication tests. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for the tests then start the server.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('ssl') - - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_config = {} - - # Setup environment variables for the test cases. - host_port_cfg = {'viewer_host': 'localhost', - 'viewer_port': env.get_free_port(), - 'viewer_product': 'ssl'} - - test_env = env.test_env(TEST_WORKSPACE) - - codechecker_cfg = { - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'analyzers': ['clangsa', 'clang-tidy'] - } - - codechecker_cfg.update(host_port_cfg) - test_config['codechecker_cfg'] = codechecker_cfg - - # Export configuration for the tests. - env.export_test_cfg(TEST_WORKSPACE, test_config) - - # Enable SSL - # ON travis auto-test fails because due to the environment - # self signed certs are not accepted - # [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:661) - # Operation not permitted - # Will need to solve this to re-enable SSL in this test. - # env.enable_ssl(TEST_WORKSPACE) - - # Enable authentication and start the CodeChecker server. - env.enable_auth(TEST_WORKSPACE) - print("Starting server to get results") - codechecker.start_server(codechecker_cfg, __STOP_SERVER) - - -def teardown_package(): - """Stop the CodeChecker server and clean up after the tests.""" - # TODO If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - __STOP_SERVER.set() - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/ssl/test_ssl.py b/web/tests/functional/ssl/test_ssl.py index f5edb31117..b6fcb0c4d9 100644 --- a/web/tests/functional/ssl/test_ssl.py +++ b/web/tests/functional/ssl/test_ssl.py @@ -10,8 +10,9 @@ SSL test. """ - +import multiprocessing import os +import shutil import subprocess import unittest @@ -26,7 +27,66 @@ class TestSSL(unittest.TestCase): Test SSL layering on the server. """ - def setUp(self): + def setup_class(self): + """Setup the environment for the tests then start the server.""" + + # Stopping event for CodeChecker server. + global __STOP_SERVER + __STOP_SERVER = multiprocessing.Event() + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('ssl') + + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_config = {} + + # Setup environment variables for the test cases. + host_port_cfg = {'viewer_host': 'localhost', + 'viewer_port': env.get_free_port(), + 'viewer_product': 'ssl'} + + test_env = env.test_env(TEST_WORKSPACE) + + codechecker_cfg = { + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'analyzers': ['clangsa', 'clang-tidy'] + } + + codechecker_cfg.update(host_port_cfg) + test_config['codechecker_cfg'] = codechecker_cfg + + # Export configuration for the tests. + env.export_test_cfg(TEST_WORKSPACE, test_config) + + # Enable SSL + # ON travis auto-test fails because due to the environment + # self signed certs are not accepted + # [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed + # (_ssl.c:661) + # Operation not permitted + # Will need to solve this to re-enable SSL in this test. + # env.enable_ssl(TEST_WORKSPACE) + + # Enable authentication and start the CodeChecker server. + env.enable_auth(TEST_WORKSPACE) + print("Starting server to get results") + codechecker.start_server(codechecker_cfg, __STOP_SERVER) + + def teardown_class(self): + """Stop the CodeChecker server and clean up after the tests.""" + # TODO If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + __STOP_SERVER.set() + __STOP_SERVER.clear() + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # Get the test workspace used to authentication tests. self._test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/statistics/__init__.py b/web/tests/functional/statistics/__init__.py index debec101c6..f30c492687 100644 --- a/web/tests/functional/statistics/__init__.py +++ b/web/tests/functional/statistics/__init__.py @@ -6,86 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the test package statistics.""" - - -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import project - - -# Test workspace should be initialized in this module. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for testing statistics.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('statistics') - - # Set the TEST_WORKSPACE used by the tests. - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_config = {} - - test_project = 'cpp' - - project_info = project.get_info(test_project) - - # Copy the test project to the workspace. The tests should - # work only on this test project. - test_proj_path = os.path.join(TEST_WORKSPACE, "test_proj") - shutil.copytree(project.path(test_project), test_proj_path) - - project_info['project_path'] = test_proj_path - - test_config['test_project'] = project_info - - # Suppress file should be set here if needed by the tests. - suppress_file = None - - # Skip list file should be set here if needed by the tests. - skip_list_file = None - - # Get an environment which should be used by the tests. - test_env = env.test_env(TEST_WORKSPACE) - - # Create a basic CodeChecker config for the tests, this should - # be imported by the tests and they should only depend on these - # configuration options. - codechecker_cfg = { - 'suppress_file': suppress_file, - 'skip_list_file': skip_list_file, - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'analyzers': ['clangsa', 'clang-tidy'] - } - - # Clean the test project, if needed by the tests. - ret = project.clean(test_project) - if ret: - sys.exit(ret) - - test_config['codechecker_cfg'] = codechecker_cfg - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/statistics/test_statistics.py b/web/tests/functional/statistics/test_statistics.py index 794b73a2f0..e7554b8874 100644 --- a/web/tests/functional/statistics/test_statistics.py +++ b/web/tests/functional/statistics/test_statistics.py @@ -12,10 +12,13 @@ from distutils import util import os -import unittest +import shutil +import sys import shlex +import unittest from libtest import env +from libtest import project from libtest.codechecker import call_command NO_STATISTICS_MESSAGE = "Statistics collector checkers are not supported" @@ -25,7 +28,72 @@ class TestSkeleton(unittest.TestCase): _ccClient = None - def setUp(self): + def setup_class(self): + """Setup the environment for testing statistics.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('statistics') + + # Set the TEST_WORKSPACE used by the tests. + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_config = {} + + test_project = 'cpp' + + project_info = project.get_info(test_project) + + # Copy the test project to the workspace. The tests should + # work only on this test project. + test_proj_path = os.path.join(TEST_WORKSPACE, "test_proj") + shutil.copytree(project.path(test_project), test_proj_path) + + project_info['project_path'] = test_proj_path + + test_config['test_project'] = project_info + + # Suppress file should be set here if needed by the tests. + suppress_file = None + + # Skip list file should be set here if needed by the tests. + skip_list_file = None + + # Get an environment which should be used by the tests. + test_env = env.test_env(TEST_WORKSPACE) + + # Create a basic CodeChecker config for the tests, this should + # be imported by the tests and they should only depend on these + # configuration options. + codechecker_cfg = { + 'suppress_file': suppress_file, + 'skip_list_file': skip_list_file, + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'analyzers': ['clangsa', 'clang-tidy'] + } + + # Clean the test project, if needed by the tests. + ret = project.clean(test_project) + if ret: + sys.exit(ret) + + test_config['codechecker_cfg'] = codechecker_cfg + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # TEST_WORKSPACE is automatically set by test package __init__.py . test_workspace = os.environ['TEST_WORKSPACE'] diff --git a/web/tests/functional/storage_of_analysis_statistics/__init__.py b/web/tests/functional/storage_of_analysis_statistics/__init__.py index fa99d4257f..f30c492687 100644 --- a/web/tests/functional/storage_of_analysis_statistics/__init__.py +++ b/web/tests/functional/storage_of_analysis_statistics/__init__.py @@ -6,82 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the package tests.""" - - -import multiprocessing -import os -import shutil -import time - -from libtest import codechecker -from libtest import env - -# Stopping event for CodeChecker server. -EVENT_1 = multiprocessing.Event() - -# Test workspace initialized at setup for authentication tests. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for the tests then start the server.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('storage_of_analysis_statistics') - - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_config = {} - - # Setup environment variables for the test cases. - host_port_cfg = {'viewer_host': 'localhost', - 'viewer_port': env.get_free_port(), - 'viewer_product': 'storage_of_analysis_statistics'} - - test_env = env.test_env(TEST_WORKSPACE) - - codechecker_cfg = { - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'viewer_product': 'storage_of_analysis_statistics', - 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), - 'analyzers': ['clangsa', 'clang-tidy'] - } - - codechecker_cfg.update(host_port_cfg) - - codechecker_cfg['run_names'] = [] - - test_config['codechecker_cfg'] = codechecker_cfg - - # Export configuration for the tests. - env.export_test_cfg(TEST_WORKSPACE, test_config) - - # Enable storage of analysis statistics and start the CodeChecker server. - env.enable_storage_of_analysis_statistics(TEST_WORKSPACE) - print("Starting server to get results") - server_access = codechecker.start_server(codechecker_cfg, EVENT_1) - - server_access['viewer_product'] = codechecker_cfg['viewer_product'] - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - -def teardown_package(): - """Stop the CodeChecker server and clean up after the tests.""" - - # TODO If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - # Let the remaining CodeChecker servers die. - EVENT_1.set() - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/storage_of_analysis_statistics/test_storage_of_analysis_statistics.py b/web/tests/functional/storage_of_analysis_statistics/test_storage_of_analysis_statistics.py index 09e292b244..18768dbbfe 100644 --- a/web/tests/functional/storage_of_analysis_statistics/test_storage_of_analysis_statistics.py +++ b/web/tests/functional/storage_of_analysis_statistics/test_storage_of_analysis_statistics.py @@ -11,6 +11,7 @@ """ +import multiprocessing import json import os import shutil @@ -42,7 +43,72 @@ class TestStorageOfAnalysisStatistics(unittest.TestCase): This class tests the CodeChecker analysis statistics storage feature. """ - def setUp(self): + def setup_class(self): + """Setup the environment for the tests then start the server.""" + + # Stopping event for CodeChecker server. + global EVENT_1 + EVENT_1 = multiprocessing.Event() + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('storage_of_analysis_statistics') + + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_config = {} + + # Setup environment variables for the test cases. + host_port_cfg = {'viewer_host': 'localhost', + 'viewer_port': env.get_free_port(), + 'viewer_product': 'storage_of_analysis_statistics'} + + test_env = env.test_env(TEST_WORKSPACE) + + codechecker_cfg = { + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'viewer_product': 'storage_of_analysis_statistics', + 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), + 'analyzers': ['clangsa', 'clang-tidy'] + } + + codechecker_cfg.update(host_port_cfg) + + codechecker_cfg['run_names'] = [] + + test_config['codechecker_cfg'] = codechecker_cfg + + # Export configuration for the tests. + env.export_test_cfg(TEST_WORKSPACE, test_config) + + # Enable storage of analysis statistics and start the CodeChecker + # server. + env.enable_storage_of_analysis_statistics(TEST_WORKSPACE) + print("Starting server to get results") + server_access = codechecker.start_server(codechecker_cfg, EVENT_1) + + server_access['viewer_product'] = codechecker_cfg['viewer_product'] + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + def teardown_class(self): + """Stop the CodeChecker server and clean up after the tests.""" + + # TODO If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + # Let the remaining CodeChecker servers die. + EVENT_1.set() + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # Get the test workspace. self.test_workspace = os.environ['TEST_WORKSPACE'] @@ -111,7 +177,7 @@ def setUp(self): xxx // Will cause a compilation error }"""] - def tearDown(self): + def teardown_method(self, method): """Restore environment after tests have ran.""" os.chdir(self.__old_pwd) diff --git a/web/tests/functional/store/__init__.py b/web/tests/functional/store/__init__.py index 7bf23a3bf8..4259749345 100644 --- a/web/tests/functional/store/__init__.py +++ b/web/tests/functional/store/__init__.py @@ -5,76 +5,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the package tests.""" - - -import os -import shutil - -from libtest import codechecker -from libtest import env -from libtest import plist_test - -# Test workspace initialized at setup for report storage tests. -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for the tests then start the server.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('store_test') - - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - # Configuration options. - codechecker_cfg = { - 'check_env': env.test_env(TEST_WORKSPACE), - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), - 'test_project': 'store_test' - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'store_test' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - # Export the test configuration to the workspace. - env.export_test_cfg(TEST_WORKSPACE, {'codechecker_cfg': codechecker_cfg}) - - # Copy test files to a temporary directory not to modify the - # files in the repository. - # Report files will be overwritten during the tests. - test_dir = os.path.dirname(os.path.realpath(__file__)) - dst_dir = os.path.join(TEST_WORKSPACE, "test_proj") - shutil.copytree(os.path.join(test_dir, "test_proj"), dst_dir) - - prefix_file_paths = [ - os.path.join(dst_dir, "divide_zero", "divide_zero.plist"), - os.path.join(dst_dir, "double_suppress", "double_suppress.plist")] - - for file_name in prefix_file_paths: - plist_test.prefix_file_path(file_name, os.path.dirname(file_name)) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/store/test_store.py b/web/tests/functional/store/test_store.py index d8ae978a78..3dc6097b4a 100644 --- a/web/tests/functional/store/test_store.py +++ b/web/tests/functional/store/test_store.py @@ -14,6 +14,7 @@ import json import os import shlex +import shutil import inspect import plistlib import shutil @@ -23,7 +24,9 @@ from codechecker_report_converter import util from codechecker_api.codeCheckerDBAccess_v6.ttypes import AnalysisInfoFilter -from libtest import codechecker, env +from libtest import codechecker +from libtest import env +from libtest import plist_test def _call_cmd(command, cwd=None, env=None): @@ -45,7 +48,66 @@ def _call_cmd(command, cwd=None, env=None): class TestStore(unittest.TestCase): """Test storage reports""" - def setUp(self): + def setup_class(self): + """Setup the environment for the tests then start the server.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('store_test') + + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + # Configuration options. + codechecker_cfg = { + 'check_env': env.test_env(TEST_WORKSPACE), + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'reportdir': os.path.join(TEST_WORKSPACE, 'reports'), + 'test_project': 'store_test' + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'store_test' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + # Export the test configuration to the workspace. + env.export_test_cfg(TEST_WORKSPACE, + {'codechecker_cfg': codechecker_cfg}) + + # Copy test files to a temporary directory not to modify the + # files in the repository. + # Report files will be overwritten during the tests. + test_dir = os.path.dirname(os.path.realpath(__file__)) + dst_dir = os.path.join(TEST_WORKSPACE, "test_proj") + shutil.copytree(os.path.join(test_dir, "test_proj"), dst_dir) + + prefix_file_paths = [ + os.path.join(dst_dir, "divide_zero", "divide_zero.plist"), + os.path.join(dst_dir, "double_suppress", "double_suppress.plist")] + + for file_name in prefix_file_paths: + plist_test.prefix_file_path(file_name, os.path.dirname(file_name)) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): # Get the test workspace used to cppcheck tests. self._test_workspace = os.environ["TEST_WORKSPACE"] diff --git a/web/tests/functional/suppress/__init__.py b/web/tests/functional/suppress/__init__.py index ed44f2a052..f30c492687 100644 --- a/web/tests/functional/suppress/__init__.py +++ b/web/tests/functional/suppress/__init__.py @@ -6,138 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the package tests.""" - - -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import project - -TEST_WORKSPACE = None - - -def setup_package(): - """Setup the environment for the tests.""" - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('suppress') - - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_project = 'suppress' - - test_config = {} - - project_info = project.get_info(test_project) - - test_proj_path = os.path.join(TEST_WORKSPACE, "test_proj") - shutil.copytree(project.path(test_project), test_proj_path) - - project_info['project_path'] = test_proj_path - - test_config['test_project'] = project_info - - # Generate a suppress file for the tests. - suppress_file = os.path.join(TEST_WORKSPACE, 'suppress_file') - if os.path.isfile(suppress_file): - os.remove(suppress_file) - _generate_suppress_file(suppress_file) - - test_env = env.test_env(TEST_WORKSPACE) - - codechecker_cfg = { - 'suppress_file': None, - 'skip_list_file': None, - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'analyzers': ['clangsa', 'clang-tidy'] - } - - ret = project.clean(test_project, test_env) - if ret: - sys.exit(ret) - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'suppress' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - test_project_name = project_info['name'] + '_' + uuid.uuid4().hex - - ret = codechecker.check_and_store(codechecker_cfg, - test_project_name, - project.path(test_project)) - - if ret: - sys.exit(1) - print("Analyzing the test project was successful.") - test_project_name_dup = test_project_name + "_duplicate" - ret = codechecker.store(codechecker_cfg, test_project_name_dup) - - codechecker_cfg['run_names'] = [test_project_name, test_project_name_dup] - test_config['codechecker_cfg'] = codechecker_cfg - - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: If environment variable is set keep the workspace - # and print out the path. - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) - - -def _generate_suppress_file(suppress_file): - """ - Create a dummy suppress file just to check if the old and the new - suppress format can be processed. - """ - print("Generating suppress file: " + suppress_file) - - import calendar - import hashlib - import random - import time - - hash_version = '1' - suppress_stuff = [] - for _ in range(10): - curr_time = calendar.timegm(time.gmtime()) - random_integer = random.randint(1, 9999999) - suppress_line = str(curr_time) + str(random_integer) - suppress_stuff.append( - hashlib.md5( - suppress_line.encode('utf-8')).hexdigest() + - '#' + hash_version) - - s_file = open(suppress_file, 'w', encoding="utf-8", errors="ignore") - for k in suppress_stuff: - s_file.write(k + '||' + 'idziei éléáálk ~!@#$#%^&*() \n') - s_file.write( - k + '||' + 'test_~!@#$%^&*.cpp' + - '||' + 'idziei éléáálk ~!@#$%^&*(\n') - s_file.write( - hashlib.md5(suppress_line.encode('utf-8')).hexdigest() + '||' + - 'test_~!@#$%^&*.cpp' + '||' + 'idziei éléáálk ~!@#$%^&*(\n') - - s_file.close() diff --git a/web/tests/functional/suppress/test_suppress_generation.py b/web/tests/functional/suppress/test_suppress_generation.py index 237db618b2..733f55cb9b 100644 --- a/web/tests/functional/suppress/test_suppress_generation.py +++ b/web/tests/functional/suppress/test_suppress_generation.py @@ -13,16 +13,57 @@ import logging import os import shlex +import shutil import subprocess +import sys from subprocess import CalledProcessError import unittest +import uuid from codechecker_api.codeCheckerDBAccess_v6.ttypes import ReviewStatus -from libtest import env, codechecker +from libtest import codechecker +from libtest import env +from libtest import project from libtest.thrift_client_to_db import get_all_run_results +def _generate_suppress_file(suppress_file): + """ + Create a dummy suppress file just to check if the old and the new + suppress format can be processed. + """ + print("Generating suppress file: " + suppress_file) + + import calendar + import hashlib + import random + import time + + hash_version = '1' + suppress_stuff = [] + for _ in range(10): + curr_time = calendar.timegm(time.gmtime()) + random_integer = random.randint(1, 9999999) + suppress_line = str(curr_time) + str(random_integer) + suppress_stuff.append( + hashlib.md5( + suppress_line.encode('utf-8')).hexdigest() + + '#' + hash_version) + + s_file = open(suppress_file, 'w', encoding="utf-8", errors="ignore") + for k in suppress_stuff: + s_file.write(k + '||' + 'idziei éléáálk ~!@#$#%^&*() \n') + s_file.write( + k + '||' + 'test_~!@#$%^&*.cpp' + + '||' + 'idziei éléáálk ~!@#$%^&*(\n') + s_file.write( + hashlib.md5(suppress_line.encode('utf-8')).hexdigest() + '||' + + 'test_~!@#$%^&*.cpp' + '||' + 'idziei éléáálk ~!@#$%^&*(\n') + + s_file.close() + + def call_cmd(command, cwd, env): try: print(' '.join(command)) @@ -46,7 +87,91 @@ class TestSuppress(unittest.TestCase): Test source-code level suppression data writing to suppress file. """ - def setUp(self): + def setup_class(self): + """Setup the environment for the tests.""" + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('suppress') + + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_project = 'suppress' + + test_config = {} + + project_info = project.get_info(test_project) + + test_proj_path = os.path.join(TEST_WORKSPACE, "test_proj") + shutil.copytree(project.path(test_project), test_proj_path) + + project_info['project_path'] = test_proj_path + + test_config['test_project'] = project_info + + # Generate a suppress file for the tests. + suppress_file = os.path.join(TEST_WORKSPACE, 'suppress_file') + if os.path.isfile(suppress_file): + os.remove(suppress_file) + _generate_suppress_file(suppress_file) + + test_env = env.test_env(TEST_WORKSPACE) + + codechecker_cfg = { + 'suppress_file': None, + 'skip_list_file': None, + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'analyzers': ['clangsa', 'clang-tidy'] + } + + ret = project.clean(test_project, test_env) + if ret: + sys.exit(ret) + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'suppress' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + test_project_name = project_info['name'] + '_' + uuid.uuid4().hex + + ret = codechecker.check_and_store(codechecker_cfg, + test_project_name, + project.path(test_project)) + + if ret: + sys.exit(1) + print("Analyzing the test project was successful.") + test_project_name_dup = test_project_name + "_duplicate" + ret = codechecker.store(codechecker_cfg, test_project_name_dup) + + codechecker_cfg['run_names'] = [test_project_name, + test_project_name_dup] + test_config['codechecker_cfg'] = codechecker_cfg + + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): self._test_workspace = os.environ['TEST_WORKSPACE'] self._testproject_data = env.setup_test_proj_cfg(self._test_workspace) diff --git a/web/tests/functional/update/__init__.py b/web/tests/functional/update/__init__.py index 0fe60a23a0..f30c492687 100644 --- a/web/tests/functional/update/__init__.py +++ b/web/tests/functional/update/__init__.py @@ -6,93 +6,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # ------------------------------------------------------------------------- - -"""Setup for the package tests.""" - - -import fnmatch -import os -import shutil -import sys -import uuid - -from libtest import codechecker -from libtest import env -from libtest import plist_test -from libtest import project - - -TEST_WORKSPACE = None - -test_dir = os.path.dirname(os.path.realpath(__file__)) - - -def setup_package(): - """Setup the environment for the tests. """ - - global TEST_WORKSPACE - TEST_WORKSPACE = env.get_workspace('update') - - os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE - - test_config = {} - - test_project_name = uuid.uuid4().hex - - test_project_path = os.path.join(test_dir, "test_proj") - - temp_test_project_data = project.prepare(test_project_path, TEST_WORKSPACE) - - test_config['test_project'] = temp_test_project_data - - test_env = env.test_env(TEST_WORKSPACE) - - base_reports = os.path.join(temp_test_project_data['test_project_reports'], - 'base') - - codechecker_cfg = { - 'suppress_file': None, - 'skip_list_file': None, - 'check_env': test_env, - 'workspace': TEST_WORKSPACE, - 'checkers': [], - 'reportdir': base_reports, - 'analyzers': ['clangsa', 'clang-tidy'] - } - - # Start or connect to the running CodeChecker server and get connection - # details. - print("This test uses a CodeChecker server... connecting...") - server_access = codechecker.start_or_get_server() - server_access['viewer_product'] = 'update' - codechecker.add_test_package_product(server_access, TEST_WORKSPACE) - - # Extend the checker configuration with the server access. - codechecker_cfg.update(server_access) - - ret = codechecker.store(codechecker_cfg, - test_project_name) - if ret: - sys.exit(1) - print("Storing the base reports was succcessful.") - - codechecker_cfg['run_names'] = [test_project_name] - - test_config['codechecker_cfg'] = codechecker_cfg - - env.export_test_cfg(TEST_WORKSPACE, test_config) - - -def teardown_package(): - """Clean up after the test.""" - - # TODO: if environment variable is set keep the workspace - # and print out the path - global TEST_WORKSPACE - - check_env = env.import_test_cfg(TEST_WORKSPACE)[ - 'codechecker_cfg']['check_env'] - codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) - - print("Removing: " + TEST_WORKSPACE) - shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) diff --git a/web/tests/functional/update/test_update_mode.py b/web/tests/functional/update/test_update_mode.py index e2a50994e8..c4f204fdea 100644 --- a/web/tests/functional/update/test_update_mode.py +++ b/web/tests/functional/update/test_update_mode.py @@ -14,20 +14,95 @@ import os +import shutil import sys import unittest +import uuid -from libtest import env from libtest import codechecker +from libtest import env +from libtest import project from libtest.debug_printer import print_run_results from libtest.thrift_client_to_db import get_all_run_results from codechecker_api.codeCheckerDBAccess_v6.ttypes import DetectionStatus +test_dir = os.path.dirname(os.path.realpath(__file__)) + class TestUpdate(unittest.TestCase): - def setUp(self): + def setup_class(self): + """Setup the environment for the tests. """ + + global TEST_WORKSPACE + TEST_WORKSPACE = env.get_workspace('update') + + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + test_config = {} + + test_project_name = uuid.uuid4().hex + + test_project_path = os.path.join(test_dir, "test_proj") + + temp_test_project_data = project.prepare( + test_project_path, TEST_WORKSPACE) + + test_config['test_project'] = temp_test_project_data + + test_env = env.test_env(TEST_WORKSPACE) + + base_reports = os.path.join( + temp_test_project_data['test_project_reports'], 'base') + + codechecker_cfg = { + 'suppress_file': None, + 'skip_list_file': None, + 'check_env': test_env, + 'workspace': TEST_WORKSPACE, + 'checkers': [], + 'reportdir': base_reports, + 'analyzers': ['clangsa', 'clang-tidy'] + } + + # Start or connect to the running CodeChecker server and get connection + # details. + print("This test uses a CodeChecker server... connecting...") + server_access = codechecker.start_or_get_server() + server_access['viewer_product'] = 'update' + codechecker.add_test_package_product(server_access, TEST_WORKSPACE) + + # Extend the checker configuration with the server access. + codechecker_cfg.update(server_access) + + ret = codechecker.store(codechecker_cfg, + test_project_name) + if ret: + sys.exit(1) + print("Storing the base reports was succcessful.") + + codechecker_cfg['run_names'] = [test_project_name] + + test_config['codechecker_cfg'] = codechecker_cfg + + env.export_test_cfg(TEST_WORKSPACE, test_config) + + def teardown_class(self): + """Clean up after the test.""" + + # TODO: if environment variable is set keep the workspace + # and print out the path + global TEST_WORKSPACE + + check_env = env.import_test_cfg(TEST_WORKSPACE)[ + 'codechecker_cfg']['check_env'] + codechecker.remove_test_package_product(TEST_WORKSPACE, check_env) + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE, ignore_errors=True) + + def setup_method(self, method): self._test_workspace = os.environ.get('TEST_WORKSPACE') test_class = self.__class__.__name__