From e0e3e6cf53a16fb52b0f4714c30f0aa484f62eb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Alfred=C3=A9en?= Date: Fri, 4 Oct 2024 09:00:07 +0200 Subject: [PATCH] Upgraded Locusat and small fixes to some tests (#14) * Upgraded Locust to version 2.31.8 for local development. Upgraded Python to 3.12. Added instructions for using pyenv. * Fixed some load tests that were missing trailing slashes or using the incorrect http verb. Namely create project and logout tasks. * Upgraded linting and other tools in the pre-commit action. Also fixed some formatting errors in a test file. * Upgraded the checkout task in the CI action. * Upgraded Locust and locust-plugins also in the custom docker image. --- .github/workflows/ci.yaml | 4 +- .github/workflows/pre-commit.yaml | 10 ++-- .github/workflows/publish.yaml | 2 + .gitignore | 3 + README.md | 40 ++++++++++--- manifests/base/deployment.yaml | 2 +- manifests/base/postgres-deployment.yaml | 10 ++-- manifests/base/secret.yaml | 8 +++ manifests/built/deployment.yaml | 2 +- manifests/built/postgres-deployment.yaml | 8 +-- manifests/overlays/production/patch.yaml | 13 +++-- pyproject.toml | 8 +-- source/Dockerfile | 7 ++- source/locust-ui.conf | 3 +- source/locust.conf | 2 +- source/requirements.txt | 2 +- source/tests-dev/appviewer_requestshtml.py | 8 ++- source/tests/base_user_types.py | 66 +++++++++++++++------- source/tests/test_verify_setup.py | 3 +- 19 files changed, 139 insertions(+), 62 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 71f775e..c66179d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,7 +23,7 @@ jobs: HADOLINT_RECURSIVE: "true" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: hadolint/hadolint-action@v3.1.0 with: dockerfile: "Dockerfile*" @@ -33,6 +33,6 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build the Docker image run: docker build source --file source/Dockerfile --tag ci:$(date +%s) diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 9bfaf9d..55e7761 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -12,19 +12,19 @@ on: jobs: pre_commit: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.12 - name: Install dependencies run: | - python -m pip install --upgrade pre-commit==3.3.3 + python -m pip install --upgrade pre-commit==3.7.1 - name: Run pre-commit run: pre-commit run --all-files diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 1746d4b..a6a6ae3 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -5,6 +5,8 @@ on: branches: [ "main" ] paths-ignore: - '**.md' + - 'argocd/**' + - 'manifests/**' # Publish semver tags as releases. tags: [ 'v*.*.*' ] diff --git a/.gitignore b/.gitignore index 68eda77..797c196 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,9 @@ docs/_build/ .pybuilder/ target/ +# pyenv +.python-version + # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: diff --git a/README.md b/README.md index 0d4db75..9ee3022 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,25 @@ Make changes in the develop branch, commit and submit pull requests to merge to ## Setup for local development +### Using virtual environments with venv + cd ./source python3 -m venv .venv source ./.venv/bin/activate python3 -m pip install --upgrade pip python3 -m pip install -r requirements.txt +### Using pyenv + + cd ./source + pyenv virtualenv 3.12.2 locust-3.12.2 + pyenv local locust-3.12.2 + pyenv version + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements.txt + +### Check the Locust version + locust -V ## Configuration options @@ -25,6 +38,12 @@ Make changes in the develop branch, commit and submit pull requests to merge to To configure the test runs, use the Locust configuration file ./source/locust.conf For options, see the [Locust docs](https://docs.locust.io/en/stable/configuration.html) +You can override all of the settings set in the configuration file by setting the same +on the command line. For example, to override the log level settings use --loglevel as in example: + + locust --headless -f ./tests/test_verify_setup.py --loglevel debug + + ## Create tests Create locust tests in the /source/tests directory. @@ -45,13 +64,13 @@ Open the generated html report and verify that there are no errors and that the Run the command - locust --config locust-ui.conf --modern-ui --class-picker -f ./tests/test_verify_setup.py --html ./reports/locust-report-verify-setup-ui.html + locust --config locust-ui.conf --class-picker -f ./tests/test_verify_setup.py --html ./reports/locust-report-verify-setup-ui.html Open a browser tab at URL http://localhost:8089/ Paste in as host - https://staging.serve-dev.scilifelab.se + https://serve-dev.scilifelab.se ## Verify the setup and access to a host (URL) @@ -80,7 +99,10 @@ Move into the source directory if not already there: cd ./source - Copy the template environment file .env.template as .env -- Edit the followinf values in the .env file according to your needs. + + cp ./.env.template .env + +- Edit the following values in the .env file according to your needs. - SERVE_LOCUST_TEST_USER_PASS=(The password of the test locust users) - SERVE_LOCUST_DO_CREATE_OBJECTS=(A boolean indicating whether to create objects in Serve such as projects and apps) @@ -97,11 +119,15 @@ Use minimum 10 users for the Normal test plan locust --headless -f ./tests/test_plan_normal.py --html ./reports/locust-report-normal.html --users 10 --run-time 30s +Or using the Web UI + + locust --config locust-ui.conf --class-picker -f ./tests/test_plan_normal.py --html ./reports/locust-report-normal-ui.html --users 10 --run-time 30s + + ### To run the Classroom test plan/scenario locust --headless -f ./tests/test_plan_classroom.py --html ./reports/locust-report-classroom.html --users 1 --run-time 30s - ## Tests under development These tests are not yet ready to be used in a load testing session. @@ -156,9 +182,9 @@ Copy the environment variables template file to .env and edit as needed. Then ru cd ./source - docker build -t serve-load-testing . + docker build -t serve-load-testing:dev . - docker run -p 8089:8089 --env-file ./.env serve-load-testing + docker run -p 8089:8089 --env-file ./.env serve-load-testing:dev ## Deploy to a kubernetes cluster in a production environment @@ -190,7 +216,7 @@ More specifically, we use the dashboards feature or locust-plugins. See https://github.com/SvenskaSpel/locust-plugins/tree/master/locust_plugins/dashboards -To setup locust-plugins, there is an option or use locust-compose or manually setup. We use manual setup so that the +To setup locust-plugins, there is an option to use locust-compose or manually setup. We use manual setup so that the dashboards can always be running. locust-plugins is integrated in the production version of this project (deployed to kubernetes using k8s manifests and a built docker image) but not used in local development. However the dashboards can be installed locally using: diff --git a/manifests/base/deployment.yaml b/manifests/base/deployment.yaml index 1cb80d1..15230b8 100644 --- a/manifests/base/deployment.yaml +++ b/manifests/base/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: locust - image: ghcr.io/scilifelabdatacentre/serve-load-testing:main-20240429-1418 + image: ghcr.io/scilifelabdatacentre/serve-load-testing:main-20240503-1109 ports: - containerPort: 8089 envFrom: diff --git a/manifests/base/postgres-deployment.yaml b/manifests/base/postgres-deployment.yaml index c10c960..4f9d885 100644 --- a/manifests/base/postgres-deployment.yaml +++ b/manifests/base/postgres-deployment.yaml @@ -42,13 +42,15 @@ spec: name: postgres-secret ports: - containerPort: 5432 + capabilities: + drop: ["ALL"] resources: limits: - cpu: "1" - memory: "1Gi" + cpu: "2" + memory: "5Gi" requests: - cpu: "250m" - memory: "64Mi" + cpu: "500m" + memory: "1Gi" volumeMounts: - name: postgres-storage mountPath: /var/lib/postgresql/data diff --git a/manifests/base/secret.yaml b/manifests/base/secret.yaml index c099047..312f081 100644 --- a/manifests/base/secret.yaml +++ b/manifests/base/secret.yaml @@ -4,6 +4,7 @@ metadata: name: locust-secrets type: Opaque data: + LOCUST_WEB_HOST: MC4wLjAuMA== SERVE_USERNAME: Y2hhbmdlLW1l SERVE_PASS: Y2hhbmdlLW1l SERVE_LOCUST_TEST_USER_PASS: cGFzc3dvcmQxMjM= @@ -13,3 +14,10 @@ data: PGPORT: NTQzMg== PGUSER: cG9zdGdyZXM= PGPASSWORD: Q2JpNk42RWVSUGc2RTdC +--- +apiVersion: v1 +kind: Secret +metadata: + name: locust-ui-secret +data: + auth: Y2hhbmdlLW1l diff --git a/manifests/built/deployment.yaml b/manifests/built/deployment.yaml index 1cb80d1..15230b8 100644 --- a/manifests/built/deployment.yaml +++ b/manifests/built/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: locust - image: ghcr.io/scilifelabdatacentre/serve-load-testing:main-20240429-1418 + image: ghcr.io/scilifelabdatacentre/serve-load-testing:main-20240503-1109 ports: - containerPort: 8089 envFrom: diff --git a/manifests/built/postgres-deployment.yaml b/manifests/built/postgres-deployment.yaml index c10c960..58eae29 100644 --- a/manifests/built/postgres-deployment.yaml +++ b/manifests/built/postgres-deployment.yaml @@ -44,11 +44,11 @@ spec: - containerPort: 5432 resources: limits: - cpu: "1" - memory: "1Gi" + cpu: "2" + memory: "5Gi" requests: - cpu: "250m" - memory: "64Mi" + cpu: "500m" + memory: "1Gi" volumeMounts: - name: postgres-storage mountPath: /var/lib/postgresql/data diff --git a/manifests/overlays/production/patch.yaml b/manifests/overlays/production/patch.yaml index 1f33ccd..b48c31b 100644 --- a/manifests/overlays/production/patch.yaml +++ b/manifests/overlays/production/patch.yaml @@ -4,11 +4,16 @@ metadata: name: locust-secrets type: Opaque data: + LOCUST_WEB_HOST: MC4wLjAuMA== SERVE_USERNAME: Y2hhbmdlLW1l SERVE_PASS: Y2hhbmdlLW1l SERVE_LOCUST_TEST_USER_PASS: cGFzc3dvcmQxMjM= SERVE_LOCUST_DO_CREATE_OBJECTS: ZmFsc2U= PROTECTED_PAGE_RELATIVE_URL: Y2hhbmdlLW1l + PGHOST: cG9zdGdyZXMtc2VydmljZS5sb2N1c3Quc3ZjLmNsdXN0ZXIubG9jYWw= + PGPORT: NTQzMg== + PGUSER: cG9zdGdyZXM= + PGPASSWORD: Q2JpNk42RWVSUGc2RTdC --- apiVersion: v1 kind: Service @@ -19,8 +24,8 @@ spec: selector: app: locust ports: + - protocol: TCP - name: web - protocol: TCP port: 80 targetPort: 8089 type: ClusterIP @@ -32,13 +37,13 @@ metadata: namespace: locust annotations: kubernetes.io/ingress.class: "nginx" - cert-manager.io/cluster-issuer: "letsencrypt-issuer" + cert-manager.io/cluster-issuer: "letsencrypt-prod" #"letsencrypt-issuer" nginx.ingress.kubernetes.io/auth-type: basic nginx.ingress.kubernetes.io/auth-secret: locust-ui-secret nginx.ingress.kubernetes.io/auth-realm: "Protected area" spec: rules: - - host: locust.serve-dev.scilifelab.se + - host: locust.serve-dev.scilifelab-1-dev.sys.kth.se http: paths: - pathType: Prefix @@ -51,5 +56,5 @@ spec: #ingressClassName: nginx tls: - hosts: - - locust.serve-dev.scilifelab.se + - locust.serve-dev.scilifelab-1-dev.sys.kth.se secretName: locust-tls diff --git a/pyproject.toml b/pyproject.toml index 8170592..abb6938 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,9 +5,9 @@ [project] name = "serve-load-testing" -version = "1.0.0" +version = "1.0.1" description = "Load testing of the SciLifeLab Serve platform." -requires-python = "=3.8" +requires-python = "=3.12" keywords = ["load testing", "locust", "python"] [tool.isort] @@ -15,7 +15,7 @@ profile = 'black' [tool.black] line-length = 120 -target-version = ['py38'] +target-version = ['py312'] include = '\.pyi?$' extend-exclude = ''' /( @@ -29,7 +29,7 @@ extend-exclude = ''' [tool.mypy] strict = false -python_version = "3.8" +python_version = "3.12" ignore_missing_imports = true warn_return_any = true exclude = ["venv", ".venv", "examples"] diff --git a/source/Dockerfile b/source/Dockerfile index c4189d6..eb71be6 100644 --- a/source/Dockerfile +++ b/source/Dockerfile @@ -1,16 +1,19 @@ # Use the Locust image as the base image -FROM locustio/locust:2.25.0 +FROM locustio/locust:2.31.8 WORKDIR /home/locust # Install the locust-plugins dashboards plugin -RUN pip3 install --no-cache-dir locust-plugins[dashboards]==4.4.2 +RUN pip3 install --no-cache-dir locust-plugins[dashboards]==4.5.3 # Copy the Locust files into the container COPY --chown=locust:locust locust-ui.conf locust.conf COPY --chown=locust:locust tests/ tests +COPY --chown=locust:locust tests-dev/ tests-dev COPY --chown=locust:locust start-script.sh start-script.sh +COPY --chown=locust:locust run_test_plan.sh run_test_plan.sh RUN chmod ug+x start-script.sh +RUN chmod ug+x run_test_plan.sh EXPOSE 8089 5557 diff --git a/source/locust-ui.conf b/source/locust-ui.conf index dcb8f97..87fdf61 100644 --- a/source/locust-ui.conf +++ b/source/locust-ui.conf @@ -1,13 +1,14 @@ # locust-ui.conf locustfile = tests/test_plan_normal.py headless = false +class-picker = true #master = true #expect-workers = 5 only-summary = true csv = stats/locust loglevel = INFO # Can be overridden in UI: -host = https://staging.serve-dev.scilifelab.se +host = https://serve-staging.serve-dev.scilifelab.se users = 1 spawn-rate = 1 run-time = 10s diff --git a/source/locust.conf b/source/locust.conf index 334bbff..ec42462 100644 --- a/source/locust.conf +++ b/source/locust.conf @@ -3,7 +3,7 @@ locustfile = tests #headless = true #master = true #expect-workers = 5 -host = https://serve-dev.scilifelab.se +host = https://serve-staging.serve-dev.scilifelab.se users = 1 spawn-rate = 1 run-time = 10s diff --git a/source/requirements.txt b/source/requirements.txt index f6f1ef5..16bc5a1 100644 --- a/source/requirements.txt +++ b/source/requirements.txt @@ -1,2 +1,2 @@ -locust>=2.20.0 +locust>=2.31.8 requests-html>=0.10.0 diff --git a/source/tests-dev/appviewer_requestshtml.py b/source/tests-dev/appviewer_requestshtml.py index 796d9de..d6266c4 100644 --- a/source/tests-dev/appviewer_requestshtml.py +++ b/source/tests-dev/appviewer_requestshtml.py @@ -1,6 +1,8 @@ """ Handles opening user apps. -This implementation does not use Locust and does not work with Locust. +This implementation does not use Locust and does not work through Locust. +It can however be run concurrently with Locust load tests to put additional +realistic load on the system. Note that shiny proxy pods are configured with - heartbeat-rate=10s @@ -22,8 +24,8 @@ # The user app URLs to open in succession URL_LIST = [ - # "https://loadtest-shinyproxy.serve-dev.scilifelab.se/app/loadtest-shinyproxy" - "https://loadtest-shinyproxy2.staging.serve-dev.scilifelab.se/app/loadtest-shinyproxy2" + "https://loadtest-shinyproxy.serve-dev.scilifelab.se/app/loadtest-shinyproxy" + # "https://loadtest-shinyproxy.serve-staging.serve-dev.scilifelab.se/app/loadtest-shinyproxy" # "https://demo-bayesianlinmod.serve.scilifelab.se/app/demo-bayesianlinmod", # "https://demo-markovchain.serve.scilifelab.se/app/demo-markovchain" ] diff --git a/source/tests/base_user_types.py b/source/tests/base_user_types.py index 43b64f4..2c4a9a6 100644 --- a/source/tests/base_user_types.py +++ b/source/tests/base_user_types.py @@ -100,14 +100,19 @@ def register_user(self): catch_response=True, ) as response: logger.debug("signup response.status_code = %s, %s", response.status_code, response.reason) - # If login succeeds then url = /accounts/login/ + # If the signup succeeds then url = /accounts/login/ logger.debug("signup response.url = %s", response.url) if "/accounts/login" in response.url: self.user_has_registered = True else: + logger.warning( + f"Register as new user {self.email} failed. \ + Response URL {response.url} does not contain /accounts/login" + ) logger.debug(response.content) response.failure( - f"Register as new user {self.email} failed. Response URL does not contain /accounts/login" + f"Register as new user {self.email} failed. \ + Response URL {response.url} does not contain /accounts/login" ) def get_token(self): @@ -174,6 +179,7 @@ def power_user_task(self): self.login() if self.is_authenticated is False: + logger.info(f"After login function but user {self.username} is not authenticated. Ending task.") return # Open user docs pages @@ -215,7 +221,7 @@ def get_token(self, relative_url: str = "/accounts/login/"): def create_project(self, project_name: str): # Update the csrf token - self.get_token("/projects/create?template=Default project") + self.get_token("/projects/create/?template=Default project") project_data = dict( name=project_name, @@ -225,27 +231,30 @@ def create_project(self, project_name: str): ) with self.client.post( - url="/projects/create?template=Default%20project", + url="/projects/create/?template=Default%20project", data=project_data, headers={"Referer": "foo"}, name="---CREATE-NEW-PROJECT", catch_response=True, ) as response: logger.debug("create project response.status_code = %s, %s", response.status_code, response.reason) - # If succeeds then url = // + # If succeeds then url = /projects// logger.debug("create project response.url = %s", response.url) if project_name in response.url: logger.info("Successfully created project %s", project_name) self.project_url = response.url else: - logger.warning(response.content) + logger.warning( + f"Create project failed. Response URL {response.url} does not contain project name {project_name}" + ) + # logger.debug(response.content) response.failure("Create project failed. Response URL does not contain project name.") def delete_project(self): # Update the csrf token self.get_token("/projects") - delete_project_url = f"{self.project_url}/delete" + delete_project_url = f"{self.project_url}delete/" # The project_url already contains a trailing slash logger.info("Deleting the project at URL: %s", delete_project_url) delete_project_data = dict(csrfmiddlewaretoken=self.csrftoken) @@ -258,12 +267,16 @@ def delete_project(self): catch_response=True, ) as response: logger.debug("delete project response.status_code = %s, %s", response.status_code, response.reason) - # If succeeds then url = /projects/ + # If succeeds then status_code == 200 and url = /projects/ logger.debug("delete project response.url = %s", response.url) - if "/projects" in response.url: + if response.status_code == 200 and "/projects" in response.url: logger.info("Successfully deleted project at %s", self.project_url) else: - logger.warning(response.content) + logger.warning( + f"Delete project failed for project {self.project_url}. \ + Response status not 200 or URL does not contain /projects." + ) + # logger.debug(response.content) response.failure("Delete project failed. Response URL does not contain /projects.") def login(self): @@ -283,17 +296,28 @@ def login(self): catch_response=True, ) as response: logger.debug("login response.status_code = %s, %s", response.status_code, response.reason) - # If login succeeds then url = /accounts/login/, else /projects/ + # If login succeeds then the url contains /projects/, else /accounts/login/ logger.debug("login response.url = %s", response.url) if "/projects" in response.url: self.is_authenticated = True else: + logger.warning( + f"Login as user {self.username} failed. Response URL {response.url} does not contain /projects" + ) response.failure(f"Login as user {self.username} failed. Response URL does not contain /projects") def logout(self): - logger.debug("Logout user %s", self.username) - # logout_data = dict(username=self.username, csrfmiddlewaretoken=self.csrftoken) - self.client.get("/accounts/logout/", name="---ON STOP---LOGOUT") + if self.is_authenticated: + logger.debug("Logout user %s", self.username) + logout_data = dict(username=self.username, csrfmiddlewaretoken=self.csrftoken) + with self.client.post( + "/accounts/logout/", + data=logout_data, + headers={"Referer": "foo"}, + name="---ON STOP---LOGOUT", + catch_response=True, + ): + pass class AppViewerUser(HttpUser): @@ -318,17 +342,17 @@ def open_user_app(self): if self.host == "https://serve-dev.scilifelab.se": # Dev - # ex: https://loadtest-shinyproxy.staging.serve-dev.scilifelab.se/app/loadtest-shinyproxy - # from host: https://staging.serve-dev.scilifelab.se + # ex: https://loadtest-shinyproxy.serve-dev.scilifelab.se/app/loadtest-shinyproxy + # from host: https://serve-dev.scilifelab.se APP_SHINYPROXY = self.host.replace("https://", "https://loadtest-shinyproxy.") APP_SHINYPROXY += "/app/loadtest-shinyproxy" elif "staging" in self.host: # Staging - # ex: https://loadtest-shinyproxy2.staging.serve-dev.scilifelab.se/app/loadtest-shinyproxy2 - # from host: https://staging.serve-dev.scilifelab.se - APP_SHINYPROXY = self.host.replace("https://", "https://loadtest-shinyproxy3.") - APP_SHINYPROXY += "/app/loadtest-shinyproxy3" + # ex: https://loadtest-shinyproxy.serve-staging.serve-dev.scilifelab.se/app/loadtest-shinyproxy + # from host: https://serve-staging.serve-dev.scilifelab.se + APP_SHINYPROXY = self.host.replace("https://", "https://loadtest-shinyproxy.") + APP_SHINYPROXY += "/app/loadtest-shinyproxy" elif "serve.scilifelab.se" in self.host: # Production @@ -336,7 +360,7 @@ def open_user_app(self): APP_SHINYPROXY = self.host.replace("https://", "https://adhd-medication-sweden.") APP_SHINYPROXY += "/app/adhd-medication-sweden" - logger.debug("making GET request to URL: %s", APP_SHINYPROXY) + logger.debug("making GET request to user app URL: %s", APP_SHINYPROXY) self.client.get(APP_SHINYPROXY, name="user-app-shiny-proxy") diff --git a/source/tests/test_verify_setup.py b/source/tests/test_verify_setup.py index b2e1533..0520dca 100644 --- a/source/tests/test_verify_setup.py +++ b/source/tests/test_verify_setup.py @@ -20,4 +20,5 @@ def on_start(self): @task def verify_task(self): - logger.info("executing simple task verify_task") + logger.debug("executing simple task verify_task. Log level = debug.") + logger.info("executing simple task verify_task. Log level = info.")