From cbc33fda6e0de458a12a0b84269ee621583493c1 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Thu, 28 Nov 2019 14:14:51 +0100 Subject: [PATCH] Convert pipelines to obsah --- bin/forklift | 61 +++++++++++++++++ docs/testing.md | 90 ++++++++++++++------------ pipelines/install/install.yaml | 7 ++ pipelines/install/metadata.obsah.yaml | 17 +++++ pipelines/install_pipeline.yml | 7 -- pipelines/pulpcore.yml | 3 - pipelines/pulpcore/metadata.obsah.yaml | 14 ++++ pipelines/pulpcore/pulpcore.yaml | 3 + pipelines/upgrade/metadata.obsah.yaml | 18 ++++++ pipelines/upgrade/upgrade.yaml | 11 ++++ pipelines/upgrade_pipeline.yml | 11 ---- pipelines/vars/install_base.yml | 1 + pipelines/vars/upgrade_base.yml | 1 + 13 files changed, 181 insertions(+), 63 deletions(-) create mode 100755 bin/forklift create mode 100644 pipelines/install/install.yaml create mode 100644 pipelines/install/metadata.obsah.yaml delete mode 100644 pipelines/install_pipeline.yml delete mode 100644 pipelines/pulpcore.yml create mode 100644 pipelines/pulpcore/metadata.obsah.yaml create mode 100644 pipelines/pulpcore/pulpcore.yaml create mode 100644 pipelines/upgrade/metadata.obsah.yaml create mode 100644 pipelines/upgrade/upgrade.yaml delete mode 100644 pipelines/upgrade_pipeline.yml diff --git a/bin/forklift b/bin/forklift new file mode 100755 index 000000000..67d96e54d --- /dev/null +++ b/bin/forklift @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# PYTHON_ARGCOMPLETE_OK + +from typing import List +from pathlib import Path + +import obsah + + +class ApplicationConfig(obsah.ApplicationConfig): + """ + A class describing the where to find various files + """ + + @staticmethod + def name() -> str: + """ + Return the name as shown to the user in the ArgumentParser + """ + return 'forklift' + + @staticmethod + def target_name() -> str: + """ + Return the name of the target in the playbook if the playbook takes a parameter. + """ + return 'pipeline' + + @staticmethod + def data_path() -> Path: + """ + Returns the root of the project + """ + return Path(__file__).absolute().parent.parent + + @classmethod + def playbooks_path(cls: obsah.ApplicationConfig) -> str: + """ + Return the data path. Houses playbooks and configs. + """ + return (cls.data_path() / 'pipelines').as_posix() + + @classmethod + def playbooks(cls: obsah.ApplicationConfig) -> List[obsah.Playbook]: + """ + Return all playbooks in the playbook path. + """ + paths = Path(cls.playbooks_path()).glob('*/*.yaml') + return sorted(obsah.Playbook(playbook_path.as_posix(), cls) for playbook_path in paths + if playbook_path.stem == playbook_path.parent.name) + + @classmethod + def inventory_path(cls) -> str: + """ + Return the inventory path + """ + return (cls.data_path() / 'inventories').as_posix() + + +if __name__ == '__main__': + obsah.main(application_config=ApplicationConfig) diff --git a/docs/testing.md b/docs/testing.md index b45033d51..7e80751d0 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -61,57 +61,63 @@ centos7-foreman-bats-ci: ## Pipeline Testing -Under `pipelines` are a series of playbooks designed around testing scenarios for various version of the Foreman and Katello stack. To run one: +Under `pipelines` are a series of playbooks designed around testing scenarios for various version of the Foreman and Katello stack. Use `./bin/forklift --help` to find out which ones and their aguments. - ansible-playbook pipelines/.yml -e forklift_state=up -e +```console +$ ./bin/forklift --help +usage: forklift [-h] action ... -When you are finished with the test, you can tear down the associated infrastructure: +positional arguments: + action which action to execute + install Run an install pipeline + upgrade Run an upgrade pipeline - ansible-playbook pipelines/.yml -e forklift_state=destroy -e - -### Existing Pipelines - -* `install_pipeline` - Installs a Server and a Proxy VMs and runs the `foreman_testing` role to verify the setup. - Expects the `pipeline_os` variable to be set to a known OS (currently: centos7, debian10) - Expects the `pipeline_type` variable to be set to a known type (currently: foreman, katello, luna) - Expects the `pipeline_version` variable to be set to a known version (currently: 3.8, 3.9, 3.10, 3.11, nightly) -* `upgrade_pipeline` - Installs a VM, upgrades it twice and runs the `foreman_testing` role to verify the final upgrade. - Expects the `pipeline_os` variable to be set to a known OS (currently: centos7, debian10) - Expects the `pipeline_type` variable to be set to a known type (currently: foreman, katello, luna) - Expects the `pipeline_version` variable to be set to a known version (currently: 3.8, 3.9, 3.10, 3.11, nightly). - -#### Examples - - ansible-playbook pipelines/install_pipeline.yml -e forklift_state=up -e pipeline_os=debian10 -e pipeline_type=foreman -e pipeline_version=nightly - ansible-playbook pipelines/upgrade_pipeline.yml -e forklift_state=up -e pipeline_os=centos7 -e pipeline_type=katello -e pipeline_version=3.10 +optional arguments: + -h, --help show this help message and exit +``` -### Creating Pipelines +Individual pipelines also have help texts: +```console +$ ./bin/forklift install --help +usage: forklift install [-h] [-v] [-e EXTRA_VARS] [--state FORKLIFT_STATE] + [--os PIPELINE_OS] [--type PIPELINE_TYPE] + [--version PIPELINE_VERSION] + +Run an install pipeline + +optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --state FORKLIFT_STATE + Forklift state to ensure + --os PIPELINE_OS Operating system to install, like centos7, debian10 or + ubuntu1804 + --type PIPELINE_TYPE Type of pipeline, like foreman, katello or luna + --version PIPELINE_VERSION + Version to install, like nightly, 1.24 or 3.14 + +advanced arguments: + -e EXTRA_VARS, --extra-vars EXTRA_VARS + set additional variables as key=value or YAML/JSON, if + filename prepend with @ +``` -If you wish to add a new version of an existing pipeline (e.g. a new Katello release), you only have to add the corresponding vars files to `pipelines/vars/`. +Pipelines typically have a state which defaults to `up`. Other valid values are `rebuild` and `destroy`. The latter one is useful to clean up which pipelines don't do by themselves. -For Katello 3.11, you'd be adding the following two files: +For example to run a Foreman Nightly installation pipeline on Debian Buster: -`pipelines/vars/katello_3.11.yml`: -```yaml -forklift_name: pipeline-katello-3.11 -forklift_boxes: - pipeline-katello-3.11-centos7: - box: centos7 - memory: 8192 - pipeline-proxy-3.11-centos7: - box: centos7 - memory: 3072 -katello_repositories_version: '3.11' -katello_repositories_pulp_version: '2.19' -foreman_repositories_version: '1.21' -foreman_client_repositories_version: "{{ foreman_repositories_version }}" +```console +$ ./bin/forklift install --os debian10 --type foreman --version nightly +... lots of output +$ ./bin/forklift install --os debian10 --type foreman --version nightly --state destroy ``` -`pipelines/vars/katello_upgrade_3.11.yml`: -```yaml -katello_version_start: '3.9' -katello_version_intermediate: '3.10' -katello_version_final: '{{ katello_version }}' +Similarly a Katello Nightly upgrade pipeline on CentOS 7: + +```console +$ ./bin/forklift upgrade --os centos7 --type katello --version nightly +... lots of output +$ ./bin/forklift upgrade --os centos7 --type katello --version nightly --state destroy ``` ## Running Robottelo Tests diff --git a/pipelines/install/install.yaml b/pipelines/install/install.yaml new file mode 100644 index 000000000..a2d0af0c6 --- /dev/null +++ b/pipelines/install/install.yaml @@ -0,0 +1,7 @@ +--- +- import_playbook: 01-boxes.yaml +- import_playbook: 02-setup.yaml +- import_playbook: 03-install_server.yaml +- import_playbook: 04-install_proxy.yaml +- import_playbook: 05-tests.yaml +- import_playbook: 06-smoker.yaml diff --git a/pipelines/install/metadata.obsah.yaml b/pipelines/install/metadata.obsah.yaml new file mode 100644 index 000000000..6259aa0ee --- /dev/null +++ b/pipelines/install/metadata.obsah.yaml @@ -0,0 +1,17 @@ +help: | + Run an install pipeline +variables: + forklift_state: + parameter: --state + help: Forklift state to ensure + # TODO: up/rebuild/destroy + pipeline_os: + parameter: --os + help: Operating system to install, like centos7, debian10 or ubuntu1804 + pipeline_type: + parameter: --type + help: Type of pipeline, like foreman, katello or luna + # TODO: choices: foreman/katello/luna + pipeline_version: + parameter: --version + help: Version to install, like nightly, 1.24 or 3.14 diff --git a/pipelines/install_pipeline.yml b/pipelines/install_pipeline.yml deleted file mode 100644 index 584e80591..000000000 --- a/pipelines/install_pipeline.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- import_playbook: install/01-boxes.yaml -- import_playbook: install/02-setup.yaml -- import_playbook: install/03-install_server.yaml -- import_playbook: install/04-install_proxy.yaml -- import_playbook: install/05-tests.yaml -- import_playbook: install/06-smoker.yaml diff --git a/pipelines/pulpcore.yml b/pipelines/pulpcore.yml deleted file mode 100644 index c13faf95c..000000000 --- a/pipelines/pulpcore.yml +++ /dev/null @@ -1,3 +0,0 @@ -- import_playbook: pulpcore/01-boxes.yaml -- import_playbook: pulpcore/02-install.yaml -- import_playbook: pulpcore/03-tests.yaml diff --git a/pipelines/pulpcore/metadata.obsah.yaml b/pipelines/pulpcore/metadata.obsah.yaml new file mode 100644 index 000000000..1683354a5 --- /dev/null +++ b/pipelines/pulpcore/metadata.obsah.yaml @@ -0,0 +1,14 @@ +help: | + Run a Pulpcore pipeline +variables: + forklift_state: + parameter: --state + help: Forklift state to ensure + # TODO: up/rebuild/destroy + pipeline_os: + parameter: --os + help: Operating system to install, like centos7, debian10 or ubuntu1804 + pipeline_type: + parameter: --type + help: Type of pipeline, like foreman, katello or luna + # TODO: choices: foreman/katello/luna diff --git a/pipelines/pulpcore/pulpcore.yaml b/pipelines/pulpcore/pulpcore.yaml new file mode 100644 index 000000000..5d1168039 --- /dev/null +++ b/pipelines/pulpcore/pulpcore.yaml @@ -0,0 +1,3 @@ +- import_playbook: 01-boxes.yaml +- import_playbook: 02-install.yaml +- import_playbook: 03-tests.yaml diff --git a/pipelines/upgrade/metadata.obsah.yaml b/pipelines/upgrade/metadata.obsah.yaml new file mode 100644 index 000000000..73ec64425 --- /dev/null +++ b/pipelines/upgrade/metadata.obsah.yaml @@ -0,0 +1,18 @@ +help: | + Run an upgrade pipeline +variables: + forklift_state: + parameter: --state + help: Forklift state to ensure + # TODO: up/rebuild/destroy + pipeline_os: + parameter: --os + help: Operating system to install, like centos7, debian10 or ubuntu1804 + pipeline_type: + parameter: --type + help: Type of pipeline, like foreman, katello or luna + # TODO: choices: foreman/katello/luna + pipeline_version: + parameter: --version + help: Version to install, like nightly, 1.24 or 3.14 + diff --git a/pipelines/upgrade/upgrade.yaml b/pipelines/upgrade/upgrade.yaml new file mode 100644 index 000000000..6e6ef3159 --- /dev/null +++ b/pipelines/upgrade/upgrade.yaml @@ -0,0 +1,11 @@ +--- +- import_playbook: 01-boxes.yaml +- import_playbook: 02-setup.yaml +- import_playbook: 03-install_server.yaml +- import_playbook: 04-install_proxy.yaml +- import_playbook: 05-server_to_intermediate.yaml +- import_playbook: 06-proxy_to_intermediate.yaml +- import_playbook: 07-server_to_final.yaml +- import_playbook: 08-proxy_to_final.yaml +- import_playbook: 09-tests.yaml +- import_playbook: 10-smoker.yaml diff --git a/pipelines/upgrade_pipeline.yml b/pipelines/upgrade_pipeline.yml deleted file mode 100644 index 824520f1f..000000000 --- a/pipelines/upgrade_pipeline.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- import_playbook: upgrade/01-boxes.yaml -- import_playbook: upgrade/02-setup.yaml -- import_playbook: upgrade/03-install_server.yaml -- import_playbook: upgrade/04-install_proxy.yaml -- import_playbook: upgrade/05-server_to_intermediate.yaml -- import_playbook: upgrade/06-proxy_to_intermediate.yaml -- import_playbook: upgrade/07-server_to_final.yaml -- import_playbook: upgrade/08-proxy_to_final.yaml -- import_playbook: upgrade/09-tests.yaml -- import_playbook: upgrade/10-smoker.yaml diff --git a/pipelines/vars/install_base.yml b/pipelines/vars/install_base.yml index 8ede4ae83..3000cfc62 100644 --- a/pipelines/vars/install_base.yml +++ b/pipelines/vars/install_base.yml @@ -1,3 +1,4 @@ +forklift_directory: "{{ playbook_dir | dirname | dirname }}" forklift_name: "pipe-{{ pipeline_type }}-{{ pipeline_version }}-{{ pipeline_os }}" forklift_server_name: "pipe-{{ pipeline_type }}-server-{{ pipeline_version }}-{{ pipeline_os }}" forklift_proxy_name: "pipe-{{ pipeline_type }}-proxy-{{ pipeline_version }}-{{ pipeline_os }}" diff --git a/pipelines/vars/upgrade_base.yml b/pipelines/vars/upgrade_base.yml index ddd13a9b1..165a3fe87 100644 --- a/pipelines/vars/upgrade_base.yml +++ b/pipelines/vars/upgrade_base.yml @@ -1,3 +1,4 @@ +forklift_directory: "{{ playbook_dir | dirname | dirname }}" forklift_name: "pipe-upgrade-{{ pipeline_type }}-{{ pipeline_version}}-{{ pipeline_os }}" forklift_server_name: "pipe-up-{{ pipeline_type }}-{{ pipeline_version}}-{{ pipeline_os }}" forklift_proxy_name: "pipe-up-{{ pipeline_type }}-proxy-{{ pipeline_version}}-{{ pipeline_os }}"