diff --git a/runner/master/jobtypes/phpunit/phpunit.sh b/runner/master/jobtypes/phpunit/phpunit.sh new file mode 100644 index 0000000..81c2024 --- /dev/null +++ b/runner/master/jobtypes/phpunit/phpunit.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# PHPUnit job type functions. + +# PHPUnit needed variables to go to the env file. +function phpunit_to_env_file() { + local env=( + DBTYPE + DBTAG + DBHOST + DBNAME + DBUSER + DBPASS + DBCOLLATION + DBREPLICAS + DBHOST_DBREPLICA + + BBBMOCKURL + MATRIXMOCKURL + + REDISTESTNAME + MEMCACHED1TESTURL + MEMCACHED2TESTURL + MONGODBTESTURL + + EXTTESTURL + LDAPTESTURL + SOLRTESTNAME + + MLBACKENDTESTNAME + ) + echo "${env[@]}" +} + +# PHPUnit information to be added to the summary. +function phpunit_to_summary() { + echo "== Moodle branch (version.php): ${MOODLE_BRANCH}" + echo "== PHP version: ${PHP_VERSION}" + echo "== DBTYPE: ${DBTYPE}" + echo "== DBTAG: ${DBTAG}" + echo "== DBREPLICAS: ${DBREPLICAS}" + echo "== MLBACKEND_PYTHON_VERSION: ${MLBACKEND_PYTHON_VERSION}" + echo "== RUNCOUNT: ${RUNCOUNT}" + echo "== PHPUNIT_FILTER: ${PHPUNIT_FILTER}" + echo "== PHPUNIT_TESTSUITE: ${PHPUNIT_TESTSUITE}" +} + +# This job type defines the following env variables +function phpunit_env() { + env=( + RUNCOUNT + PHPUNIT_FILTER + PHPUNIT_TESTSUITE + EXITCODE + ) + echo "${env[@]}" +} + +# PHPUnit needed modules. Note that the order is important. +function phpunit_modules() { + local modules=( + env + summary + docker + docker-logs + git + plugins + docker-database + docker-mocks + docker-caches + docker-exttests + docker-ldap + docker-solr + docker-mlbackend + docker-php + moodle-core-copy + docker-healthy + docker-summary + ) + echo "${modules[@]}" +} + +# PHPUnit job type checks. +function phpunit_check() { + # Check all module dependencies. + verify_modules $(phpunit_modules) + + # These env variables must be set for the job to work. + verify_env UUID ENVIROPATH WEBSERVER +} + +# PHPUnit job type init. +function phpunit_config() { + # Apply some defaults. + RUNCOUNT="${RUNCOUNT:-1}" + PHPUNIT_FILTER="${PHPUNIT_FILTER:-}" + PHPUNIT_TESTSUITE="${PHPUNIT_TESTSUITE:-}" + EXITCODE=0 +} + +# PHPUnit job type setup. +function phpunit_setup() { + # Init the PHPUnit site. + echo + echo ">>> startsection Initialising PHPUnit environment at $(date)<<<" + echo "============================================================================" + docker exec -t -u www-data "${WEBSERVER}" \ + php admin/tool/phpunit/cli/init.php \ + --force + echo "============================================================================" + echo ">>> stopsection <<<" +} + +# PHPUnit job type run. +function phpunit_run() { + # Run the job type. + echo + if [[ RUNCOUNT -gt 1 ]]; then + echo ">>> startsection Starting ${RUNCOUNT} PHPUnit runs at $(date) <<<" + else + echo ">>> startsection Starting PHPUnit run at $(date) <<<" + fi + echo "============================================================================" + # Build the complete command + local cmd=( + php vendor/bin/phpunit + --disallow-test-output + --fail-on-risky + --log-junit /shared/log.junit + --verbose + ) + if [[ -n "${PHPUNIT_FILTER}" ]]; then + cmd+=(--filter "${PHPUNIT_FILTER}") + fi + if [[ -n "${PHPUNIT_TESTSUITE}" ]]; then + cmd+=(--testsuite "${PHPUNIT_TESTSUITE}") + fi + + echo "Running: ${cmd[*]}" + + # Run the command RUNCOUNT times. + local iter=1 + while [[ ${iter} -le ${RUNCOUNT} ]]; do + echo + echo ">>> PHPUnit run ${iter} at $(date) <<<" + docker exec -t "${WEBSERVER}" "${cmd[@]}" + EXITCODE=$((EXITCODE + $?)) + iter=$((iter+1)) + done + + echo "============================================================================" + echo ">>> stopsection <<<" +} \ No newline at end of file diff --git a/runner/master/lib.sh b/runner/master/lib.sh new file mode 100644 index 0000000..19fb138 --- /dev/null +++ b/runner/master/lib.sh @@ -0,0 +1,436 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Some useful variables to control the orchestration. +# Already checked modules. +__CHECKED_MODULES=() + +# Various utility functions used by the CI scripts. + +# Verify that all the specified env variables are set. +function verify_env() { + local error= + local var= + for var in "$@"; do + if [[ ! -v "${var}" ]]; then + print_error "${var} is not set." + error=1 + fi + done + + if [[ -n ${error} ]]; then + exit 1 + fi +} + +# Verify that all the specified utilities are installed and available. +function verify_utilities() { + local error= + local util= + for util in "$@"; do + if ! command -v "${util}" > /dev/null 2>&1; then + print_error "${util} is not installed." + error=1 + fi + done + + if [[ -n ${error} ]]; then + exit 1 + fi +} + +# Verify that all the specified modules have been already added and checked. +function verify_modules() { + local error= + local module= + for module in "$@"; do + if ! in_array "${module}" "${__CHECKED_MODULES[@]}"; then + print_error "${module} module needs to be added before." + error=1 + fi + done + + if [[ -n ${error} ]]; then + exit 1 + fi +} + +# Print a error message to stdout. +function print_error() { + echo "ERROR: $*" +} + +# Print a warning message to stdout. +function print_warning() { + echo "WARNING: $*" +} + +# Print message to stdout and exit. +function exit_error() { + print_error "$@" + exit 1 +} + +# "catch-1" grep function, so it doesn't fail the script when there aren't any matches. +function c1grep() { + grep "$@" || test $? = 1 +} + +# reverse a string by words. +function string_reverse_by_words() { + # Using tac, reverse a string by words + echo "$1" | tac -s ' ' | tr '\n' ' ' +} + +# verify if a value is in array. +function in_array() { + local value=$1 + shift + local item= + local array=("$@") + for item in "${array[@]}"; do + if [[ ${item} == "${value}" ]]; then + return + fi + done + return 1 +} + +# Main run function to orchestrate and get the job type done. +# $1 - Job type to run. +function run() { + local jobtype=${1:-} + # Verify that the job type has been passed and that it exists. + if [[ -z ${jobtype} ]]; then + exit_error "No job type specified." + fi + if [[ ! -f "${BASEDIR}/jobtypes/${jobtype}/${jobtype}.sh" ]]; then + exit_error "Job type ${1} does not exist." + fi + + # Source the job type. + # shellcheck source=jobtypes/phpunit.sh # (so we have a reliable job type for other checks) + source "${BASEDIR}/jobtypes/${jobtype}/${jobtype}.sh" + + # Setup job env variables (modules may need them). + run_job_env "${jobtype}" + + # Setup all modules env variables and run their checks. One by one. + run_modules_env_and_check "${jobtype}" + + # Run job checks (after modules env variables have been set). + run_job_check "${jobtype}" + + # Now it's time to run all the modules config functions (they are optional). + run_modules_config "${jobtype}" + run_job_config "${jobtype}" + + # Now it's time to run all the modules setup functions (they are optional). + run_modules_setup "${jobtype}" + + # Now it's time to run the job setup function (it's optional). + run_job_setup "${jobtype}" + + # We always run the job with exit on error disabled, so the job can manage the exit codes by itself. + set +e + run_job_run "${jobtype}" + set -e + + # Now it's time to run the job and all the modules teardown functions (they are optional) + # but we aren't doing that here because it's controlled by traps. + # Note that they are executed in reverse order, first the job one and, then the + # modules ones (in the opposite order they were setup). All them are optional. +} + +# Get all the environment variables that the job type needs to be exported to env. file +# (the env module uses it to generate the env file). +function get_job_to_env_file() { + local jobtype=${1:-} + # Verify that the job type has been passed. + if [[ -z ${jobtype} ]]; then + exit_error "No job type specified." + fi + # The job type env function is optional, skip if not present. + if ! type "${jobtype}_to_env_file" > /dev/null 2>&1; then + return + fi + "${jobtype}_to_env_file" +} + +# Get all the information that the job wants to add to the summary. +# (the summary module uses it to generate the summary). +function get_job_to_summary() { + local jobtype=${1:-} + # Verify that the job type has been passed. + if [[ -z ${jobtype} ]]; then + exit_error "No job type specified." + fi + # The job type summary function is optional, skip if not present. + if ! type "${jobtype}_to_summary" > /dev/null 2>&1; then + return + fi + "${jobtype}_to_summary" +} + +# Set module variables and run their checks. +# +# Register all job variables (and accumulate them in the __ENV_VARIABLES array) +# and, immediately, run its checks. +function run_job_env() { + local jobtype=${1:-} + local exitcode= + local hasenv= + # Verify that the job type has been passed. + if [[ -z ${jobtype} ]]; then + exit_error "No job type specified." + fi + # The job must have an env function. + if ! type "${jobtype}_env" > /dev/null 2>&1; then + exit_error "Job ${jobtype} does not have an env function." + fi + # Create (as global) and accumulate the env variables. + for variable in $("${jobtype}_env"); do + if [[ "${variable}" == "EXITCODE" ]]; then + exitcode=1 + fi + declare -g "${variable}=${!variable:-}" + __ENV_VARIABLES+=("${variable}") + hasenv=1 + done + # Ensure that the job always has the EXITCODE env variable. + if [[ -z ${exitcode} ]]; then + exit_error "Job ${jobtype} does not have the EXITCODE env variable." + fi + # If the job had env variables, it must have a config function. + if [[ -n ${hasenv} ]] && ! type "${jobtype}_config" > /dev/null 2>&1; then + exit_error "Job ${jobtype} has env variables and is missing the config function." + fi +} + +# Run job checks. +function run_job_check() { + local jobtype=${1:-} + local exitcode= + # Verify that the job type has been passed. + if [[ -z ${jobtype} ]]; then + exit_error "No job type specified." + fi + # The job must have a check function. + if ! type "${jobtype}_check" > /dev/null 2>&1; then + exit_error "Job ${jobtype} does not have a check function." + fi + # Run the job type check. + echo -n "Checking ${jobtype} job... " + "${jobtype}_check" + echo "OK" +} + +# Execute the config method for the job type being executed. +function run_job_config() { + local jobtype=${1:-} + # Verify that the job type has been passed. + if [[ -z ${jobtype} ]]; then + exit_error "No job type specified." + fi + # The config function is optional, skip if not present. + if ! type "${jobtype}_config" > /dev/null 2>&1; then + return + fi + echo "Configuring ${jobtype} job... " + "${jobtype}_config" +} + +# Execute the before_run method for the job type being executed. +function run_job_setup() { + local jobtype=${1:-} + # Verify that the job type has been passed. + if [[ -z ${jobtype} ]]; then + exit_error "No job type specified." + fi + # The setup function is optional, skip if not present. + if ! type "${jobtype}_setup" > /dev/null 2>&1; then + return + fi + echo "Setting up ${jobtype} job... " + "${jobtype}_setup" +} + +# Execute the run method for the job type being executed. +function run_job_run() { + local jobtype=${1:-} + # Verify that the job type has been passed. + if [[ -z ${jobtype} ]]; then + exit_error "No job type specified." + fi + # The job must have a run function. + if ! type "${jobtype}_run" > /dev/null 2>&1; then + exit_error "Job ${jobtype} does not have a run function." + fi + echo "Running ${jobtype} job... " + "${jobtype}_run" +} + +# Execute the teardown method for the job type being executed. +function run_job_teardown() { + local jobtype=${1:-} + # Verify that the job type has been passed. + if [[ -z ${jobtype} ]]; then + exit_error "No job type specified." + fi + # The teardown function is optional, skip if not present. + if ! type "${jobtype}_teardown" > /dev/null 2>&1; then + return 0 # This is executed by the trap, so we need to specify the exit code to allow it to continue. + fi + echo "Finishing ${jobtype} job... " + "${jobtype}_teardown" +} + +# Set module variables and run their checks. +# +# Register all modules variables (and accumulate them in the __ENV_VARIABLES array) +# and, immediately, run their checks. +function run_modules_env_and_check() { + local module + + for module in $("${jobtype}_modules"); do + local variable= + local hasenv= + + # Check if the module exists. + if [[ ! -f "${BASEDIR}/modules/${module}/${module}.sh" ]]; then + exit_error "Module ${module} does not exist." + fi + # shellcheck source=modules/docker/docker.sh # (so we have a reliable module for other checks) + source "${BASEDIR}/modules/${module}/${module}.sh" + + # All modules must have an env function. + if ! type "${module}_env" > /dev/null 2>&1; then + exit_error "Module ${module} does not have an env function." + fi + # Create (as global) and accumulate the env variables. + for variable in $("${module}_env"); do + declare -g "${variable}=${!variable:-}" + __ENV_VARIABLES+=("${variable}") + hasenv=1 + done + + # If the module had env variables, it must have a config function. + if [[ -n ${hasenv} ]] && ! type "${module}_config" > /dev/null 2>&1; then + exit_error "Module ${module} has env variables and is missing the config function." + fi + + # All modules must have a check function. + if ! type "${module}_check" > /dev/null 2>&1; then + exit_error "Module ${module} does not have a check function." + fi + # Run the module check. + echo -n "Checking ${module} module... " + "${module}_check" + __CHECKED_MODULES+=("${module}") + echo "OK" + done +} + +# Execute all the config functions for the modules that the job type needs (they are optional). +function run_modules_config() { + local module= + for module in $("${jobtype}_modules"); do + # Check if the module exists. + if [[ ! -f "${BASEDIR}/modules/${module}/${module}.sh" ]]; then + exit_error "Module ${module} does not exist." + fi + # shellcheck source=modules/docker/docker.sh # (so we have a reliable module for other checks) + source "${BASEDIR}/modules/${module}/${module}.sh" + # The config function is optional, skip if not present. + if ! type "${module}_config" > /dev/null 2>&1; then + continue + fi + echo "Configuring ${module} module... " + "${module}_config" + done +} + +# Execute all the setup functions for the modules that the job type needs (they are optional). +function run_modules_setup() { + local module= + for module in $("${jobtype}_modules"); do + # Check if the module exists. + if [[ ! -f "${BASEDIR}/modules/${module}/${module}.sh" ]]; then + exit_error "Module ${module} does not exist." + fi + # shellcheck source=modules/docker/docker.sh # (so we have a reliable module for other checks) + source "${BASEDIR}/modules/${module}/${module}.sh" + # The before_run function is optional, skip if not present. + if ! type "${module}_setup" > /dev/null 2>&1; then + continue + fi + echo "Setting up ${module} module... " + "${module}_setup" + done +} + +# Execute all the teardown functions for the modules that the job type needs (they are optional). +function run_modules_teardown() { + local module= + local modules= + local reversed_modules= + local jobtype=${1:-} + # Verify that the job type has been passed. + if [[ -z ${jobtype} ]]; then + exit_error "No job type specified." + fi + # The teardown module functions are executed in reverse order. + modules=$("${jobtype}_modules") + reversed_modules="$(string_reverse_by_words "${modules}")" + for module in ${reversed_modules}; do + # Check if the module exists. + if [[ ! -f "${BASEDIR}/modules/${module}/${module}.sh" ]]; then + exit_error "Module ${module} does not exist." + fi + # shellcheck source=modules/docker/docker.sh # (so we have a reliable module for other checks) + source "${BASEDIR}/modules/${module}/${module}.sh" + # The teardown function is optional, skip if not present. + if ! type "${module}_teardown" > /dev/null 2>&1; then + continue + fi + echo "Finishing ${module} module... " + "${module}_teardown" + done +} + +# To trap any exit gracefully. +function trap_exit() { + local exitcode=$? + run_job_teardown "${JOBTYPE}" + run_modules_teardown "${JOBTYPE}" + echo + echo "============================================================================" + echo "== Exit summary": + echo "== Job type: ${JOBTYPE}" + echo "== Date: $(date)" + echo "== Exit code: ${exitcode}" + echo "============================================================================" +} + +# To trap Crtl-C and exit gracefully. +function trap_ctrl_c() { + echo + echo "============================================================================" + echo "Job was cancelled at user request" + echo "============================================================================" + exit 255 +} diff --git a/runner/master/modules/docker-caches/docker-caches.sh b/runner/master/modules/docker-caches/docker-caches.sh new file mode 100644 index 0000000..ac66aca --- /dev/null +++ b/runner/master/modules/docker-caches/docker-caches.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Caches module functions. + +# This module defines the following env variables. +function docker-caches_env() { + env=( + REDISTESTNAME + MEMCACHED1TESTURL + MEMCACHED2TESTURL + MONGODBTESTURL + ) + echo "${env[@]}" +} + +# Caches module checks. +function docker-caches_check() { + # Check all module dependencies. + verify_modules docker + + # These env variables must be set for the database module to work. + verify_env NETWORK UUID +} + +# Caches module config. +function docker-caches_config() { + # Apply some defaults. + REDISTESTNAME=redis"${UUID}" + MEMCACHED1TESTURL=memcached1"${UUID}" + MEMCACHED2TESTURL=memcached2"${UUID}" + MONGODB=mongodb"${UUID}" + MONGODBTESTURL="mongodb://${MONGODB}:27017" +} + +# Caches module setup, launch the containers. +function docker-caches_setup() { + echo + echo ">>> startsection Starting caching services <<<" + echo "============================================================================" + + # Start the Redis server. + docker run \ + --detach \ + --name "${REDISTESTNAME}" \ + --network "${NETWORK}" \ + redis:3 + echo "Redis URL: ${REDISTESTNAME}" + echo "Redis logs:" + docker logs "${REDISTESTNAME}" + + echo + + # Start the Memcached servers + docker run \ + --detach \ + --name "${MEMCACHED1TESTURL}" \ + --network "${NETWORK}" \ + memcached:1.4 + echo "Memcached 1 URL: ${MEMCACHED1TESTURL}" + echo "Memcached 1 logs:" + docker logs "${MEMCACHED1TESTURL}" + + echo + + docker run \ + --detach \ + --name "${MEMCACHED2TESTURL}" \ + --network "${NETWORK}" \ + memcached:1.4 + echo "Memcached 2 URL: ${MEMCACHED2TESTURL}" + echo "Memcached 2 logs:" + docker logs "${MEMCACHED2TESTURL}" + + echo + + # TODO: We only need this for Moodle <= 4.1 (401). See MDL-77163. + docker run \ + --detach \ + --name "${MONGODB}" \ + --network "${NETWORK}" \ + mongo:4.0 + echo "MongoDB URL: ${MONGODBTESTURL}" + echo "MongoDB logs:" + docker logs "${MONGODB}" + + echo "============================================================================" + echo ">>> stopsection <<<" +} diff --git a/runner/master/modules/docker-database/docker-database.sh b/runner/master/modules/docker-database/docker-database.sh new file mode 100644 index 0000000..d3f0291 --- /dev/null +++ b/runner/master/modules/docker-database/docker-database.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Database module functions. + +# This module defines the following env variables. +function docker-database_env() { + env=( + DBTYPE + DBTAG + DBREPLICAS + DBHOST + DBNAME + DBUSER + DBPASS + DBCOLLATION + DBHOST_DBREPLICA + ) + echo "${env[@]}" +} + +# Database module checks. +function docker-database_check() { + # Check all module dependencies. + verify_modules docker + + # These env variables must be set for the database module to work. + verify_env NETWORK UUID +} + +# Database module init. +function docker-database_config() { + # Apply some defaults. + DBTYPE="${DBTYPE:-pgsql}" + DBTAG="${DBTAG:-auto}" + DBREPLICAS="${DBREPLICAS:-0}" + DBHOST=database"${UUID}" + DBNAME="${DBNAME:-moodle}" + DBUSER="${DBUSER:-moodle}" + DBPASS="${DBPASS:-moodle}" + DBCOLLATION= + if [[ "${DBREPLICAS}" -ne 0 ]]; then + DBHOST_DBREPLICA="database_replica${UUID}" + fi + + # Let's decide the DBTAG to use. + database_apply_default_dbtag + + # Let's apply some configuration depending on DBTYPE / DBTAG... + database_apply_config +} + +# Database module setup, apply for the correct db tags and launch the containers. +function docker-database_setup() { + + # We are ready to launch the database containers. Each DBTYPE has its own way to be launched. + local dbscript="${BASEDIR}/modules/docker-database/${DBTYPE}.d/${DBTYPE}_init.sh" + if [[ ! -f "${dbscript}" ]]; then + exit_error "Wrong DBTYPE: ${DBTYPE}. The ${dbscript} script does not exist." + fi + # shellcheck source=modules/docker-database/mysqli.d/pgsql_config.sh # (so we have a reliable database for other checks) + source "${dbscript}" + # Function to be executed to launch the database containers. + local dbinitfunc="${DBTYPE}_config_standalone" + # Replicas use a different one + if [[ "${DBREPLICAS}" -ne 0 ]]; then + dbinitfunc="${DBTYPE}_config_with_replicas" + fi + # The function must exist. + if ! type "${dbinitfunc}" > /dev/null 2>&1; then + exit_error "Database ${DBTYPE} does not have a ${dbinitfunc} function (at ${DBTYPE}.d/${DBTYPE}_config.sh)." + fi + + # Launch the database containers. + echo ">>> startsection Starting database <<<" + echo "============================================================================" + "${dbinitfunc}" + echo "============================================================================" + echo ">>> stopsection <<<" + + # Print the database logs + echo + echo ">>> startsection Database summary <<<" + echo "============================================================================" + echo "== DBTYPE: ${DBTYPE}" + echo "== DBTAG: ${DBTAG}" + echo "== DBHOST: ${DBHOST}" + echo "== DBNAME: ${DBNAME}" + echo "== DBUSER: ${DBUSER}" + echo "== DBPASS: ${DBPASS}" + echo "== DBCOLLATION: ${DBCOLLATION}" + echo "== DBREPLICAS: ${DBREPLICAS}" + if [[ -n "${DBHOST_DBREPLICA}" ]]; then + echo "== DBHOST_DBREPLICA: ${DBHOST_DBREPLICA}" + fi + echo + echo "Database logs:" + docker logs "${DBHOST}" + + if [ "${DBHOST_DBREPLICA}" != "" ] + then + echo + echo "Database replica logs:" + docker logs "${DBHOST_DBREPLICA}" + fi + + echo "============================================================================" + echo ">>> stopsection <<<" +} + +# Let's decide the default DBTAG to use if none was specified. +function database_apply_default_dbtag() { + # Here it's where we pin any DBTAG docker tag (versions), when needed. Don't change it elsewhere. + # We only apply these pinned defaults when no DBTAG has been explicitly defined. And we only apply + # them to databases know to need them (some bug prevents to use "latest"). Every pinned case should + # include a comment with the reason for it. + if [[ "${DBTAG}" == "auto" ]]; then + case ${DBTYPE} in + mariadb) + DBTAG=10.7 # Because there is a problem with the >= 10.8 images not working with older hosts OS. + ;; + mysqli) + DBTAG=8.0 # Because it's the primary lowest supported version and we need it covered by default. + ;; + sqlsrv) + DBTAG=latest # No pin, right now 2019-latest + ;; + oci) + DBTAG=latest # No pin, right now this is 21c + ;; + pgsql) + DBTAG=13 # Because it's the primary lowest supported version and we need it covered by default. + ;; + *) + exit_error "Wrong DBTYPE: ${DBTYPE}. Fix it, or add support for that DBTYPE above" + ;; + esac + fi +} + +# Every DBTYPE / DBTAG ... combination may need some specific configuration. +function database_apply_config() { + case ${DBTYPE} in + mariadb) + # MariaDB needs a collation. + DBCOLLATION="${DBCOLLATION:-utf8mb4_unicode_ci}" + ;; + mysqli) + # MySQLi needs a collation. + DBCOLLATION="${DBCOLLATION:-utf8mb4_unicode_ci}" + ;; + sqlsrv) + # These are the only ones working with our image. + DBUSER="sa" + DBPASS="Passw0rd!" + ;; + oci) + # These are the only ones working with our image. + DBPASS="m@0dl3ing" + DBNAME="XE" + if [[ "${DBTAG}" == "23" ]]; then + # The Oracle 23 image is using the FREE database instead of the XE one. + DBNAME="FREE" + fi + ;; + pgsql) + ;; + *) + exit_error "Wrong DBTYPE: ${DBTYPE}. Fix it, or add support for that DBTYPE above" + ;; + esac +} \ No newline at end of file diff --git a/runner/master/modules/docker-database/mariadb.d/mariadb_init.sh b/runner/master/modules/docker-database/mariadb.d/mariadb_init.sh new file mode 100644 index 0000000..08ad10f --- /dev/null +++ b/runner/master/modules/docker-database/mariadb.d/mariadb_init.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Functions needed to init the database. +# This database supports both standalone and primary-replica setups. + +# Init a standalone database container. Without replicas. +function mariadb_config_standalone() { + echo "Starting standalone database..." + docker run \ + --detach \ + --name "${DBHOST}" \ + --network "${NETWORK}" \ + -e MYSQL_ROOT_PASSWORD="${DBPASS}" \ + -e MYSQL_DATABASE="${DBNAME}" \ + -e MYSQL_USER="${DBUSER}" \ + -e MYSQL_PASSWORD="${DBPASS}" \ + --tmpfs /var/lib/mysql:rw,noexec,nosuid,size=1024m \ + -v "${BASEDIR}/modules/docker-database/mariadb.d/standalone/conf.d:/etc/mysql/conf.d" \ + mariadb:"${DBTAG}" + + # Wait few secs, before executing commands. + # TODO: Find a better way to wait for the database to be ready. + sleep 20 +} + +# Init a primary database container. With replicas. +function mariadb_config_with_replicas() { + echo "Starting primary database..." + docker run \ + --detach \ + --name "${DBHOST}" \ + --network "${NETWORK}" \ + -e MYSQL_ROOT_PASSWORD="${DBPASS}" \ + -e MYSQL_DATABASE="${DBNAME}" \ + -e MYSQL_USER="${DBUSER}" \ + -e MYSQL_PASSWORD="${DBPASS}" \ + -e DBHOST_DBREPLICA="${DBHOST_DBREPLICA}" \ + --tmpfs /var/lib/mysql:rw \ + -v "${BASEDIR}/modules/docker-database/mariadb.d/primary/conf.d:/etc/mysql/conf.d" \ + -v "${BASEDIR}/modules/docker-database/mariadb.d/primary/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d" \ + mariadb:"${DBTAG}" + + # Wait few secs, before executing commands. + # TODO: Find a better way to wait for the database to be ready. + sleep 20 + + echo "Starting replica database..." + docker run \ + --detach \ + --name "${DBHOST_DBREPLICA}" \ + --network "${NETWORK}" \ + -e MYSQL_ROOT_PASSWORD="${DBPASS}" \ + -e MYSQL_DATABASE="${DBNAME}" \ + -e MYSQL_USER="${DBUSER}" \ + -e MYSQL_PASSWORD="${DBPASS}" \ + -e DBHOST="${DBHOST}" \ + -e DBHOST_DBREPLICA="${DBHOST_DBREPLICA}" \ + -v "${BASEDIR}/modules/docker-database/mariadb.d/replica/conf.d:/etc/mysql/conf.d" \ + -v "${BASEDIR}/modules/docker-database/mariadb.d/replica/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d" \ + --tmpfs /var/lib/mysql:rw \ + mariadb:"${DBTAG}" + + # Wait few secs, before executing commands. + # TODO: Find a better way to wait for the database to be ready. + sleep 20 +} \ No newline at end of file diff --git a/runner/master/mariadb.d/master/conf.d/master.cnf b/runner/master/modules/docker-database/mariadb.d/primary/conf.d/primary.cnf similarity index 100% rename from runner/master/mariadb.d/master/conf.d/master.cnf rename to runner/master/modules/docker-database/mariadb.d/primary/conf.d/primary.cnf diff --git a/runner/master/mariadb.d/master/docker-entrypoint-initdb.d/01_privileges.sql b/runner/master/modules/docker-database/mariadb.d/primary/docker-entrypoint-initdb.d/01_privileges.sql similarity index 100% rename from runner/master/mariadb.d/master/docker-entrypoint-initdb.d/01_privileges.sql rename to runner/master/modules/docker-database/mariadb.d/primary/docker-entrypoint-initdb.d/01_privileges.sql diff --git a/runner/master/mariadb.d/slave/conf.d/moodle.cnf b/runner/master/modules/docker-database/mariadb.d/replica/conf.d/moodle.cnf similarity index 100% rename from runner/master/mariadb.d/slave/conf.d/moodle.cnf rename to runner/master/modules/docker-database/mariadb.d/replica/conf.d/moodle.cnf diff --git a/runner/master/mariadb.d/slave/conf.d/slave.cnf b/runner/master/modules/docker-database/mariadb.d/replica/conf.d/replica.cnf similarity index 100% rename from runner/master/mariadb.d/slave/conf.d/slave.cnf rename to runner/master/modules/docker-database/mariadb.d/replica/conf.d/replica.cnf diff --git a/runner/master/mysql.d/slave/docker-entrypoint-initdb.d/10_replicate.sh b/runner/master/modules/docker-database/mariadb.d/replica/docker-entrypoint-initdb.d/10_replicate.sh similarity index 89% rename from runner/master/mysql.d/slave/docker-entrypoint-initdb.d/10_replicate.sh rename to runner/master/modules/docker-database/mariadb.d/replica/docker-entrypoint-initdb.d/10_replicate.sh index d1b99dc..282f4af 100755 --- a/runner/master/mysql.d/slave/docker-entrypoint-initdb.d/10_replicate.sh +++ b/runner/master/modules/docker-database/mariadb.d/replica/docker-entrypoint-initdb.d/10_replicate.sh @@ -16,21 +16,21 @@ mysqldump -u root -pmoodle -h ${DBHOST} --opt moodle > /tmp/moodle.sql mysql -u root -pmoodle -h ${DBHOST} -e "UNLOCK TABLES;" moodle echo "Done" -echo "Master status:" +echo "Primary status:" mysql -u root -pmoodle -h ${DBHOST} -e "SHOW MASTER STATUS;" moodle position=`mysql -u root -pmoodle -h ${DBHOST} -e "SHOW MASTER STATUS;" moodle | grep 'mysql-bin' | awk '{print $2}'` replfile=`mysql -u root -pmoodle -h ${DBHOST} -e "SHOW MASTER STATUS;" moodle | grep 'mysql-bin' | awk '{print $1}'` -echo "Master dump complete" +echo "Primary dump complete" echo "Current position is {$replfile} {$position}" -echo "Restoring into client" +echo "Restoring into replica" mysql -u root -pmoodle moodle < /tmp/moodle.sql mysql -u root -pmoodle moodle << EOSQL CHANGE MASTER TO MASTER_HOST='${DBHOST}', - MASTER_USER='replication', - MASTER_PASSWORD='replication', + MASTER_USER='replica', + MASTER_PASSWORD='replica', MASTER_LOG_FILE='$replfile', MASTER_LOG_POS=$position; START SLAVE; @@ -41,4 +41,4 @@ sleep 5 mysql -u root -pmoodle moodle << EOSQL SHOW SLAVE STATUS\G -EOSQL +EOSQL \ No newline at end of file diff --git a/runner/master/mariadb.d/standalone/conf.d/moodle.cnf b/runner/master/modules/docker-database/mariadb.d/standalone/conf.d/moodle.cnf similarity index 100% rename from runner/master/mariadb.d/standalone/conf.d/moodle.cnf rename to runner/master/modules/docker-database/mariadb.d/standalone/conf.d/moodle.cnf diff --git a/runner/master/modules/docker-database/mysqli.d/mysqli_init.sh b/runner/master/modules/docker-database/mysqli.d/mysqli_init.sh new file mode 100644 index 0000000..7a800f6 --- /dev/null +++ b/runner/master/modules/docker-database/mysqli.d/mysqli_init.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Functions needed to init the database. +# This database supports both standalone and primary-replica setups. + +# Init a standalone database container. Without replicas. +function mysqli_config_standalone() { + echo "Starting standalone database..." + docker run \ + --detach \ + --name "${DBHOST}" \ + --network "${NETWORK}" \ + -e MYSQL_DATABASE="${DBNAME}" \ + -e MYSQL_ROOT_PASSWORD="${DBPASS}" \ + -e MYSQL_USER="${DBUSER}" \ + -e MYSQL_PASSWORD="${DBPASS}" \ + --tmpfs /var/lib/mysql:rw,noexec,nosuid,size=1024m \ + -v "${BASEDIR}/modules/docker-database/mysqli.d/standalone/conf.d:/etc/mysql/conf.d" \ + mysql:"${DBTAG}" + + # Wait few secs, before executing commands. + # TODO: Find a better way to wait for the database to be ready. + sleep 20 +} + +# Init a primary database container. With replicas. +function mysqli_config_with_replicas() { + echo "Starting primary database..." + docker run \ + --detach \ + --name "${DBHOST}" \ + --network "${NETWORK}" \ + -e MYSQL_DATABASE="${DBNAME}" \ + -e MYSQL_ROOT_PASSWORD="${DBPASS}" \ + -e MYSQL_USER="${DBUSER}" \ + -e MYSQL_PASSWORD="${DBPASS}" \ + -e DBHOST_DBREPLICA="${DBHOST_DBREPLICA}" \ + --tmpfs /var/lib/mysql:rw \ + -v "${BASEDIR}/modules/docker-database/mysqli.d/primary/conf.d:/etc/mysql/conf.d" \ + -v "${BASEDIR}/modules/docker-database/mysqli.d/primary/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d" \ + mysql:"${DBTAG}" + + # Wait few secs, before executing commands. + # TODO: Find a better way to wait for the database to be ready. + sleep 20 + + echo "Starting replica database..." + docker run \ + --detach \ + --name "${DBHOST_DBREPLICA}" \ + --network "${NETWORK}" \ + -e MYSQL_DATABASE="${DBNAME}" \ + -e MYSQL_ROOT_PASSWORD="${DBPASS}" \ + -e MYSQL_USER="${DBUSER}" \ + -e MYSQL_PASSWORD="${DBPASS}" \ + -e DBHOST="${DBHOST}" \ + -e DBHOST_DBREPLICA="${DBHOST_DBREPLICA}" \ + -v "${BASEDIR}/modules/docker-database/mysqli.d/replica/conf.d:/etc/mysql/conf.d" \ + -v "${BASEDIR}/modules/docker-database/mysqli.d/replica/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d" \ + --tmpfs /var/lib/mysql:rw \ + mysql:"${DBTAG}" + + # Wait few secs, before executing commands. + # TODO: Find a better way to wait for the database to be ready. + sleep 20 +} \ No newline at end of file diff --git a/runner/master/mysql.d/master/conf.d/master.cnf b/runner/master/modules/docker-database/mysqli.d/primary/conf.d/primary.cnf similarity index 100% rename from runner/master/mysql.d/master/conf.d/master.cnf rename to runner/master/modules/docker-database/mysqli.d/primary/conf.d/primary.cnf diff --git a/runner/master/mysql.d/master/docker-entrypoint-initdb.d/01_privileges.sql b/runner/master/modules/docker-database/mysqli.d/primary/docker-entrypoint-initdb.d/01_privileges.sql similarity index 100% rename from runner/master/mysql.d/master/docker-entrypoint-initdb.d/01_privileges.sql rename to runner/master/modules/docker-database/mysqli.d/primary/docker-entrypoint-initdb.d/01_privileges.sql diff --git a/runner/master/mysql.d/slave/conf.d/moodle.cnf b/runner/master/modules/docker-database/mysqli.d/replica/conf.d/moodle.cnf similarity index 100% rename from runner/master/mysql.d/slave/conf.d/moodle.cnf rename to runner/master/modules/docker-database/mysqli.d/replica/conf.d/moodle.cnf diff --git a/runner/master/mysql.d/slave/conf.d/slave.cnf b/runner/master/modules/docker-database/mysqli.d/replica/conf.d/replica.cnf similarity index 100% rename from runner/master/mysql.d/slave/conf.d/slave.cnf rename to runner/master/modules/docker-database/mysqli.d/replica/conf.d/replica.cnf diff --git a/runner/master/mariadb.d/slave/docker-entrypoint-initdb.d/10_replicate.sh b/runner/master/modules/docker-database/mysqli.d/replica/docker-entrypoint-initdb.d/10_replicate.sh similarity index 94% rename from runner/master/mariadb.d/slave/docker-entrypoint-initdb.d/10_replicate.sh rename to runner/master/modules/docker-database/mysqli.d/replica/docker-entrypoint-initdb.d/10_replicate.sh index d1b99dc..7bc388d 100755 --- a/runner/master/mariadb.d/slave/docker-entrypoint-initdb.d/10_replicate.sh +++ b/runner/master/modules/docker-database/mysqli.d/replica/docker-entrypoint-initdb.d/10_replicate.sh @@ -16,14 +16,14 @@ mysqldump -u root -pmoodle -h ${DBHOST} --opt moodle > /tmp/moodle.sql mysql -u root -pmoodle -h ${DBHOST} -e "UNLOCK TABLES;" moodle echo "Done" -echo "Master status:" +echo "Primary status:" mysql -u root -pmoodle -h ${DBHOST} -e "SHOW MASTER STATUS;" moodle position=`mysql -u root -pmoodle -h ${DBHOST} -e "SHOW MASTER STATUS;" moodle | grep 'mysql-bin' | awk '{print $2}'` replfile=`mysql -u root -pmoodle -h ${DBHOST} -e "SHOW MASTER STATUS;" moodle | grep 'mysql-bin' | awk '{print $1}'` -echo "Master dump complete" +echo "Primary dump complete" echo "Current position is {$replfile} {$position}" -echo "Restoring into client" +echo "Restoring into replica" mysql -u root -pmoodle moodle < /tmp/moodle.sql mysql -u root -pmoodle moodle << EOSQL diff --git a/runner/master/mysql.d/standalone/conf.d/moodle.cnf b/runner/master/modules/docker-database/mysqli.d/standalone/conf.d/moodle.cnf similarity index 100% rename from runner/master/mysql.d/standalone/conf.d/moodle.cnf rename to runner/master/modules/docker-database/mysqli.d/standalone/conf.d/moodle.cnf diff --git a/runner/master/modules/docker-database/oci.d/oci_init.sh b/runner/master/modules/docker-database/oci.d/oci_init.sh new file mode 100644 index 0000000..626e12c --- /dev/null +++ b/runner/master/modules/docker-database/oci.d/oci_init.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Functions needed to init the database. +# This database only supports standalone setups, not primary-replica ones. + +# Init a standalone database container. Without replicas. +function oci_config_standalone() { + echo "Starting standalone database..." + # Need to adjust how we use tmpfs database depending on the database tag. + local tmpfsinit=() + local tmpfsmount=() + if [[ "${DBTAG}" == "11" ]]; then + tmpfsinit=( + "-v" + "${BASEDIR}/modules/docker-database/oci.d/tmpfs.sh:/docker-entrypoint-initdb.d/tmpfs.sh" + ) + tmpfsmount=( + "--tmpfs" + "/var/lib/oracle" + "--shm-size" + "2g" + ) + else + # Let's mount the whole (XE/FREE) data directory using tmpfs and + # use it. Mounting individual databases doesn't work because of + # a recent change in the upstream images from zip to 7z, later + # causing issues with the database creation. + # See https://github.com/gvenzl/oci-oracle-xe/issues/202 + # Until then we'll be mounting the whole data directory. + tmpfsmount=( + "--mount" + "type=tmpfs,destination=/opt/oracle/oradata" + "--shm-size" + "6g" + ) + fi + + docker run \ + --detach \ + --name "${DBHOST}" \ + --network "${NETWORK}" \ + "${tmpfsinit[@]}" "${tmpfsmount[@]}" \ + -e ORACLE_DISABLE_ASYNCH_IO=true \ + moodlehq/moodle-db-oracle-r2:"${DBTAG}" + + # Wait few secs, before executing commands. + # TODO: Find a better way to wait for the database to be ready. + sleep 140 +} \ No newline at end of file diff --git a/runner/master/oracle.d/tmpfs.sh b/runner/master/modules/docker-database/oci.d/tmpfs.sh similarity index 100% rename from runner/master/oracle.d/tmpfs.sh rename to runner/master/modules/docker-database/oci.d/tmpfs.sh diff --git a/runner/master/modules/docker-database/pgsql.d/pgsql_init.sh b/runner/master/modules/docker-database/pgsql.d/pgsql_init.sh new file mode 100644 index 0000000..1c9e31d --- /dev/null +++ b/runner/master/modules/docker-database/pgsql.d/pgsql_init.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Functions needed to init the database. +# This database supports both standalone and primary-replica setups. + +# Init a standalone database container. Without replicas. +function pgsql_config_standalone() { + echo "Starting standalone database..." + docker run \ + --detach \ + --name "${DBHOST}" \ + --network "${NETWORK}" \ + -e POSTGRES_DB="${DBNAME}" \ + -e POSTGRES_USER=moodle \ + -e POSTGRES_PASSWORD=moodle \ + --tmpfs /var/lib/postgresql/data:rw,noexec,nosuid,size=1024m \ + -v "${BASEDIR}/modules/docker-database/pgsql.d/standalone:/docker-entrypoint-initdb.d" \ + postgres:"${DBTAG}" + + # Wait few secs, before executing commands. + # TODO: Find a better way to wait for the database to be ready. + sleep 10 +} + +# Init a primary database container. With replicas. +function pgsql_config_with_replicas() { + echo "Starting primary database..." + docker run \ + --detach \ + --name "${DBHOST}" \ + --network "${NETWORK}" \ + -e POSTGRES_DB="${DBNAME}" \ + -e POSTGRES_USER=moodle \ + -e POSTGRES_PASSWORD=moodle \ + -e DBHOST_DBREPLICA="${DBHOST_DBREPLICA}" \ + --tmpfs /var/lib/postgresql/data:rw \ + -v "${BASEDIR}/modules/docker-database/pgsql.d/primary:/docker-entrypoint-initdb.d" \ + postgres:"${DBTAG}" + + # Wait few secs, before executing commands. + # TODO: Find a better way to wait for the database to be ready. + sleep 10 + + echo "Starting replica database..." + docker run \ + --detach \ + --name "${DBHOST_DBREPLICA}" \ + --network "${NETWORK}" \ + -e POSTGRES_DB="${DBNAME}" \ + -e POSTGRES_USER=moodle \ + -e POSTGRES_PASSWORD=moodle \ + -e DBHOST="${DBHOST}" \ + -e DBHOST_DBREPLICA="${DBHOST_DBREPLICA}" \ + --tmpfs /var/lib/postgresql/data:rw \ + -v "${BASEDIR}/modules/docker-database/pgsql.d/replica:/docker-entrypoint-initdb.d" \ + postgres:"${DBTAG}" + + # Wait few secs, before executing commands. + # TODO: Find a better way to wait for the database to be ready. + sleep 10 +} \ No newline at end of file diff --git a/runner/master/pgsql.d/master/hba.sh b/runner/master/modules/docker-database/pgsql.d/primary/hba.sh similarity index 100% rename from runner/master/pgsql.d/master/hba.sh rename to runner/master/modules/docker-database/pgsql.d/primary/hba.sh diff --git a/runner/master/pgsql.d/master/moodle.sql b/runner/master/modules/docker-database/pgsql.d/primary/moodle.sql similarity index 89% rename from runner/master/pgsql.d/master/moodle.sql rename to runner/master/modules/docker-database/pgsql.d/primary/moodle.sql index 9150c16..a0bd2f1 100755 --- a/runner/master/pgsql.d/master/moodle.sql +++ b/runner/master/modules/docker-database/pgsql.d/primary/moodle.sql @@ -1,4 +1,4 @@ ---- Tuning the master server for performance +--- Tuning the primary server for performance ALTER SYSTEM SET shared_buffers TO '2GB'; ALTER SYSTEM SET work_mem TO '128MB'; ALTER SYSTEM SET maintenance_work_mem TO '256MB'; diff --git a/runner/master/pgsql.d/master/wal.sql b/runner/master/modules/docker-database/pgsql.d/primary/wal.sql similarity index 100% rename from runner/master/pgsql.d/master/wal.sql rename to runner/master/modules/docker-database/pgsql.d/primary/wal.sql diff --git a/runner/master/pgsql.d/slave/replica.sh b/runner/master/modules/docker-database/pgsql.d/replica/replica.sh similarity index 60% rename from runner/master/pgsql.d/slave/replica.sh rename to runner/master/modules/docker-database/pgsql.d/replica/replica.sh index eaf0f12..da9a229 100755 --- a/runner/master/pgsql.d/slave/replica.sh +++ b/runner/master/modules/docker-database/pgsql.d/replica/replica.sh @@ -5,20 +5,20 @@ set -e PGPASSWORD=${POSTGRES_PASSWORD} PGDATA=/var/lib/postgresql/data -echo "Polling until master is available" +echo "Polling until primary is available" until psql -h ${DBHOST} -U ${POSTGRES_USER} ${POSTGRES_DB} -c '\q'; do - >&2 echo "Postgres is unavailable - sleeping" + echo "Postgres is unavailable - sleeping" sleep 1 done -echo "Stopping replica service to copy from master" +echo "Stopping replica service to copy from primary" pg_ctl stop echo "Cleaning replica PGDATA (${PGDATA})" rm -fr ${PGDATA}/* -echo "Restoring PGDATA backup from master" +echo "Restoring PGDATA backup from primary" pg_basebackup -h ${DBHOST} -U ${POSTGRES_USER} -D "${PGDATA}" -P --wal-method=stream --slot=replica1 -R -echo "Starting replica service after copy from master" +echo "Starting replica service after copy from primary" pg_ctl start diff --git a/runner/master/pgsql.d/standalone/moodle.sql b/runner/master/modules/docker-database/pgsql.d/standalone/moodle.sql similarity index 100% rename from runner/master/pgsql.d/standalone/moodle.sql rename to runner/master/modules/docker-database/pgsql.d/standalone/moodle.sql diff --git a/runner/master/modules/docker-database/sqlsrv.d/sqlsrv_init.sh b/runner/master/modules/docker-database/sqlsrv.d/sqlsrv_init.sh new file mode 100644 index 0000000..9eb2142 --- /dev/null +++ b/runner/master/modules/docker-database/sqlsrv.d/sqlsrv_init.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Functions needed to init the database. +# This database only supports standalone setups, not primary-replica ones. + +# Init a standalone database container. Without replicas. +function sqlsrv_config_standalone() { + echo "Starting standalone database..." + docker run \ + --detach \ + --name "${DBHOST}" \ + --network "${NETWORK}" \ + -e ACCEPT_EULA=Y \ + -e SA_PASSWORD="${DBPASS}" \ + moodlehq/moodle-db-mssql:"${DBTAG}" + + # Wait few secs, before executing commands. + # TODO: Find a better way to wait for the database to be ready. + sleep 10 +} \ No newline at end of file diff --git a/runner/master/modules/docker-exttests/docker-exttests.sh b/runner/master/modules/docker-exttests/docker-exttests.sh new file mode 100644 index 0000000..cc485f1 --- /dev/null +++ b/runner/master/modules/docker-exttests/docker-exttests.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Ext. tests module functions. + +# This module defines the following env variables. +function docker-exttests_env() { + env=( + EXTTESTURL + ) + echo "${env[@]}" +} + +# Ext. tests module checks. +function docker-exttests_check() { + # Check all module dependencies. + verify_modules docker + + # These env variables must be set for the database module to work. + verify_env NETWORK UUID +} + +# Ext. tests module config. +function docker-exttests_config() { + EXTTESTS=exttests"${UUID}" + EXTTESTURL="http://${EXTTESTS}" +} + +# Ext. tests module setup, launch the containers. +function docker-exttests_setup() { + echo + echo ">>> startsection Starting exttests server <<<" + echo "============================================================================" + + # Start the exttests server + docker run \ + --detach \ + --name "${EXTTESTS}" \ + --network "${NETWORK}" \ + moodlehq/moodle-exttests:latest + echo "Ext. tests URL: ${EXTTESTURL}" + echo "Ext. tests logs:" + docker logs "${EXTTESTS}" + + echo "============================================================================" + echo ">>> stopsection <<<" +} diff --git a/runner/master/modules/docker-healthy/docker-healthy.sh b/runner/master/modules/docker-healthy/docker-healthy.sh new file mode 100644 index 0000000..7e1739c --- /dev/null +++ b/runner/master/modules/docker-healthy/docker-healthy.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Docker healthy module functions. + +# This module defines the following env variables. +function docker-healthy_env() { + env=() + echo "${env[@]}" +} + +# Docker healthy module checks. +function docker-healthy_check() { + # Check all module dependencies. + verify_modules docker + + # These env variables must be set for the module to work. + verify_env UUID SHAREDDIR +} + +# Docker healthy module setup. +function docker-healthy_setup() { + echo + echo ">>> startsection Waiting for all containers to become healthy<<<" + echo "============================================================================" + local waitperiod= + local startingcount= + + for waitperiod in {0..90}; do + # Note we cannot use the 'health' filter due to https://github.com/moby/moby/issues/35920 + startingcount=$(($(docker ps -a --filter name="${UUID}" | c1grep -e starting -e unhealthy | wc -l))) + if [[ ${startingcount} -lt 1 ]]; then + break + fi + echo "Waiting ${waitperiod} seconds for ${startingcount} containers to become healthy" + sleep 1 + done + + startingcount=$(($(docker ps -a --filter name="${UUID}" | c1grep -e starting -e unhealthy | wc -l))) + if [[ ${startingcount} -gt 0 ]]; then + print_error "Some containers were too slow. Aborting the run:" + exit_error "$(docker ps -a --filter name="${UUID}" --filter | c1grep -e starting -e unhealthy)" + fi + echo "All containers started OK" + + echo "============================================================================" + echo ">>> stopsection <<<" +} \ No newline at end of file diff --git a/runner/master/modules/docker-ldap/docker-ldap.sh b/runner/master/modules/docker-ldap/docker-ldap.sh new file mode 100644 index 0000000..f08d1d6 --- /dev/null +++ b/runner/master/modules/docker-ldap/docker-ldap.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# LDAP module functions. + +# This module defines the following env variables. +function docker-ldap_env() { + env=( + LDAPTESTURL + ) + echo "${env[@]}" +} + +# LDAP module checks. +function docker-ldap_check() { + # Check all module dependencies. + verify_modules docker + + # These env variables must be set for the database module to work. + verify_env NETWORK UUID +} + +# LDAP module config. +function docker-ldap_config() { + LDAP=ldap"${UUID}" + LDAPTESTURL="ldap://${LDAP}" +} + +# LDAP module setup, launch the containers. +function docker-ldap_setup() { + echo + echo ">>> startsection Starting LDAP server <<<" + echo "============================================================================" + + # Start the ldap server + docker run \ + --detach \ + --name "${LDAP}" \ + --network "${NETWORK}" \ + larrycai/openldap + + echo "LDAP: URL: ${LDAPTESTURL}" + echo "LDAP logs:" + docker logs "${LDAP}" + + echo "============================================================================" + echo ">>> stopsection <<<" +} \ No newline at end of file diff --git a/runner/master/modules/docker-logs/docker-logs.sh b/runner/master/modules/docker-logs/docker-logs.sh new file mode 100644 index 0000000..a386a33 --- /dev/null +++ b/runner/master/modules/docker-logs/docker-logs.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Docker logs module functions. + +# This module defines the following env variables. +function docker-logs_env() { + env=() + echo "${env[@]}" +} + +# Docker logs module checks. +function docker-logs_check() { + # Check all module dependencies. + verify_modules docker + + # These env variables must be set for the module to work. + verify_env UUID SHAREDDIR +} + +# Docker logs module teardown. Let's copy all the containers logs to the shared dir. +function docker-logs_teardown() { + echo ">>> startsection Exporting docker logs<<<" + echo "============================================================================" + echo "Exporting all docker logs for UUID: ${UUID}" + # Store the docker container logs. + for container in $(docker ps -a --format "{{.ID}}~{{.Names}}" --filter name="${UUID}"); do + image=$(echo "${container}" | cut -d'~' -f1) + name=$(echo "${container}" | cut -d'~' -f2) + name=${name%"${UUID}"} # Get rid of the UUID for naming log files. + echo " - ${name} logs to ${SHAREDDIR}/${name}.gz" + docker logs "${image}" 2>&1 | gzip > "${SHAREDDIR}/${name}.gz" + done + +# TODO: Why do we need this? +# Only if the container exists and is running. +if [[ -n $(docker ps --filter name="${WEBSERVER}" --filter status=running --quiet) ]]; then + docker exec -t "${WEBSERVER}" \ + chown -R "${UID}:${GROUPS[0]}" /shared +fi + +echo "============================================================================" +echo ">>> stopsection <<<" +} \ No newline at end of file diff --git a/runner/master/modules/docker-mlbackend/docker-mlbackend.sh b/runner/master/modules/docker-mlbackend/docker-mlbackend.sh new file mode 100644 index 0000000..64afb9b --- /dev/null +++ b/runner/master/modules/docker-mlbackend/docker-mlbackend.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Python ML backend module functions. + +# This module defines the following env variables. +function docker-mlbackend_env() { + env=( + MLBACKENDTESTNAME + ) + echo "${env[@]}" +} + +# Python ML backend module checks. +function docker-mlbackend_check() { + # Check all module dependencies. + verify_modules docker + + # These env variables must be set for the database module to work. + verify_env NETWORK UUID +} + +# Python ML backend config. +function docker-mlbackend_config() { + # Apply some defaults. + MLBACKEND_PYTHON_VERSION=${MLBACKEND_PYTHON_VERSION:-} + if [[ -n "${MLBACKEND_PYTHON_VERSION}" ]]; then + # Only if it has been explicitly requested. + MLBACKENDTESTNAME=mlpython"${UUID}" + fi +} + +# Python ML backend module setup, launch the containers. +function docker-mlbackend_setup() { + # Only if it has been explicitly requested. + if [[ -n "${MLBACKEND_PYTHON_VERSION}" ]]; then + echo + echo ">>> startsection Starting Python ML backend server <<<" + echo "============================================================================" + + # Start the Python ML backend server. + docker run \ + --detach \ + --name "${MLBACKENDTESTNAME}" \ + --network "${NETWORK}" \ + moodlehq/moodle-mlbackend-python:"${MLBACKEND_PYTHON_VERSION}" + echo "Python ML backend: URL: ${MLBACKENDTESTNAME}" + echo "Python ML backend logs:" + docker logs "${MLBACKENDTESTNAME}" + + echo "============================================================================" + echo ">>> stopsection <<<" + else + echo "Python ML backend server not requested." + fi +} diff --git a/runner/master/modules/docker-mocks/docker-mocks.sh b/runner/master/modules/docker-mocks/docker-mocks.sh new file mode 100644 index 0000000..3a7c11a --- /dev/null +++ b/runner/master/modules/docker-mocks/docker-mocks.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Mocks module functions. + +# This module defines the following env variables. +function docker-mocks_env() { + env=( + BBBMOCKURL + MATRIXMOCKURL + ) + echo "${env[@]}" +} + +# Mocks module checks. +function docker-mocks_check() { + # Check all module dependencies. + verify_modules docker + + # These env variables must be set for the database module to work. + verify_env NETWORK UUID +} + +# Mocks module config. +function docker-mocks_config() { + # Apply some defaults. + BBBMOCK="bbbmock${UUID}" + BBBMOCKURL="http://${BBBMOCK}" + + MATRIXMOCK="matrixmock${UUID}" + MATRIXMOCKURL="http://${MATRIXMOCK}" +} + +# Mocks module setup, launch the mock containers. +function docker-mocks_setup() { + echo + echo ">>> startsection Starting mocking services <<<" + echo "============================================================================" + + # Start the BigBlueButton mock. + docker run \ + --detach \ + --name "${BBBMOCK}" \ + --network "${NETWORK}" \ + moodlehq/bigbluebutton_mock:latest + echo "BBB mock url: ${BBBMOCKURL}" + echo "BBB mock logs:" + docker logs "${BBBMOCK}" + + echo + + # Start the Matrix mock. + docker run \ + --detach \ + --name "${MATRIXMOCK}" \ + --network "${NETWORK}" \ + moodlehq/matrixsynapse_mock:latest + echo "Matrix mock url: ${MATRIXMOCKURL}" + echo "Matrix mock logs:" + docker logs "${MATRIXMOCK}" + + echo "============================================================================" + echo ">>> stopsection <<<" +} \ No newline at end of file diff --git a/runner/master/config.template.php b/runner/master/modules/docker-php/config.template.php similarity index 99% rename from runner/master/config.template.php rename to runner/master/modules/docker-php/config.template.php index d814550..acd9be0 100644 --- a/runner/master/config.template.php +++ b/runner/master/modules/docker-php/config.template.php @@ -27,11 +27,11 @@ $CFG->prefix = 'm_'; $CFG->dboptions = ['dbcollation' => getenv('DBCOLLATION')]; -if ($slave = getenv('DBHOST_SLAVE')) { +if ($replica = getenv('DBHOST_REPLICA')) { $CFG->dboptions['readonly'] = [ 'instance' => [ [ - 'dbhost' => $slave, + 'dbhost' => $replica, ], ], ]; diff --git a/runner/master/modules/docker-php/docker-php.sh b/runner/master/modules/docker-php/docker-php.sh new file mode 100644 index 0000000..666ff7a --- /dev/null +++ b/runner/master/modules/docker-php/docker-php.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Docker PHP module functions. + +# This module defines the following env variables. +function docker-php_env() { + env=( + PHP_VERSION + DOCKER_PHP + WEBSERVER + ) + echo "${env[@]}" +} + +# Docker module checks. +function docker-php_check() { + # Check all module dependencies. + verify_modules docker env + + # These env variables must be set for the module to work. + verify_env ENVIROPATH NETWORK UUID COMPOSERCACHE SHAREDDIR +} + +# Docker module init. +function docker-php_config() { + # Apply some defaults. + PHP_VERSION="${PHP_VERSION:-8.0}" + DOCKER_PHP="${DOCKER_PHP:-moodlehq/moodle-php-apache:${PHP_VERSION}}" + WEBSERVER=webserver"${UUID}" +} + +# Docker module setup, start the PHP webserver container. +function docker-php_setup() { + echo ">>> startsection Starting web server <<<" + echo "============================================================================" + + docker run \ + --network "${NETWORK}" \ + --name "${WEBSERVER}" \ + --detach \ + --env-file "${ENVIROPATH}" \ + -v "${COMPOSERCACHE}:/var/www/.composer:rw" \ + -v "${SHAREDDIR}":/shared \ + "${DOCKER_PHP}" + + echo + echo "Webserver logs:" + docker logs "${WEBSERVER}" + echo "============================================================================" + echo ">>> stopsection <<<" +} diff --git a/runner/master/modules/docker-solr/docker-solr.sh b/runner/master/modules/docker-solr/docker-solr.sh new file mode 100644 index 0000000..655b69c --- /dev/null +++ b/runner/master/modules/docker-solr/docker-solr.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Solr module functions. + +# This module defines the following env variables. +function docker-solr_env() { + env=( + SOLRTESTNAME + ) + echo "${env[@]}" +} + +# Solr module checks. +function docker-solr_check() { + # Check all module dependencies. + verify_modules docker + + # These env variables must be set for the database module to work. + verify_env NETWORK UUID +} + +# Solr module config. +function docker-solr_config() { + # Apply some defaults. + SOLRTESTNAME=solr"${UUID}" +} + +# Solr module setup, launch the containers. +function docker-solr_setup() { + echo + echo ">>> startsection Starting Solr server <<<" + echo "============================================================================" + + # Start the solr server + docker run \ + --detach \ + --name "${SOLRTESTNAME}" \ + --network "${NETWORK}" \ + solr:7 \ + solr-precreate test + echo "Solr: URL: ${SOLRTESTNAME}" + echo "Solr logs:" + docker logs "${SOLRTESTNAME}" + + echo "============================================================================" + echo ">>> stopsection <<<" +} diff --git a/runner/master/modules/docker-summary/docker-summary.sh b/runner/master/modules/docker-summary/docker-summary.sh new file mode 100644 index 0000000..ca7c82a --- /dev/null +++ b/runner/master/modules/docker-summary/docker-summary.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Docker sumary module functions. + +# This module defines the following env variables. +function docker-summary_env() { + env=() + echo "${env[@]}" +} + +# Docker summary module checks. +function docker-summary_check() { + # Check all module dependencies. + verify_modules docker + + # These env variables must be set for the module to work. + verify_env UUID SHAREDDIR +} + +# Docker summary module setup. +function docker-summary_setup() { + # Prepare the summary of images being used by the run (creation date & digest) + echo + echo ">>> startsection Details about the images being used by the run<<<" + echo "============================================================================" + docker ps --filter "name=${UUID}" --format='{{.Image}}' | sort | uniq | xargs -I{} \ + docker image inspect \ + --format '{} {{if .Created}}created:{{.Created}}{{end}} {{if .RepoDigests}}{{index .RepoDigests 0}}{{end}}' {} | \ + tr '@' ' ' | cut -f1,2,4 -d' ' + + echo "============================================================================" + echo ">>> stopsection <<<" +} \ No newline at end of file diff --git a/runner/master/modules/docker/docker.sh b/runner/master/modules/docker/docker.sh new file mode 100644 index 0000000..975ed7e --- /dev/null +++ b/runner/master/modules/docker/docker.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Docker module functions. + +# This module defines the following env variables. +function docker_env() { + env=( + NETWORK + ) + echo "${env[@]}" +} + +# Docker module checks. +function docker_check() { + if ! docker --version > /dev/null 2>&1; then + exit_error "Docker is not installed. Please install it and try again." + fi + + if ! docker-compose --version > /dev/null 2>&1; then + exit_error "Docker Compose is not installed. Please install it and try again." + fi + + # These env variables must be set for the module to work. + verify_env UUID +} + +# Docker module config. +function docker_config() { + # Apply some defaults. + NETWORK="${NETWORK:-moodle}" +} + +# Docker module setup, basically create the network that will be used by all the containers. +function docker_setup() { + echo ">>> startsection Checking networks <<<" + echo "============================================================================" + local networkid= + networkid=$(docker network list -q --filter name="${NETWORK}$") + if [[ -z ${networkid} ]] + then + echo "Creating new network '${NETWORK}'" + NETWORK=$(docker network create "${NETWORK}") + fi + echo "Found network '${NETWORK}' with identifier ${networkid}" + echo "============================================================================" + echo ">>> stopsection <<<" +} + +# Docker module teardown, stop and remove all the containers that were used in this run (by UUID). +function docker_teardown() { + echo ">>> startsection Cleaning up docker containers <<<" + echo "============================================================================" + echo "Stopping and removing all docker containers for UUID: ${UUID}" + docker ps -a --filter name="${UUID}" + echo + for container in $(docker ps -a -q --filter name="${UUID}"); do + echo -n " - Stopping and removing container " + docker stop "${container}" | xargs docker rm --volumes + done + echo "============================================================================" + echo ">>> stopsection <<<" +} \ No newline at end of file diff --git a/runner/master/modules/env/env.sh b/runner/master/modules/env/env.sh new file mode 100644 index 0000000..e258205 --- /dev/null +++ b/runner/master/modules/env/env.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Environment module functions. + +# This module defines the following env variables. +function env_env() { + env=( + ENVIROPATH + ) + echo "${env[@]}" +} + +# Environment module checks. +function env_check() { + # These env variables must be set for the module to work. + verify_env JOBTYPE SHAREDDIR +} + +# Environment module config. +function env_config() { + # Apply some defaults. + ENVIROPATH="${SHAREDDIR}"/moodle-ci-runner.env +} + +# Environment module setup, Create the environment file to be used by other modules. +# (this is executed after all the modules and the job have been able to set their env variables). +function env_setup() { + # Reset the environment file. + rm -f "${ENVIROPATH}" + touch "${ENVIROPATH}" + + # Add all the variables that the job type requires. + for var in $(get_job_to_env_file "${JOBTYPE}"); do + echo "${var}"="${!var}" >> "${ENVIROPATH}" + done +} + +# Environment module teardown. Remove the environment file. +function env_teardown() { + rm -f "${ENVIROPATH}" +} diff --git a/runner/master/modules/git/git.sh b/runner/master/modules/git/git.sh new file mode 100644 index 0000000..a2495eb --- /dev/null +++ b/runner/master/modules/git/git.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Git module functions. + +# This module defines the following env variables +function git_env() { + env=( + GIT_COMMIT + ) + echo "${env[@]}" +} + +# Git module checks. +function git_check() { + if ! git --version > /dev/null 2>&1; then + exit_error "Git is not installed. Please install it and try again." + fi + + # These env variables must be set for the module to work. + verify_env CODEDIR +} + +# Git module config. +function git_config() { + # Apply some defaults. + GIT_COMMIT="N/A" + + # If available, which commit hash is being tested. + if [[ -d "${CODEDIR}"/.git ]]; then + GIT_COMMIT=$(cd "${CODEDIR}" && git rev-parse HEAD) + fi +} diff --git a/runner/master/modules/moodle-core-copy/moodle-core-copy.sh b/runner/master/modules/moodle-core-copy/moodle-core-copy.sh new file mode 100644 index 0000000..42ddcff --- /dev/null +++ b/runner/master/modules/moodle-core-copy/moodle-core-copy.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Moodle core (copy) module functions. + +# This module defines the following env variables. +function moodle-core-copy_env() { + env=( + MOODLE_BRANCH + ) + echo "${env[@]}" +} + +# Moodle core copy module checks. +function moodle-core-copy_check() { + # Check all module dependencies. + verify_modules docker plugins docker-php + + # These env variables must be set for the module to work. + verify_env BASEDIR CODEDIR PLUGINSDIR WEBSERVER +} + +# Moodle core copy module config. +function moodle-core-copy_config() { + # Get the Moodle branch from code, so we can make decisions based on it. + MOODLE_BRANCH=$(grep "\$branch" "${CODEDIR}"/version.php | sed "s/';.*//" | sed "s/^\$.*'//") +} + +# Moodle core copy module setup. +function moodle-core-copy_setup() { + + echo ">>> startsection Copying source files <<<" + echo "============================================================================" + + # Copy the code to the web server. + echo "== Copying code in place." + docker cp "${CODEDIR}"/. "${WEBSERVER}":/var/www/html + + # Copy the config.php in place + echo "== Copying configuration in place." + docker cp "${BASEDIR}/modules/docker-php/config.template.php" "${WEBSERVER}":/var/www/html/config.php + + # Copy the plugins in place. + if [[ -n "$PLUGINSTOINSTALL" ]]; then + echo "== Copying external plugins in place." + docker cp "${PLUGINSDIR}"/. "${WEBSERVER}":/var/www/html + fi + + # Copy composer-phar if available in caches. + if [[ -f "${COMPOSERCACHE}/composer.phar" ]]; then + echo "== Copying composer.phar in place." + docker cp "${COMPOSERCACHE}/composer.phar" "${WEBSERVER}":/var/www/html/composer.phar + fi + + echo "============================================================================" + echo ">>> stopsection <<<" +} \ No newline at end of file diff --git a/runner/master/modules/plugins/plugins.sh b/runner/master/modules/plugins/plugins.sh new file mode 100644 index 0000000..c37cd81 --- /dev/null +++ b/runner/master/modules/plugins/plugins.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Plugins module functions. + +# This module defines the following env variables. +# The PLUGINSTOINSTALL variable could be set to install external plugins in the CODEDIR folder. +# The following information is needed for each plugin: git repo, folder and branch (optional). +# The plugin fields should be separated by "|" and each plugin should be separated using ";": +# "gitrepoplugin1|gitfolderplugin1|gitbranchplugin1;gitrepoplugin2|gitfolderplugin2|gitbranchplugin2[...]" +# +# Example: "https://github.com/moodlehq/moodle-local_mobile.git|local/mobile|MOODLE_37_STABLE;git@github.com:jleyva/moodle-block_configurablereports.git|blocks/configurable_reports" +# +# The PLUGINSDIR is the directory where the plugins will be downloaded (or made available +# by any other method). In a moodle-like structure (mod, local, etc). +function plugins_env() { + env=( + PLUGINSTOINSTALL + PLUGINSDIR + ) + echo "${env[@]}" +} + +# Plugins module checks. +function plugins_check() { + # Check all module dependencies. + verify_modules docker git + + # These env variables must be set for the module to work. + verify_env WORKSPACE +} + +# Plugins module config. +function plugins_config() { + # Apply some defaults. + PLUGINSTOINSTALL="${PLUGINSTOINSTALL:-}" + PLUGINSDIR="${PLUGINSDIR:-${WORKSPACE}/plugins}" +} + +# Plugins module setup, download all the requested plugins to workspace area. +function plugins_setup() { + # Create the plugins directory. Always. + mkdir -p "${PLUGINSDIR}" + # Download the plugins, if any. + if [[ -n "${PLUGINSTOINSTALL}" ]]; then + local plugins= + local plugin= + echo ">>> startsection Download external plugins <<<" + echo "============================================================================" + # Download all the plugins in a temporary folder. + IFS=';' read -ra plugins <<< "${PLUGINSTOINSTALL}" + for plugin in "${plugins[@]}"; do + local gitrepo= + local gitbranch= + local directory= + local branch= + local singlebranch= + if [[ -n "${plugin}" ]]; then + gitrepo=$(echo "$plugin" | cut -f1 -d'|') + directory=$(echo "$plugin" | cut -f2 -d'|') + gitbranch=$(echo "$plugin" | cut -f3 -d'|') + echo "Cloning ${gitrepo}/${gitbranch}" + + if [[ -n "${gitbranch}" ]]; then + # Only download this branch. + branch="--branch=${gitbranch}" + singlebranch="--single-branch" + fi + + # Clone the plugin repository in the defined folder. + git clone --quiet "${branch}" "${singlebranch}" "${gitrepo}" "${PLUGINSDIR}/${directory}" + echo "Cloned. HEAD is @ $(cd "${PLUGINSDIR}/${directory}" && git rev-parse HEAD)" + echo + fi + done + echo "============================================================================" + echo ">>> stopsection <<<" + fi +} \ No newline at end of file diff --git a/runner/master/modules/summary/summary.sh b/runner/master/modules/summary/summary.sh new file mode 100644 index 0000000..f6ecc89 --- /dev/null +++ b/runner/master/modules/summary/summary.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Summary module functions. + +# This module defines the following env variables +function summary_env() { + env=() + echo "${env[@]}" +} + +# Summary module checks. +function summary_check() { + # In order to print this as soon as possible, we aren't going to check + # for dependencies here, but will do it in the summary_setup() function. + # (once all the modules have already defined their env variables). + # Note that this is highly exceptional and should not be done in other modules. + true +} + +# Summary module setup. +function summary_setup() { + # We are checking for dependencies here exceptionally. See summary_check() above for more info. + # Check all module dependencies. + verify_modules env docker-php moodle-core-copy + + # These env variables must be set for the database module to work. + verify_env JOBTYPE BUILD_ID WORKSPACE CODEDIR SHAREDDIR UUID GIT_COMMIT ENVIROPATH + + # Print a summary of the job. + # Note that this only includes the variables that are common for all the job types. + # Each job is responsible (_to_summary() function) to complete the list with its own variables + # or anything else that it wants to add to the summary. + echo "============================================================================" + echo "= Job summary <<<" + echo "============================================================================" + echo "== JOBTYPE: ${JOBTYPE}" + echo "== Build Id: ${BUILD_ID}" + echo "== Workspace: ${WORKSPACE}" + echo "== Code directory: ${CODEDIR}" + echo "== Shared directory: ${SHAREDDIR}" + echo "== UUID / Container suffix: ${UUID}" + echo "== Environment: ${ENVIROPATH}" + echo "== GIT commit: ${GIT_COMMIT}" + + # Add all the summary information that the job type wants to add. + get_job_to_summary "${JOBTYPE}" + echo "============================================================================" +} \ No newline at end of file diff --git a/runner/master/run.sh b/runner/master/run.sh index a831ce8..1a3c4a6 100755 --- a/runner/master/run.sh +++ b/runner/master/run.sh @@ -1,6 +1,6 @@ -#!/bin/bash +#!/usr/bin/env bash # -# This file is part of the Moodle Continous Integration Project. +# This file is part of the Moodle Continuous Integration Project. # # Moodle is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -13,171 +13,138 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Moodle. If not, see . +# along with Moodle. If not, see . set -u +set -e set -o pipefail -SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# Let's define some variables that will be used by the scripts. -CACHEDIR="${CACHEDIR:-${HOME}/caches}" -export COMPOSERCACHE="${COMPOSERCACHE:-${CACHEDIR}/composer}" -export CODEDIR="${CODEDIR:-${WORKSPACE}/moodle}" -export OUTPUTDIR="${WORKSPACE}"/"${BUILD_ID}" -export ENVIROPATH="${OUTPUTDIR}"/environment.list -# The PLUGINSTOINSTALL variable could be set to install external plugins in the CODEDIR folder. The following information is needed -# for each plugin: gitrepo, folder and branch (optional). The plugin fields should be separated by "|" and each plugin should -# be separated using ";": "gitrepoplugin1|gitfolderplugin1|gitbranchplugin1;gitrepoplugin2|gitfolderplugin2|gitbranchplugin2[...]" -# Example: "https://github.com/moodlehq/moodle-local_mobile.git|local/mobile|MOODLE_37_STABLE;git@github.com:jleyva/moodle-block_configurablereports.git|blocks/configurable_reports" -export PLUGINSTOINSTALL="${PLUGINSTOINSTALL:-}" -# Plugin folder where the plugins to install will be downloaded. -export PLUGINSDIR="${PLUGINSDIR:-${WORKSPACE}/plugins}" - -mkdir -p "${PLUGINSDIR}" -if [ -n "$PLUGINSTOINSTALL" ]; -then - echo ">>> startsection Download external plugins <<<" - echo "============================================================================" - # Download all the plugins in a temporary folder. - IFS=';' read -ra PLUGINS <<< "$PLUGINSTOINSTALL" - for PLUGIN in "${PLUGINS[@]}"; - do - if [ -n "$PLUGIN" ] - then - PLUGINGITREPO=$(echo "$PLUGIN" | cut -f1 -d'|') - PLUGINFOLDER=$(echo "$PLUGIN" | cut -f2 -d'|') - PLUGINBRANCH=$(echo "$PLUGIN" | cut -f3 -d'|') - echo "Cloning ${PLUGINGITREPO}/${PLUGINBRANCH}" +# Base directory where the scripts are located. +BASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - if [ -n "${PLUGINBRANCH}" ] - then - # Only download this branch. - PLUGINBRANCH="-b ${PLUGINBRANCH} --single-branch" - fi +# Include the functions. +source "${BASEDIR}/lib.sh" - # Clone the plugin repository in the defined folder. - git clone ${PLUGINBRANCH} ${PLUGINGITREPO} "${PLUGINSDIR}/${PLUGINFOLDER}" - echo "Cloned. HEAD is @ $(cd "${PLUGINSDIR}/${PLUGINFOLDER}" && git rev-parse HEAD)" - echo - fi - done - unset IFS - echo "============================================================================" - echo ">>> stopsection <<<" - echo +# Trap to finish the execution (exit and Ctrl+C). +trap trap_exit EXIT +trap trap_ctrl_c INT + +# Verify that all the needed utilities are installed and available. +verify_utilities awk grep head mktemp pwd sed sha1sum sort tac tr true uniq uuid xargs + +# Base directory to be used as workspace for the execution. +if [[ -z ${WORKSPACE:-} ]]; then + # If not defined, create one. + MKTEMP=$(mktemp -d) + WORKSPACE="${MKTEMP}/workspace" fi -# Which PHP Image to use. -export PHP_VERSION="${PHP_VERSION:-7.1}" -export PHP_SERVER_DOCKER="${PHP_SERVER_DOCKER:-moodlehq/moodle-php-apache:${PHP_VERSION}}" +# Base directory where the code is located. +CODEDIR="${CODEDIR:-${WORKSPACE}/moodle}" + +# BUILD_ID, if not defined use the current PID. +BUILD_ID="${BUILD_ID:-$$}" + +# Base directory to be shared with some containers that will read/write information there (timing, environment, logs... etc.). +SHAREDDIR="${WORKSPACE}"/"${BUILD_ID}" + +# Ensure that the output directory exists. +# It must also be set with the sticky bit, and world writable. +mkdir -p "${SHAREDDIR}" +chmod -R g+sw,a+sw "${SHAREDDIR}" + +# UUID to be used as suffix for the containers and other stuff. +UUID=$(uuid | sha1sum | awk '{print $1}' | head -c 16) -# If available, which commit hash is being tested. -export GIT_COMMIT="N/A" -if [ -d "${CODEDIR}"/.git ]; then - export GIT_COMMIT=$(cd "${CODEDIR}" && git rev-parse HEAD) +# Job type to run (from "jobtypes" directory). +JOBTYPE="${JOBTYPE:-phpunit}" + +# Ensure that the job type is valid. +if [[ ! -f ${BASEDIR}"/jobtypes/"${JOBTYPE}/phpunit.sh ]]; then + exit_error "ERROR: Invalid jobtype: ${JOBTYPE}" +fi + +# Caches directories, used for composer, to accelerate git operations... +CACHEDIR="${CACHEDIR:-${HOME}/caches}" +COMPOSERCACHE="${COMPOSERCACHE:-${CACHEDIR}/composer}" + +# BC compatibility with old replica names. +# TODO: Remove this once all the uses in CI are updated to use the new ones. +DBREPLICAS="${DBREPLICAS:-${DBSLAVES:-}}" +if [[ -n ${DBSLAVES:-} ]]; then + print_warning "DBSLAVES variable is deprecated, use DBREPLICAS instead." fi -# Which Moodle version (XY) is being used. -export MOODLE_VERSION=$(grep "\$branch" "${CODEDIR}"/version.php | sed "s/';.*//" | sed "s/^\$.*'//") -# Which Mobile app version is used: latest (stable), next (master), x.y.z. +# BC compatibility with old phpunit and behat variable names. +# TODO: Remove this once all the uses in CI are updated to use the new ones. +# PHPUnit: +PHPUNIT_TESTSUITE="${PHPUNIT_TESTSUITE:-${TESTSUITE:-}}" +PHPUNIT_FILTER="${PHPUNIT_FILTER:-${TAGS:-}}" +# Behat: +BEHAT_TAGS="${BEHAT_TAGS:-${TAGS:-}}" +BEHAT_NAME="${BEHAT_NAME:-${NAME:-}}" +# Print a warning if the old variables are used. +if [[ -n ${TESTSUITE:-} ]]; then + print_warning "TESTSUITE variable is deprecated, use PHPUNIT_TESTSUITE instead." +fi +if [[ -n ${TAGS:-} ]]; then + print_warning "TAGS variable is deprecated, use PHPUNIT_FILTER or BEHAT_TAGS instead." +fi +if [[ -n ${NAME:-} ]]; then + print_warning "NAME variable is deprecated, use BEHAT_NAME instead." +fi + +# Everything is ready, let's run the job. +run "${JOBTYPE}" + +# All done, exit with the exit code of the job. +exit "${EXITCODE}" + + # If the MOBILE_VERSION is not defined, the moodlemobile docker won't be executed. export MOBILE_VERSION="${MOBILE_VERSION:-}" export MOBILE_APP_PORT="${MOBILE_APP_PORT:-8100}" -# Default type of test to run. -# phpunit or behat. -export TESTTORUN="${TESTTORUN:-phpunit}" - -# Default DB settings. -# Todo: Tidy this up properly. -export DBTYPE="${DBTYPE:-pgsql}" -export DBTAG="${DBTAG:-auto}" # Optional docker image tag to be used (defaults to "auto", to pin later if needed. -export DBTORUN="${DBTORUN:-}" -export DBSLAVES="${DBSLAVES:-0}" -# Here it's where we pin any DBTAG docker tag (versions), when needed. Don't change it elsewhere. -# We only apply these pinned defaults when no DBTAG has been explicitly defined. And we only apply -# them to databases know to need them (some bug prevents to use "latest"). Every pinned case should -# include a comment with the reason for it. - -if [ "${DBTAG}" == "auto" ] -then - case ${DBTYPE} in - mariadb) - DBTAG=10.7 # Because there is a problem with the >= 10.8 images not working with older hosts OS. - ;; - mysqli) - DBTAG=8.0 # Because it's the master lowest supported version and we need it covered by default. - ;; - mssql | sqlsrv) - DBTAG=latest # No pin, right now 2019-latest - ;; - oci) - DBTAG=latest # No pin, right now this is 21c - ;; - pgsql) - DBTAG=13 # Because it's the master lowest supported version and we need it covered by default. - ;; - *) - echo "Wrong DBTYPE: ${DBTYPE}. Fix the run, or add support for that DB above" - exit 1 - ;; - esac -fi # Test defaults export BROWSER="${BROWSER:-chrome}" export BROWSER_DEBUG="${BROWSER_DEBUG:-}" export BROWSER_HEADLESS="${BROWSER_HEADLESS:-}" export DISABLE_MARIONETTE= -export MLBACKEND_PYTHON_VERSION="${MLBACKEND_PYTHON_VERSION:-}" export BEHAT_SUITE="${BEHAT_SUITE:-}" +export BEHAT_TAGS="${BEHAT_TAGS:-}" +export BEHAT_NAME="${BEHAT_NAME:-}" export BEHAT_TOTAL_RUNS="${BEHAT_TOTAL_RUNS:-3}" export BEHAT_NUM_RERUNS="${BEHAT_NUM_RERUNS:-1}" -export TAGS="${TAGS:-}" -export NAME="${NAME:-}" -export TESTSUITE="${TESTSUITE:-}" -export RUNCOUNT="${RUNCOUNT:-1}" export BEHAT_TIMING_FILENAME="${BEHAT_TIMING_FILENAME:-}" export BEHAT_INCREASE_TIMEOUT="${BEHAT_INCREASE_TIMEOUT:-}" -# Remove some stuff that, simply, cannot be there based on $TESTTORUN -if [ "${TESTTORUN}" == "phpunit" ] -then - BROWSER= - BEHAT_SUITE= - BEHAT_TOTAL_RUNS= - BEHAT_NUM_RERUNS= - BEHAT_TIMING_FILENAME= - BEHAT_INCREASE_TIMEOUT= - NAME= -elif [ "${TESTTORUN}" == "behat" ] + +# Remove some stuff that, simply, cannot be there based on $JOBTYPE +if [ "${JOBTYPE}" == "behat" ] then - TESTSUITE= + PHPUNIT_TESTSUITE= + PHPUNIT_FILTER= # If the --name option is going to be used, then disable any parallel execution, it's not worth # instantiating N sites for just running one feature/scenario. - if [[ -n "${NAME}" ]] && [[ "${BEHAT_TOTAL_RUNS}" -gt 1 ]]; then - echo "Note: parallel option disabled because of NAME (--name) behat option being used." + if [[ -n "${BEHAT_NAME}" ]] && [[ "${BEHAT_TOTAL_RUNS}" -gt 1 ]]; then + echo "Note: parallel option disabled because of BEHAT_NAME (--name) behat option being used." BEHAT_TOTAL_RUNS=1 fi # If the composer.json contains instaclick then we must disable marionette and use an older version of firefox. - hasinstaclick=$((`grep instaclick "${CODEDIR}"/composer.json | wc -l`)) + hasinstaclick=$((`c1grep instaclick "${CODEDIR}"/composer.json | wc -l`)) if [[ ${hasinstaclick} -ge 1 ]] then export DISABLE_MARIONETTE=1 fi fi -# Ensure that the output directory exists. -# It must also be set with the sticky bit, and world writable. -# Apache and Behat run as www-data, and must be able to write to this directory, but there is no reliabel UID mapping -# between the container and host. -mkdir -p "${OUTPUTDIR}" -rm -f "${ENVIROPATH}" -touch "${ENVIROPATH}" if [ ! -z "$BEHAT_TIMING_FILENAME" ] then @@ -186,76 +153,21 @@ then if [ -f "${TIMINGSOURCE}" ] then - cp "${TIMINGSOURCE}" "${OUTPUTDIR}"/timing.json - else - touch "${OUTPUTDIR}"/timing.json - fi -fi - -chmod -R g+sw,a+sw "${OUTPUTDIR}" - -# Select db to use today. -if [ -n "$DBTORUN" ] -then - DBTORUN=(`echo ${DBTORUN}`); - # Find which db to run today. - dayoftheweek=`date +"%u"` - if [[ -z ${DBTORUN} ]]; then - DBTYPE=pgsql + cp "${TIMINGSOURCE}" "${SHAREDDIR}"/timing.json else - DBTYPE=${DBTORUN[ $(( ${dayoftheweek} - 1 )) ]} + touch "${SHAREDDIR}"/timing.json fi - echo "Running against ${DBTYPE}" fi -# Setup Environment -UUID=$(uuid | sha1sum | awk '{print $1}') -UUID=${UUID:0:16} -export DBHOST=database"${UUID}" -export DBTYPE="${DBTYPE:-pgsql}" -export DBTAG="${DBTAG:-latest}" -export DBUSER="${DBUSER:-moodle}" -export DBPASS="${DBPASS:-moodle}" -export DBHOST="${DBHOST:-${DBTYPE}}" -export DBHOST_SLAVE="" -export DBNAME="moodle" - -echo "DBTYPE" >> "${ENVIROPATH}" -echo "DBTAG" >> "${ENVIROPATH}" -echo "DBSLAVES" >> "${ENVIROPATH}" -echo "DBHOST" >> "${ENVIROPATH}" -echo "DBHOST_SLAVE" >> "${ENVIROPATH}" -echo "DBUSER" >> "${ENVIROPATH}" -echo "DBPASS" >> "${ENVIROPATH}" -echo "DBNAME" >> "${ENVIROPATH}" -echo "DBCOLLATION" >> "${ENVIROPATH}" -echo "BROWSER" >> "${ENVIROPATH}" -echo "BROWSER_DEBUG" >> "${ENVIROPATH}" -echo "BROWSER_HEADLESS" >> "${ENVIROPATH}" -echo "WEBSERVER" >> "${ENVIROPATH}" -echo "BEHAT_TOTAL_RUNS" >> "${ENVIROPATH}" -echo "BEHAT_NUM_RERUNS" >> "${ENVIROPATH}" -echo "BEHAT_TIMING_FILENAME" >> "${ENVIROPATH}" -echo "BEHAT_INCREASE_TIMEOUT" >> "${ENVIROPATH}" -echo "DISABLE_MARIONETTE" >> "${ENVIROPATH}" -echo "MLBACKEND_PYTHON_VERSION" >> "${ENVIROPATH}" echo "============================================================================" echo "= Job summary <<<" echo "============================================================================" -echo "== Workspace: ${WORKSPACE}" -echo "== Build Id: ${BUILD_ID}" -echo "== Output directory: ${OUTPUTDIR}" -echo "== UUID: ${UUID}" -echo "== Container prefix: ${UUID}" -echo "== GIT commit: ${GIT_COMMIT}" echo "== PHP version: ${PHP_VERSION}" -echo "== Moodle branch (version.php): ${MOODLE_VERSION}" -echo "== DBTORUN: ${DBTORUN}" +echo "== Moodle branch (version.php): ${MOODLE_BRANCH}" echo "== DBTYPE: ${DBTYPE}" echo "== DBTAG: ${DBTAG}" -echo "== DBSLAVES: ${DBSLAVES}" -echo "== TESTTORUN: ${TESTTORUN}" +echo "== DBREPLICAS: ${DBREPLICAS}" echo "== BROWSER: ${BROWSER}" echo "== BROWSER_DEBUG: ${BROWSER_DEBUG}" echo "== BROWSER_HEADLESS: ${BROWSER_HEADLESS}" @@ -267,436 +179,14 @@ echo "== BEHAT_NUM_RERUNS: ${BEHAT_NUM_RERUNS}" echo "== BEHAT_INCREASE_TIMEOUT: ${BEHAT_INCREASE_TIMEOUT}" echo "== BEHAT_SUITE: ${BEHAT_SUITE}" echo "== TAGS: ${TAGS}" -echo "== NAME: ${NAME}" +echo "== BEHAT_NAME: ${BEHAT_NAME}" echo "== MOBILE_APP_PORT: ${MOBILE_APP_PORT}" echo "== MOBILE_VERSION: ${MOBILE_VERSION}" echo "== PLUGINSTOINSTALL: ${PLUGINSTOINSTALL}" -echo "== TESTSUITE: ${TESTSUITE}" -echo "== Environment: ${ENVIROPATH}" echo "============================================================================" -# Setup the image cleanup. -function finish { - echo - echo ">>> startsection Cleaning up docker images <<<" - echo "============================================================================" - echo "Stopping all docker images for ${UUID}" - docker ps -a --filter name=${UUID} - for image in `docker ps -a -q --filter name=${UUID}` - do - docker stop $image - done - echo "============================================================================" - echo ">>> stopsection <<<" -} -trap finish EXIT - -function ctrl_c() { - echo - echo "============================================================================" - echo "Job was cancelled at user request" - echo "============================================================================" - exit 255 -} -trap ctrl_c INT - -echo -echo ">>> startsection Checking networks <<<" -echo "============================================================================" -NETWORKNAME="${NETWORKNAME:-moodle}" -NETWORK=$(docker network list -q --filter name="${NETWORKNAME}$") -if [[ -z ${NETWORK} ]] -then - echo "Creating new network '${NETWORKNAME}'" - NETWORK=$(docker network create "${NETWORKNAME}") -fi -echo "Found network '${NETWORKNAME}' with identifier ${NETWORK}" -echo "============================================================================" -echo ">>> stopsection <<<" - -echo -echo ">>> startsection Starting database server <<<" -echo "============================================================================" - -if [ "${DBTYPE}" == "mysqli" ] -then - - if [ "${DBSLAVES}" -ne 0 ] - then - export DBHOST_SLAVE="${DBHOST}_slave" - - echo "Starting master" - docker run \ - --detach \ - --name ${DBHOST} \ - --network "${NETWORK}" \ - -e MYSQL_ROOT_PASSWORD="${DBPASS}" \ - -e MYSQL_DATABASE="${DBNAME}" \ - -e MYSQL_USER="${DBUSER}" \ - -e MYSQL_PASSWORD="${DBPASS}" \ - -e DBHOST_SLAVE=$DBHOST_SLAVE \ - --tmpfs /var/lib/mysql:rw \ - -v $SCRIPTPATH/mysql.d/master/conf.d:/etc/mysql/conf.d \ - -v $SCRIPTPATH/mysql.d/master/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d \ - mysql:${DBTAG} - - echo "Starting slave" - docker run \ - --detach \ - --name ${DBHOST_SLAVE} \ - --network "${NETWORK}" \ - -e MYSQL_ROOT_PASSWORD="${DBPASS}" \ - -e MYSQL_DATABASE="${DBNAME}" \ - -e MYSQL_USER="${DBUSER}" \ - -e MYSQL_PASSWORD="${DBPASS}" \ - -e DBHOST=$DBHOST \ - -e DBHOST_SLAVE=$DBHOST_SLAVE \ - -v $SCRIPTPATH/mysql.d/slave/conf.d:/etc/mysql/conf.d \ - -v $SCRIPTPATH/mysql.d/slave/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d \ - --tmpfs /var/lib/mysql:rw \ - mysql:${DBTAG} - else - echo "Starting standalone" - docker run \ - --detach \ - --name ${DBHOST} \ - --network "${NETWORK}" \ - -e MYSQL_ROOT_PASSWORD="${DBPASS}" \ - -e MYSQL_DATABASE="${DBNAME}" \ - -e MYSQL_USER="${DBUSER}" \ - -e MYSQL_PASSWORD="${DBPASS}" \ - --tmpfs /var/lib/mysql:rw,noexec,nosuid,size=1024m \ - -v $SCRIPTPATH/mysql.d/standalone/conf.d:/etc/mysql/conf.d \ - mysql:${DBTAG} - fi - - export DBCOLLATION=utf8mb4_bin - - # Wait few sec, before executing commands. - sleep 20 - -elif [ "${DBTYPE}" == "mariadb" ] -then - if [ "${DBSLAVES}" != "0" ] - then - export DBHOST_SLAVE="${DBHOST}_slave" - - echo "Starting master" - docker run \ - --detach \ - --name ${DBHOST} \ - --network "${NETWORK}" \ - -e MYSQL_ROOT_PASSWORD="${DBPASS}" \ - -e MYSQL_DATABASE="${DBNAME}" \ - -e MYSQL_USER="${DBUSER}" \ - -e MYSQL_PASSWORD="${DBPASS}" \ - -e DBHOST_SLAVE=$DBHOST_SLAVE \ - --tmpfs /var/lib/mysql:rw \ - -v $SCRIPTPATH/mariadb.d/master/conf.d:/etc/mysql/conf.d \ - -v $SCRIPTPATH/mariadb.d/master/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d \ - mariadb:${DBTAG} - - echo "Starting slave" - docker run \ - --detach \ - --name ${DBHOST_SLAVE} \ - --network "${NETWORK}" \ - -e MYSQL_ROOT_PASSWORD="${DBPASS}" \ - -e MYSQL_DATABASE="${DBNAME}" \ - -e MYSQL_USER="${DBUSER}" \ - -e MYSQL_PASSWORD="${DBPASS}" \ - -e DBHOST=$DBHOST \ - -e DBHOST_SLAVE=$DBHOST_SLAVE \ - -v $SCRIPTPATH/mariadb.d/slave/conf.d:/etc/mysql/conf.d \ - -v $SCRIPTPATH/mariadb.d/slave/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d \ - --tmpfs /var/lib/mysql:rw \ - mariadb:${DBTAG} - else - echo "Starting standalone" - docker run \ - --detach \ - --name ${DBHOST} \ - --network "${NETWORK}" \ - -e MYSQL_ROOT_PASSWORD="${DBPASS}" \ - -e MYSQL_DATABASE="${DBNAME}" \ - -e MYSQL_USER="${DBUSER}" \ - -e MYSQL_PASSWORD="${DBPASS}" \ - --tmpfs /var/lib/mysql:rw,noexec,nosuid,size=1024m \ - -v $SCRIPTPATH/mariadb.d/standalone/conf.d:/etc/mysql/conf.d \ - mariadb:${DBTAG} - fi - export DBCOLLATION=utf8mb4_bin - - # Wait few sec, before executing commands. - sleep 20 - -elif [ "${DBTYPE}" == "oci" ] -then - # Need to adjust how we use tmpfs database depending on the database tag. - # For newer versions, do this (no tmpfs, but apply system settings - it's impossible to have both together). - tmpfsinit= - tmpfsmount= - if [ "${DBTAG}" == "11" ] - then - tmpfsinit="-v $SCRIPTPATH/oracle.d/tmpfs.sh:/docker-entrypoint-initdb.d/tmpfs.sh" - tmpfsmount="--tmpfs /var/lib/oracle --shm-size=2g" - else - # Let's try to mount the whole (XE) database available using tmpfs and - # use it. Note that we could use the pluggable XEPDB1 one, but we have run - # tests and there isn't any real benefit doing that. So we are going to - # continue using the XE database (CDB for Oracle 21c and up) always (unless - # we find in the future some other trick to make the PDB to perform better). - tmpfsmount="--mount type=tmpfs,destination=/opt/oracle/oradata --shm-size=6g" - fi - docker run \ - --detach \ - --name ${DBHOST} \ - --network "${NETWORK}" \ - ${tmpfsinit} ${tmpfsmount} \ - -e ORACLE_DISABLE_ASYNCH_IO=true \ - moodlehq/moodle-db-oracle-r2:${DBTAG} - - # Wait few sec, before executing commands. - sleep 140 - - export DBPASS="m@0dl3ing" - export DBNAME="XE" - -elif [ "${DBTYPE}" == "mssql" ] || [ "${DBTYPE}" == "sqlsrv" ] -then - - export DBUSER="sa" - export DBPASS="Passw0rd!" - - docker run \ - --detach \ - --name ${DBHOST} \ - --network "${NETWORK}" \ - -e ACCEPT_EULA=Y \ - -e SA_PASSWORD="${DBPASS}" \ - moodlehq/moodle-db-mssql:${DBTAG} - - # Wait few sec, before executing commands. - sleep 10 - -elif [ "${DBTYPE}" == "pgsql" ] -then - if [ "${DBSLAVES}" -ne 0 ] - then - export DBHOST_SLAVE="${DBHOST}_slave" - - echo "Starting master" - docker run \ - --detach \ - --name ${DBHOST} \ - --network "${NETWORK}" \ - -e POSTGRES_DB="${DBNAME}" \ - -e POSTGRES_USER=moodle \ - -e POSTGRES_PASSWORD=moodle \ - -e DBHOST_SLAVE=$DBHOST_SLAVE \ - --tmpfs /var/lib/postgresql/data:rw \ - -v $SCRIPTPATH/pgsql.d/master:/docker-entrypoint-initdb.d \ - postgres:${DBTAG} - - # Wait few sec, before executing commands. - sleep 10 - - echo "Starting slave" - docker run \ - --detach \ - --name ${DBHOST_SLAVE} \ - --network "${NETWORK}" \ - -e POSTGRES_DB="${DBNAME}" \ - -e POSTGRES_USER=moodle \ - -e POSTGRES_PASSWORD=moodle \ - -e DBHOST=$DBHOST \ - -e DBHOST_SLAVE=$DBHOST_SLAVE \ - --tmpfs /var/lib/postgresql/data:rw \ - -v $SCRIPTPATH/pgsql.d/slave:/docker-entrypoint-initdb.d \ - postgres:${DBTAG} - else - echo "Starting standalone" - docker run \ - --detach \ - --name ${DBHOST} \ - --network "${NETWORK}" \ - -e POSTGRES_DB="${DBNAME}" \ - -e POSTGRES_USER=moodle \ - -e POSTGRES_PASSWORD=moodle \ - --tmpfs /var/lib/postgresql/data:rw,noexec,nosuid,size=1024m \ - -v $SCRIPTPATH/pgsql.d/standalone:/docker-entrypoint-initdb.d \ - postgres:${DBTAG} - fi - - # Wait few sec, before executing commands for all nodes to come up. - sleep 10 - -else - - echo "Unknown database type ${DBTYPE}" - exit 255 - -fi - -echo "============================================================================" -echo ">>> stopsection <<<" - -echo -echo ">>> startsection Database summary <<<" -echo "============================================================================" -echo "== DBTORUN: ${DBTORUN}" -echo "== DBTYPE: ${DBTYPE}" -echo "== DBTAG: ${DBTAG}" -echo "== DBHOST: ${DBHOST}" -echo "== DBHOST_SLAVE: ${DBHOST_SLAVE}" -echo "== DBUSER: ${DBUSER}" -echo "== DBPASS: ${DBPASS}" -echo "== DBNAME: ${DBNAME}" - -docker logs "${DBHOST}" - -if [ "${DBHOST_SLAVE}" != "" ] -then - echo - echo ">>> startsection Database slave summary <<<" - echo "============================================================================" - docker logs "${DBHOST_SLAVE}" - echo "============================================================================" - echo ">>> stopsection <<<" -fi - -echo "============================================================================" -echo ">>> stopsection <<<" - -echo -echo ">>> startsection Starting supplemental services <<<" -echo "============================================================================" -BBBMOCK=bbbmock"${UUID}" -docker run \ - --detach \ - --name ${BBBMOCK} \ - --network "${NETWORK}" \ - moodlehq/bigbluebutton_mock:latest - -export BBBMOCKURL="http://${BBBMOCK}" -echo BBBMOCKURL >> "${ENVIROPATH}" -docker logs ${BBBMOCK} - -MATRIXMOCK=matrixmock"${UUID}" -docker run \ - --detach \ - --name ${MATRIXMOCK} \ - --network "${NETWORK}" \ - moodlehq/matrixsynapse_mock:latest - -export MATRIXMOCKURL="http://${MATRIXMOCK}" -echo MATRIXMOCKURL >> "${ENVIROPATH}" -docker logs ${MATRIXMOCK} - -if [ "${TESTTORUN}" == "phpunit" ] -then - EXTTESTNAME=exttests"${UUID}" - - docker run \ - --detach \ - --name ${EXTTESTNAME} \ - --network "${NETWORK}" \ - moodlehq/moodle-exttests:latest - - export EXTTESTURL="http://${EXTTESTNAME}" - echo EXTTESTURL >> "${ENVIROPATH}" - docker logs ${EXTTESTNAME} - - - LDAPTESTNAME=ldap"${UUID}" - - docker run \ - --detach \ - --name ${LDAPTESTNAME} \ - --network "${NETWORK}" \ - larrycai/openldap - - export LDAPTESTURL="ldap://${LDAPTESTNAME}" - echo LDAPTESTURL >> "${ENVIROPATH}" - docker logs ${LDAPTESTNAME} - - - export SOLRTESTNAME=solr"${UUID}" - docker run \ - --detach \ - --name ${SOLRTESTNAME} \ - --network "${NETWORK}" \ - solr:7 \ - solr-precreate test - - echo SOLRTESTNAME >> "${ENVIROPATH}" - docker logs ${SOLRTESTNAME} - - if [ -n "${MLBACKEND_PYTHON_VERSION}" ] - then - export MLBACKENDTESTNAME=mlpython"${UUID}" - docker run \ - --detach \ - --name ${MLBACKENDTESTNAME} \ - --network "${NETWORK}" \ - moodlehq/moodle-mlbackend-python:${MLBACKEND_PYTHON_VERSION} - - echo MLBACKENDTESTNAME >> "${ENVIROPATH}" - docker logs ${MLBACKENDTESTNAME} - fi - - export REDISTESTNAME=redis"${UUID}" - docker run \ - --detach \ - --name ${REDISTESTNAME} \ - --network "${NETWORK}" \ - redis:3 - - echo REDISTESTNAME >> "${ENVIROPATH}" - docker logs ${REDISTESTNAME} - - - MEMCACHED1TESTNAME=memcached1"${UUID}" - docker run \ - --detach \ - --name ${MEMCACHED1TESTNAME} \ - --network "${NETWORK}" \ - memcached:1.4 - - export MEMCACHED1TESTURL="${MEMCACHED1TESTNAME}:11211" - echo MEMCACHED1TESTURL >> "${ENVIROPATH}" - docker logs ${MEMCACHED1TESTNAME} - - - MEMCACHED2TESTNAME=memcached2"${UUID}" - docker run \ - --detach \ - --name ${MEMCACHED2TESTNAME} \ - --network "${NETWORK}" \ - memcached:1.4 - - export MEMCACHED2TESTURL="${MEMCACHED2TESTNAME}:11211" - echo MEMCACHED2TESTURL >> "${ENVIROPATH}" - docker logs ${MEMCACHED2TESTNAME} - - - MONGODBTESTNAME=mongodb"${UUID}" - docker run \ - --detach \ - --name ${MONGODBTESTNAME} \ - --network "${NETWORK}" \ - mongo:4.0 - - export MONGODBTESTURL="mongodb://${MONGODBTESTNAME}:27017" - echo MONGODBTESTURL >> "${ENVIROPATH}" - docker logs ${MONGODBTESTNAME} -fi - -echo "============================================================================" -echo ">>> stopsection <<<" - -if [ "$TESTTORUN" == "behat" ] +if [ "$JOBTYPE" == "behat" ] then echo echo ">>> startsection Starting selenium server <<<" @@ -806,88 +296,21 @@ then echo ">>> stopsection <<<" fi -# Start the test server. -echo -echo ">>> startsection Starting web server <<<" -echo "============================================================================" -export WEBSERVER=webserver"${UUID}" -docker run \ - --network "${NETWORK}" \ - --name "${WEBSERVER}" \ - --detach \ - --env-file "${ENVIROPATH}" \ - -v "${COMPOSERCACHE}:/var/www/.composer:rw" \ - -v "${OUTPUTDIR}":/shared \ - ${PHP_SERVER_DOCKER} - -# Copy code in place. -echo "== Copying code in place" -docker cp "${CODEDIR}"/. "${WEBSERVER}":/var/www/html -if [ -n "$PLUGINSTOINSTALL" ]; -then - echo "== Copying external plugins in place" - docker cp "${PLUGINSDIR}"/. "${WEBSERVER}":/var/www/html -fi -# Copy the config.php in place -echo "== Copying configuration" -docker cp "${SCRIPTPATH}/config.template.php" "${WEBSERVER}":/var/www/html/config.php -COMPOSERPHAR="${COMPOSERCACHE}/composer.phar" -if [ -f "${COMPOSERPHAR}" ] -then - docker cp "${COMPOSERPHAR}" "${WEBSERVER}":/var/www/html/composer.phar - docker exec -t "${WEBSERVER}" bash -c 'chown -R www-data:www-data /var/www/html/composer.phar' -fi -echo "============================================================================" -docker logs "${WEBSERVER}" -echo "============================================================================" -echo ">>> stopsection <<<" -echo -echo ">>> startsection Waiting for all containers to become healthy<<<" -echo "============================================================================" -for waitperiod in {0..90} -do - # Note we cannot use the 'health' filter due to https://github.com/moby/moby/issues/35920 - startingcount=$((`docker ps -a --filter name=${UUID} | grep -e starting -e unhealthy | wc -l`)) - if [[ ${startingcount} -lt 1 ]] - then - break - fi - echo "Waiting for ${startingcount} containers to become healthy" - sleep 1 -done -startingcount=$((`docker ps -a --filter name=${UUID} | grep -e starting -e unhealthy | wc -l`)) -if [[ ${startingcount} -gt 0 ]] -then - echo "Some containers were too slow. Aborting the run:" - docker ps -a --filter name=${UUID} --filter | grep -e starting -e unhealthy - exit 1 -fi -echo "All containers started" -echo "============================================================================" -echo ">>> stopsection <<<" -# Prepare the summary of images being used by the run (creation date & digest) -echo -echo ">>> startsection Details about the images being used by the run<<<" -echo "============================================================================" -docker ps --filter "name=${UUID}" --format='{{.Image}}' | sort | uniq | xargs -I{} \ - docker image inspect \ - --format '{} {{if .Created}}created:{{.Created}}{{end}} {{if .RepoDigests}}{{index .RepoDigests 0}}{{end}}' {} | \ - tr '@' ' ' | cut -f1,2,4 -d' ' -echo "============================================================================" -echo ">>> stopsection <<<" -# Setup the DB. -echo -echo ">>> startsection Initialising test environment<<<" -echo "============================================================================" -if [ "$TESTTORUN" == "behat" ] + + + + + +# INIT +if [ "$JOBTYPE" == "behat" ] then BEHAT_INIT_SUITE="" BEHAT_RUN_SUITE="" @@ -912,16 +335,14 @@ then ${BEHAT_INIT_SUITE} \ --axe \ -j="${BEHAT_TOTAL_RUNS}" -else - docker exec -t -u www-data "${WEBSERVER}" \ - php admin/tool/phpunit/cli/init.php \ - --force fi echo "============================================================================" -echo ">>> stopsection <<<" + + + # Run the test. -if [ "$TESTTORUN" == "behat" ] +if [ "$JOBTYPE" == "behat" ] then echo @@ -945,9 +366,9 @@ then TAGS="--tags=${TAGS}" fi - if [ -n "${NAME}" ] + if [ -n "${BEHAT_NAME}" ] then - NAME="--name=${NAME}" + NAME="--name=${BEHAT_NAME}" fi CMD=(php admin/tool/behat/cli/run.php @@ -994,11 +415,11 @@ then if [ "$BEHAT_TOTAL_RUNS" -le 1 ] then # Was single - for RERUN in `seq 1 "${BEHAT_NUM_RERUNS}"` + for RERUN in $(seq 1 "${BEHAT_NUM_RERUNS}") do NEWEXITCODE=0 CONFIGPATH="/var/www/behatdata/run/behatrun/behat/behat.yml" - if [ "$MOODLE_VERSION" -lt "32" ] + if [ "$MOODLE_BRANCH" -lt "32" ] then CONFIGPATH="/var/www/behatdata/run/behat/behat.yml" fi @@ -1044,7 +465,7 @@ then fi CONFIGPATH="/var/www/behatdata/run/behatrun${RUN}/behat/behat.yml" - if [ "$MOODLE_VERSION" -lt "32" ] + if [ "$MOODLE_BRANCH" -lt "32" ] then CONFIGPATH="/var/www/behatdata/run${RUN}/behat/behat.yml" fi @@ -1090,75 +511,11 @@ then # Update the timing file if [ ! -z "$BEHAT_TIMING_FILENAME" ] then - cp "${OUTPUTDIR}"/timing.json "${TIMINGSOURCE}" - fi -else - - echo - echo ">>> startsection Starting phpunit run at $(date) <<<" - echo "============================================================================" - - if [ -n "${TAGS}" ] - then - PHPUNIT_FILTER="--filter ${TAGS}" - else - PHPUNIT_FILTER="" - fi - - if [ -n "${TESTSUITE}" ] - then - PHPUNIT_SUITE="--testsuite ${TESTSUITE}" - else - PHPUNIT_SUITE="" + cp "${SHAREDDIR}"/timing.json "${TIMINGSOURCE}" fi - CMD="php vendor/bin/phpunit" - CMD="${CMD} --disallow-test-output" - CMD="${CMD} --fail-on-risky" - CMD="${CMD} --log-junit /shared/log.junit" - CMD="${CMD} ${PHPUNIT_FILTER}" - CMD="${CMD} ${PHPUNIT_SUITE}" - CMD="${CMD} --verbose" - - ITER=0 - EXITCODE=0 - while [[ ${ITER} -lt ${RUNCOUNT} ]] - do - docker exec -t "${WEBSERVER}" ${CMD} - EXITCODE=$(($EXITCODE + $?)) - ITER=$(($ITER+1)) - done - - echo "============================================================================" - echo ">>> stopsection <<<" - fi -echo -echo ">>> startsection Cleaning workspace<<<" -echo "============================================================================" - -# Store the docker container logs. -docker ps -a --filter name=${UUID} -for container in `docker ps -a --format "{{.ID}}~{{.Names}}" --filter name=${UUID}` -do - image=$(echo $container | cut -d'~' -f1) - name=$(echo $container | cut -d'~' -f2) - name=${name%"${UUID}"} # Get rid of the UUID for naming log files. - echo "Exporting ${name} logs to ${OUTPUTDIR}/${name}.gz" - docker logs "${image}" 2>&1 | gzip > "${OUTPUTDIR}"/${name}.gz -done - -docker exec -t "${WEBSERVER}" \ - chown -R "${UID}:${GROUPS[0]}" /shared -echo "============================================================================" -echo ">>> stopsection <<<" -echo -echo "============================================================================" -echo "== Exit summary": -echo "== Exit code: ${EXITCODE}" -echo "============================================================================" -exit $EXITCODE